diff --git a/templates/github/.github/workflows/bleeding_ci.yml.j2 b/templates/github/.github/workflows/bleeding_ci.yml.j2 new file mode 100644 index 00000000..41a88799 --- /dev/null +++ b/templates/github/.github/workflows/bleeding_ci.yml.j2 @@ -0,0 +1,71 @@ +{% include 'header.j2' %} +{% from 'macros.j2' import + matrix_env, + pulpcore_ref_input, +with context %} +{%- set bleeding_edge_plugins = ['pulp_ansible', 'pulp_container', 'pulp_deb', 'pulp_python', 'pulp_rpm'] %} +--- +name: "{{ plugin_app_label | camel }} Bleeding Edge CI" +on: + workflow_dispatch: + inputs: + {%- if plugin_name == "pulpcore" %} + {%- set extra_description = "This will trigger bleeding edge CI on: " ~ bleeding_edge_plugins | join(", ") ~ "." %} + {{ pulpcore_ref_input(required=True, extra_description=extra_description) | indent(6) }} + {%- else %} + {{ pulpcore_ref_input(required=True) | indent(6) }} + {%- endif %} + +concurrency: + {%- raw %} + group: "${{ github.ref_name }}-${{ github.workflow }}" + {%- endraw %} + cancel-in-progress: true + +{%- if plugin_name == "pulpcore" %} + +jobs: + dispatch: + runs-on: "ubuntu-latest" + steps: + - name: "Trigger Plugin Workflows" + uses: "actions/github-script@v7" + with: + github-token: "{{ '${{ secrets.ORG_DISPATCH_TOKEN }}' }}" + script: | + const plugins = [{% for p in bleeding_edge_plugins %}'{{ p }}'{% if not loop.last %}, {% endif %}{% endfor %}]; + const owner = context.repo.owner; + + for (const repo of plugins) { + console.log(`Triggering bleeding edge CI in ${repo}...`); + await github.rest.actions.createWorkflowDispatch({ + owner: owner, + repo: repo, + workflow_id: 'bleeding_ci.yml', + ref: 'main', + inputs: { + pulpcore_ref: "{{ '${{ inputs.pulpcore_ref }}' }}" + } + }); + } +{%- else %} + +jobs: + build: + uses: "./.github/workflows/build.yml" + with: + {%- raw %} + pulpcore_ref: "${{ inputs.pulpcore_ref }}" + {%- endraw %} + + test: + needs: "build" + uses: "./.github/workflows/test.yml" + with: + matrix_env: | + {{ matrix_env(lowerbounds=false) | from_yaml | tojson }} + {%- raw %} + pulpcore_ref: "${{ inputs.pulpcore_ref }}" + {%- endraw %} +{%- endif %} +... diff --git a/templates/github/.github/workflows/build.yml.j2 b/templates/github/.github/workflows/build.yml.j2 index 1041015e..24daca4c 100644 --- a/templates/github/.github/workflows/build.yml.j2 +++ b/templates/github/.github/workflows/build.yml.j2 @@ -1,14 +1,20 @@ {% include 'header.j2' %} {% from 'macros.j2' import checkout, - setup_python, install_python_deps, + pulpcore_ref_input, + resolve_pulpcore_ref, run_script, + setup_python, with context %} --- name: "Build" on: workflow_call: + {%- if plugin_name != "pulpcore" %} + inputs: + {{ pulpcore_ref_input() | indent(6) }} + {%- endif %} defaults: run: @@ -35,7 +41,13 @@ jobs: twine check dist/* - name: "Install built packages" run: | - pip install dist/{{ plugin_name | snake }}-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt + {%- if plugin_name != "pulpcore" %} + BLEEDING_PULPCORE_REF="{{ '${{ inputs.pulpcore_ref }}' }}" + if [ -n "$BLEEDING_PULPCORE_REF" ]; then + {{ resolve_pulpcore_ref() | indent(12) }} + fi + {%- endif %} + pip install dist/{{ plugin_name | snake }}-*-py3-none-any.whl{% if plugin_name != "pulpcore" %} ${PULPCORE_SOURCE:-}{% endif %} -c .ci/assets/ci_constraints.txt - name: "Generate api specs" run: | pulpcore-manager openapi --file "api.json" diff --git a/templates/github/.github/workflows/scripts/before_install.sh.j2 b/templates/github/.github/workflows/scripts/before_install.sh.j2 index 4ea5a4fe..76596e0f 100755 --- a/templates/github/.github/workflows/scripts/before_install.sh.j2 +++ b/templates/github/.github/workflows/scripts/before_install.sh.j2 @@ -1,12 +1,20 @@ #!/usr/bin/env bash {% include 'header.j2' %} +{% from 'macros.j2' import resolve_pulpcore_ref with context %} # This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file. # # It requires the following environment: # TEST - The name of the scenario to prepare. # +{%- if plugin_name != "pulpcore" %} +# Optional environment: +# BLEEDING_PULPCORE_REF - A pulpcore branch, commit SHA, or PR number to test against. +# When set, installs pulpcore from source and removes version pins +# so the custom ref can be installed without constraint conflicts. +# +{%- endif %} # It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario. set -eu -o pipefail @@ -44,7 +52,13 @@ fi if [[ "$TEST" = "lowerbounds" ]]; then python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt fi - +{% if plugin_name != "pulpcore" %} +# Relax constraints if a custom pulpcore is provided +if [ -n "${BLEEDING_PULPCORE_REF:-}" ]; then + {{ resolve_pulpcore_ref() | indent(2) }} + echo "$PULPCORE_SOURCE" > ci_requirements.txt +fi +{% endif %} # Compose the scenario definition. mkdir -p .ci/ansible/vars diff --git a/templates/github/.github/workflows/scripts/before_script.sh.j2 b/templates/github/.github/workflows/scripts/before_script.sh.j2 index b3828577..8bc5bb7c 100755 --- a/templates/github/.github/workflows/scripts/before_script.sh.j2 +++ b/templates/github/.github/workflows/scripts/before_script.sh.j2 @@ -34,6 +34,12 @@ echo echo "# Constraints Files:" # They need not even exist. tail -v -n +1 ../*/*constraints.txt || true +{%- if plugin_name != "pulpcore" %} + +echo +echo "# ci_requirements.txt:" +tail -v -n +1 ci_requirements.txt || true +{%- endif %} echo echo "# pip list outside the container" diff --git a/templates/github/.github/workflows/test.yml.j2 b/templates/github/.github/workflows/test.yml.j2 index dac22fcb..b03e1df0 100644 --- a/templates/github/.github/workflows/test.yml.j2 +++ b/templates/github/.github/workflows/test.yml.j2 @@ -4,6 +4,7 @@ display_logs, install_python_deps, install_uv, + pulpcore_ref_input, run_script, setup_env, setup_python, @@ -16,6 +17,9 @@ on: matrix_env: required: true type: "string" + {%- if plugin_name != "pulpcore" %} + {{ pulpcore_ref_input() | indent(6) }} + {%- endif %} defaults: run: @@ -74,7 +78,11 @@ jobs: {{ setup_env() | indent(6) }} - {{ run_script(name="Prepare Scenario Definition", file="before_install.sh") | indent(6) }} + {{ run_script( + name="Prepare Scenario Definition", + file="before_install.sh", + with_bleeding_pulpcore_ref=(plugin_name != "pulpcore"), + ) | indent(6) }} {{ run_script(name="Install", file="install.sh") | indent(6) }} diff --git a/templates/include/macros.j2 b/templates/include/macros.j2 index 76477dc5..518d4f78 100644 --- a/templates/include/macros.j2 +++ b/templates/include/macros.j2 @@ -1,4 +1,7 @@ -{%- macro set_env_vars(extra_env=None) -%} +{%- macro set_env_vars( + extra_env=None, + with_bleeding_pulpcore_ref=False +) -%} PY_COLORS: "1" ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "{{ '${{ secrets.GITHUB_TOKEN }}' }}" @@ -6,6 +9,9 @@ GITHUB_CONTEXT: "{{ '${{ github.event.pull_request.commits_url }}' }}" {%- for key, value in ci_env.items() %} {{ key }}: "{{ value }}" {%- endfor %} +{%- if with_bleeding_pulpcore_ref %} +BLEEDING_PULPCORE_REF: "{{ '${{ inputs.pulpcore_ref }}' }}" +{%- endif %} {%- if extra_env is not none %} {%- for key, value in extra_env.items() %} {{ key }}: "{{ value }}" @@ -99,7 +105,42 @@ GITHUB_CONTEXT: "{{ '${{ github.event.pull_request.commits_url }}' }}" {%- endmacro -%} -{%- macro run_script(name, file, withenv=True, condition=None, extra_env=None, path=None) -%} +{%- macro pulpcore_ref_input(required=False, extra_description=None) -%} +pulpcore_ref: + # WARNING: PRs should always be tested against released pulpcore versions. + # This is intended to be used in the add-hoc bleeding_ci workflow + description: > + pulpcore ref to test against: branch name, commit SHA, or PR number. + + PR numbers are automatically converted to refs/pull//head. + {%- if extra_description %} + + {{ extra_description }} + {%- endif %} + required: {{ "true" if required else "false" }} + type: "string" + {%- if not required %} + default: "" + {%- endif %} +{%- endmacro -%} + + +{%- macro resolve_pulpcore_ref(output_var="PULPCORE_SOURCE") -%} +if [[ "$BLEEDING_PULPCORE_REF" =~ ^[0-9]+$ ]]; then + BLEEDING_PULPCORE_REF="refs/pull/${BLEEDING_PULPCORE_REF}/head" +fi +{{ output_var }}="git+https://github.com/pulp/pulpcore.git@${BLEEDING_PULPCORE_REF}" +sed -i '/^pulpcore/d' .ci/assets/ci_constraints.txt +if [ -f upperbounds_constraints.txt ]; then + sed -i '/^pulpcore/d' upperbounds_constraints.txt +fi +{%- endmacro -%} + + +{%- macro run_script( + name, file, withenv=True, condition=None, extra_env=None, path=None, + with_bleeding_pulpcore_ref=False +) -%} - name: "{{ name }}" {%- if path %} working-directory: "{{ path }}" @@ -112,7 +153,10 @@ GITHUB_CONTEXT: "{{ '${{ github.event.pull_request.commits_url }}' }}" shell: "bash" {%- if withenv %} env: - {{ set_env_vars(extra_env) | indent(4) }} + {{ set_env_vars( + extra_env, + with_bleeding_pulpcore_ref=with_bleeding_pulpcore_ref, + ) | indent(4) }} {%- endif %} {%- endmacro -%} @@ -140,7 +184,7 @@ GITHUB_CONTEXT: "{{ '${{ github.event.pull_request.commits_url }}' }}" {%- endmacro -%} -{%- macro matrix_env(performance=false) -%} +{%- macro matrix_env(performance=false, lowerbounds=true) -%} --- - TEST: "pulp" {%- if test_azure %} @@ -152,7 +196,7 @@ GITHUB_CONTEXT: "{{ '${{ github.event.pull_request.commits_url }}' }}" {%- if test_s3 %} - TEST: "s3" {%- endif %} -{%- if test_lowerbounds %} +{%- if lowerbounds and test_lowerbounds %} - TEST: "lowerbounds" {%- endif %} {%- if performance and test_performance %}