diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index db29d87ad..e79e034a2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,6 +23,62 @@ jobs: - name: Validate run: npm run validate + - name: Check path filters + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + wpt: + - 'src/technologies/*.json' + - 'src/categories.json' + - 'src/groups.json' + - 'tests/*.js' + - '.github/workflows/wpt.yml' + + - name: Parse WebPageTest URLs + if: steps.filter.outputs.wpt == 'true' + id: wpt-urls + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + lines=() + while IFS= read -r line; do + lines+=("${line//[$'\r\n']}") + done <<< "$PR_BODY" + + start_index=-1 + for ((i=0; i<${#lines[@]}; i++)); do + if [[ "${lines[$i]}" == *"**Test websites**:"* ]]; then + start_index=$((i + 1)) + break + fi + done + + URLS="" + if [ $start_index -gt -1 ]; then + url_pattern="(https:\/\/[^[:space:]]+)" + for ((i=start_index; i<${#lines[@]}; i++)); do + if [[ ${lines[$i]} =~ $url_pattern ]]; then + URLS+="${BASH_REMATCH[1]} " + fi + done + fi + echo "urls=$URLS" >> $GITHUB_OUTPUT + + - name: Save PR info + if: steps.filter.outputs.wpt == 'true' + run: | + mkdir -p pr-metadata + echo "${{ github.event.pull_request.number }}" > pr-metadata/pr_number + echo "${{ steps.wpt-urls.outputs.urls }}" > pr-metadata/urls + + - name: Upload PR Metadata + if: steps.filter.outputs.wpt == 'true' + uses: actions/upload-artifact@v4 + with: + name: pr-metadata + path: pr-metadata/ + dependabot: name: Dependabot auto-merge runs-on: ubuntu-slim diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 5afd86bb1..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Tests - -on: - pull_request_target: - branches: - - main - paths: - - "src/technologies/*.json" - - "src/categories.json" - - "src/groups.json" - - "tests/*.js" - - ".github/workflows/test.yml" - workflow_dispatch: - -jobs: - test: - name: WebPageTest Test Cases - runs-on: ubuntu-slim - permissions: - contents: read - pull-requests: write - if: | - ${{ github.repository == 'HTTPArchive/wappalyzer' || - github.event.pull_request.base.repo.full_name == 'HTTPArchive/wappalyzer' }} - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install dependencies - run: npm install - - - name: Validate - run: npm run validate - - - name: Run WebPageTest with unit tests - id: unit-test - env: - WPT_SERVER: "webpagetest.httparchive.org" - WPT_API_KEY: ${{ secrets.HA_API_KEY }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: npm run test - - - name: Run WebPageTest for more websites - id: wpt-test - env: - WPT_SERVER: "webpagetest.httparchive.org" - WPT_API_KEY: ${{ secrets.HA_API_KEY }} - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_BODY: ${{ github.event.pull_request.body }} - run: | - # Read PR body into an array, removing line breaks and carriage returns - lines=() - while IFS= read -r line; do - lines+=("${line//[$'\r\n']}") - done <<< "$PR_BODY" - - # Find the index of the line after "**Test websites**:" - start_index=-1 - for ((i=0; i<${#lines[@]}; i++)); do - if [[ "${lines[$i]}" == *"**Test websites**:"* ]]; then - start_index=$((i + 1)) - break - fi - done - - # If the index is valid, then parse the URLs - if [ $start_index -gt -1 ]; then - # Initialize an array for URLs - URLS=() - url_pattern="(https:\/\/[^[:space:]]+)" - - for ((i=start_index; i<${#lines[@]}; i++)); do - if [[ ${lines[$i]} =~ $url_pattern ]]; then - URLS+=("${BASH_REMATCH[1]}") - fi - done - - # Run WebPageTest for each URL - for TEST_WEBSITE in "${URLS[@]}"; do - echo "::group::Running WPT test for $TEST_WEBSITE" - node tests/wpt.js "$TEST_WEBSITE" - echo "::endgroup::" - done - else - echo "No websites found." - fi - - - name: Add comment with results - uses: mshick/add-pr-comment@v3 - if: steps.unit-test.outcome == 'success' || steps.wpt-test.outcome == 'success' - with: - refresh-message-position: true - message-path: test-results.md diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml new file mode 100644 index 000000000..326facbdf --- /dev/null +++ b/.github/workflows/wpt.yml @@ -0,0 +1,97 @@ +name: WPT Run & Comment + +on: + workflow_run: + workflows: ["Lint, validate & Dependabot auto-merge"] + types: [completed] + +jobs: + wpt: + name: Run WebPageTest + runs-on: ubuntu-slim + if: ${{ github.event.workflow_run.conclusion == 'success' }} + permissions: + contents: read + pull-requests: write + actions: read + steps: + - name: Checkout Trusted Code + uses: actions/checkout@v7 + + - name: Download PR Metadata + uses: actions/download-artifact@v4 + id: download-artifact + continue-on-error: true + with: + name: pr-metadata + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: ${{ runner.temp }}/pr-metadata + + - name: Check if WPT run is needed + id: check-run + run: | + if [ "${{ steps.download-artifact.outcome }}" = "success" ] && [ -f "${{ runner.temp }}/pr-metadata/pr_number" ]; then + PR_NUMBER_RAW="$(cat "${{ runner.temp }}/pr-metadata/pr_number")" + if ! printf '%s' "$PR_NUMBER_RAW" | grep -Eq '^[0-9]+$'; then + echo "run_wpt=false" >> "$GITHUB_OUTPUT" + echo "Invalid PR number metadata." + exit 0 + fi + + WPT_URLS_RAW="" + if [ -f "${{ runner.temp }}/pr-metadata/urls" ]; then + WPT_URLS_RAW="$(cat "${{ runner.temp }}/pr-metadata/urls")" + fi + + # Prevent output/env-file injection by stripping CR/LF from untrusted artifact content. + WPT_URLS_SAFE="$(printf '%s' "$WPT_URLS_RAW" | tr '\r\n' ' ')" + + echo "run_wpt=true" >> "$GITHUB_OUTPUT" + printf 'pr_number=%s\n' "$PR_NUMBER_RAW" >> "$GITHUB_OUTPUT" + printf 'wpt_urls=%s\n' "$WPT_URLS_SAFE" >> "$GITHUB_OUTPUT" + echo "Starting WPT run." + else + echo "run_wpt=false" >> "$GITHUB_OUTPUT" + echo "No WPT run needed." + fi + + - name: Install dependencies + if: steps.check-run.outputs.run_wpt == 'true' + run: npm install + + - name: Run WebPageTest with unit tests + if: steps.check-run.outputs.run_wpt == 'true' + id: unit-test + env: + WPT_SERVER: "webpagetest.httparchive.org" + WPT_API_KEY: ${{ secrets.HA_API_KEY }} + PR_NUMBER: ${{ steps.check-run.outputs.pr_number }} + run: npm run test + + - name: Run WebPageTest for more websites + if: steps.check-run.outputs.run_wpt == 'true' + id: wpt-test + env: + WPT_SERVER: "webpagetest.httparchive.org" + WPT_API_KEY: ${{ secrets.HA_API_KEY }} + PR_NUMBER: ${{ steps.check-run.outputs.pr_number }} + WPT_URLS: ${{ steps.check-run.outputs.wpt_urls }} + run: | + if [ -n "$WPT_URLS" ]; then + for TEST_WEBSITE in $WPT_URLS; do + echo "::group::Running WPT test for $TEST_WEBSITE" + node tests/wpt.js "$TEST_WEBSITE" + echo "::endgroup::" + done + else + echo "No websites found." + fi + + - name: Add comment with results + if: steps.check-run.outputs.run_wpt == 'true' && (steps.unit-test.outcome == 'success' || steps.wpt-test.outcome == 'success') + uses: mshick/add-pr-comment@v3 + with: + refresh-message-position: true + message-path: test-results.md + issue: ${{ steps.check-run.outputs.pr_number }}