Add caching to query compilation and dependencies installation

.github/actions/install-codeql-packs/action.yml

@@ -7,6 +7,11 @@ inputs:
       The path to the CodeQL CLI directory.
     required: false
 
+  common_caches:
+    description: |
+      The path to the CodeQL common caches directory.
+    required: false
+
   mode:
     description: |
       The `--mode` option to `codeql pack install`.
@@ -22,4 +27,6 @@ runs:
         CODEQL_CLI: ${{ inputs.cli_path }}
       run: |
         PATH=$PATH:$CODEQL_CLI
-        python scripts/install-packs.py --mode ${{ inputs.mode }}
+        python scripts/install-packs.py \
+          --mode ${{ inputs.mode }} \
+          --common-caches "${{ inputs.common_caches }}"

.github/workflows/code-scanning-pack-gen.yml

@@ -20,7 +20,6 @@ env:
   XARGS_MAX_PROCS: 4
 
 jobs:
-
   prepare-code-scanning-pack-matrix:
     name: Prepare CodeQL Code Scanning pack matrix
     runs-on: ubuntu-22.04
@@ -62,10 +61,19 @@ jobs:
           codeql-home: ${{ github.workspace }}/codeql_home
           add-to-path: false
 
+      - name: Cache queries compilation
+        id: cache-queries-compilation
+        uses: actions/cache/restore@v6
+        with:
+          path: ${{ github.workspace }}/codeql_cache
+          key: ${{ runner.os }}-codeql-compilation-code-scanning-pack-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-codeql-compilation-code-scanning-pack-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}
+
       - name: Install CodeQL packs
         uses: ./.github/actions/install-codeql-packs
         with:
           cli_path: ${{ github.workspace }}/codeql_home/codeql
+          common_caches: ${{ github.workspace }}/codeql_cache
 
       - name: Determine ref for external help files
         id: determine-ref
@@ -105,11 +113,22 @@ jobs:
           PATH=$PATH:$CODEQL_HOME/codeql
           # Precompile all queries, and use a compilation cache larger than default
           # to ensure we cache all the queries for later steps
-          codeql query compile --precompile --threads 0 --compilation-cache-size=1024 cpp c
+          codeql query compile \
+            --common-caches=${{ github.workspace }}/codeql_cache \
+            --precompile \
+            --threads 0 \
+            --compilation-cache-size=1024 \
+            cpp c
 
           cd ..
           zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/schemas
 
+      - name: Save queries compilation cache
+        uses: actions/cache/save@v6
+        with:
+          path: ${{ github.workspace }}/codeql_cache
+          key: ${{ steps.cache-queries-compilation.outputs.cache-primary-key }}
+
       - name: Upload GHAS Query Pack
         uses: actions/upload-artifact@v7
         with:
@@ -119,20 +138,21 @@ jobs:
       - name: Create qlpack bundles
         env:
           CODEQL_HOME: ${{ github.workspace }}/codeql_home
+          CODEQL_CACHE: ${{ github.workspace }}/codeql_cache
         run: |
           PATH=$PATH:$CODEQL_HOME/codeql
 
-          codeql pack bundle --output=common-cpp-coding-standards.tgz cpp/common/src
-          codeql pack bundle --output=common-c-coding-standards.tgz c/common/src
-          codeql pack bundle --output=misra-c-coding-standards.tgz c/misra/src
-          codeql pack bundle --output=cert-c-coding-standards.tgz c/cert/src
-          codeql pack bundle --output=cert-cpp-coding-standards.tgz cpp/cert/src
-          codeql pack bundle --output=autosar-cpp-coding-standards.tgz cpp/autosar/src
-          codeql pack bundle --output=misra-cpp-coding-standards.tgz cpp/misra/src
-          codeql pack bundle --output=report-coding-standards.tgz cpp/report/src
+          codeql pack bundle --common-caches="$CODEQL_CACHE" --output=common-cpp-coding-standards.tgz cpp/common/src
+          codeql pack bundle --common-caches="$CODEQL_CACHE" --output=common-c-coding-standards.tgz c/common/src
+          codeql pack bundle --common-caches="$CODEQL_CACHE" --output=misra-c-coding-standards.tgz c/misra/src
+          codeql pack bundle --common-caches="$CODEQL_CACHE" --output=cert-c-coding-standards.tgz c/cert/src
+          codeql pack bundle --common-caches="$CODEQL_CACHE" --output=cert-cpp-coding-standards.tgz cpp/cert/src
+          codeql pack bundle --common-caches="$CODEQL_CACHE" --output=autosar-cpp-coding-standards.tgz cpp/autosar/src
+          codeql pack bundle --common-caches="$CODEQL_CACHE" --output=misra-cpp-coding-standards.tgz cpp/misra/src
+          codeql pack bundle --common-caches="$CODEQL_CACHE" --output=report-coding-standards.tgz cpp/report/src
 
       - name: Upload qlpack bundles
         uses: actions/upload-artifact@v7
         with:
           name: coding-standards-codeql-packs
-          path: '*-coding-standards.tgz'
+          path: "*-coding-standards.tgz"

.github/workflows/codeql_unit_tests.yml

@@ -17,7 +17,6 @@ on:
       - "rc/**"
 
 jobs:
-
   prepare-unit-test-matrix:
     name: Prepare CodeQL unit test matrix
     runs-on: ubuntu-22.04
@@ -59,7 +58,7 @@ jobs:
 
       - name: Cache CodeQL
         id: cache-codeql
-        uses: actions/cache@v5
+        uses: actions/cache@v6
         with:
           # A list of files, directories, and wildcard patterns to cache and restore
           path: ${{github.workspace}}/codeql_home
@@ -75,16 +74,34 @@ jobs:
           codeql-home: ${{ github.workspace }}/codeql_home
           add-to-path: false
 
+      - name: Cache queries compilation
+        id: cache-queries-compilation
+        uses: actions/cache/restore@v6
+        with:
+          path: ${{ github.workspace }}/codeql_cache
+          key: ${{ runner.os }}-codeql-compilation-${{ matrix.language }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }}-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-codeql-compilation-${{ matrix.language }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }}
+
       - name: Install CodeQL packs
         uses: ./.github/actions/install-codeql-packs
         with:
           cli_path: ${{ github.workspace }}/codeql_home/codeql
+          common_caches: ${{ github.workspace }}/codeql_cache
 
       - name: Pre-Compile Queries
         id: pre-compile-queries
         run: |
-         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 ${{ matrix.language }}
-
+          ${{ github.workspace }}/codeql_home/codeql/codeql query compile \
+            --common-caches=${{ github.workspace }}/codeql_cache \
+            --compilation-cache-size=1024 \
+            --threads 0 \
+            ${{ matrix.language }}
+
+      - name: Save queries compilation cache
+        uses: actions/cache/save@v6
+        with:
+          path: ${{ github.workspace }}/codeql_cache
+          key: ${{ steps.cache-queries-compilation.outputs.cache-primary-key }}
 
       - name: Run test suites
         id: run-test-suites
@@ -135,7 +152,7 @@ jobs:
               os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
               test_report_file = open(test_report_path, 'w')
               files_to_close.append(test_report_file)
-              procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
+              procs.append(subprocess.Popen([codeql_bin, "test", "run", "--common-caches=${{ github.workspace }}/codeql_cache", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
 
             for p in procs:
               _, err = p.communicate()

scripts/install-packs.py

@@ -6,6 +6,7 @@
 parser = argparse.ArgumentParser(description="Install CodeQL library pack dependencies.")
 parser.add_argument('--mode', required=False, choices=['use-lock', 'update', 'verify', 'no-lock'], default="use-lock", help="Installation mode, identical to the `--mode` argument to `codeql pack install`")
 parser.add_argument('--codeql', required=False, default='codeql', help="Path to the `codeql` executable.")
+parser.add_argument('--common-caches', required=False, help="Path to the CodeQL common caches directory.")
 args = parser.parse_args()
 
 # Find the root of the repo
@@ -19,5 +20,7 @@
     pack_path = os.path.join(root, pack)
     # Run `codeql pack install` to install dependencies.
     command = [args.codeql, 'pack', 'install', '--allow-prerelease', '--mode', args.mode, pack_path]
+    if args.common_caches:
+        command.insert(3, f'--common-caches={args.common_caches}')
     print(f'Running `{" ".join(command)}`')
     subprocess.check_call(command)