diff --git a/build.gradle b/build.gradle index 2c32a11c90..4c4d53ab42 100644 --- a/build.gradle +++ b/build.gradle @@ -221,5 +221,8 @@ tasks.register('listRepositories') { doLast { println "Repositories:" project.repositories.each { println "Name: " + it.name + "; url: " + it.url } + // Plugin repositories have no url to report, being wrappers. + println "Buildscript repositories:" + buildscript.repositories.each { println "Name: " + it.name } } } diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index c520450455..64dea564db 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -7,5 +7,15 @@ plugins { } repositories { - mavenCentral() + // Mirrors the root project so that CI resolves through the local Nexus. + val centralRepo = providers.gradleProperty("centralRepo") + if (centralRepo.isPresent) { + maven { + name = "MavenCentral" + url = uri(centralRepo.get()) + isAllowInsecureProtocol = true // Local Nexus in CI uses HTTP + } + } else { + mavenCentral() + } } diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts new file mode 100644 index 0000000000..b656c43e6b --- /dev/null +++ b/buildSrc/settings.gradle.kts @@ -0,0 +1,31 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pluginManagement { + repositories { + // buildSrc resolves its plugins separately from the root project, and + // the Kotlin DSL plugin is only published to the plugin portal. + val pluginRepo = providers.gradleProperty("pluginRepo") + if (pluginRepo.isPresent) { + maven { + name = "GradlePlugins" + url = uri(pluginRepo.get()) + isAllowInsecureProtocol = true // Local Nexus in CI uses HTTP + } + } else { + gradlePluginPortal() + } + + val centralRepo = providers.gradleProperty("centralRepo") + if (centralRepo.isPresent) { + maven { + name = "MavenCentral" + url = uri(centralRepo.get()) + isAllowInsecureProtocol = true // Local Nexus in CI uses HTTP + } + } else { + mavenCentral() + } + } +} diff --git a/settings.gradle b/settings.gradle index 253b7b6dab..9cb3821643 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,7 +1,29 @@ pluginManagement { repositories { - google() - mavenCentral() + // Without these, plugin resolution reaches past the local Nexus in CI to + // the real repositories, and whatever it fetches is missing from the + // android-gradle-dependencies artifact. + def googleRepo = providers.gradleProperty("googleRepo") + if (googleRepo.present) { + maven { + name "Google" + url googleRepo.get() + allowInsecureProtocol true // Local Nexus in CI uses HTTP + } + } else { + google() + } + + def centralRepo = providers.gradleProperty("centralRepo") + if (centralRepo.present) { + maven { + name "MavenCentral" + url centralRepo.get() + allowInsecureProtocol true // Local Nexus in CI uses HTTP + } + } else { + mavenCentral() + } } } diff --git a/taskcluster/docker/base/Dockerfile b/taskcluster/docker/base/Dockerfile index 802cb7e83e..3c83458e8c 100644 --- a/taskcluster/docker/base/Dockerfile +++ b/taskcluster/docker/base/Dockerfile @@ -42,10 +42,11 @@ RUN apt-get update -qq \ locales \ unzip \ mercurial \ + zstd \ && apt-get clean RUN pip3 install --upgrade pip -RUN pip3 install taskcluster +RUN pip3 install taskcluster zstandard RUN locale-gen "$LANG" diff --git a/taskcluster/kinds/toolchain/android.yml b/taskcluster/kinds/toolchain/android.yml index e406a832d4..82eadf945a 100644 --- a/taskcluster/kinds/toolchain/android.yml +++ b/taskcluster/kinds/toolchain/android.yml @@ -38,10 +38,11 @@ linux64-android-gradle-dependencies: resources: - '*.gradle' - 'app/*.gradle' + - 'buildSrc/**' - 'gradle.properties' - 'gradle/**' - taskcluster/scripts/toolchain/android-gradle-dependencies/** - toolchain-artifact: public/build/android-gradle-dependencies.tar.xz + toolchain-artifact: public/build/android-gradle-dependencies.tar.zst toolchain-alias: android-gradle-dependencies treeherder: symbol: TL(gradle-dependencies) diff --git a/taskcluster/rb_taskgraph/job.py b/taskcluster/rb_taskgraph/job.py index 1b29421dc4..a8256dacfe 100644 --- a/taskcluster/rb_taskgraph/job.py +++ b/taskcluster/rb_taskgraph/job.py @@ -95,10 +95,14 @@ def _extract_gradlew_command(run, fetches_dir): maven_dependencies_dir = path.join(fetches_dir, "android-gradle-dependencies") gradle_repos_args = [ - "-P{repo_name}Repo=file://{dir}/{repo_name}".format( - dir=maven_dependencies_dir, repo_name=repo_name + "-P{property_name}=file://{dir}/{repo_name}".format( + dir=maven_dependencies_dir, property_name=property_name, repo_name=repo_name + ) + for property_name, repo_name in ( + ("centralRepo", "central"), + ("googleRepo", "google"), + ("pluginRepo", "gradle-plugins"), ) - for repo_name in ("google", "central") ] gradle_command = ["./gradlew"] + gradle_repos_args + ["listRepositories"] + run.pop("gradlew") post_gradle_commands = run.pop("post-gradlew", []) diff --git a/taskcluster/scripts/toolchain/android-gradle-dependencies.sh b/taskcluster/scripts/toolchain/android-gradle-dependencies.sh index d5f7c26a4b..ef8efcdec4 100755 --- a/taskcluster/scripts/toolchain/android-gradle-dependencies.sh +++ b/taskcluster/scripts/toolchain/android-gradle-dependencies.sh @@ -18,8 +18,25 @@ pushd $PROJECT_DIR . taskcluster/scripts/toolchain/android-gradle-dependencies/before.sh -# We build everything to be sure to fetch all dependencies -./gradlew --no-daemon -Pcoverage -PgoogleRepo='http://localhost:8081/nexus/content/repositories/google/' -PcentralRepo='http://localhost:8081/nexus/content/repositories/central/' assemble assembleAndroidTest bundle test lint ktlint detekt +GRADLE_FLAGS=( + # Overrides the daemon=false the base image puts in GRADLE_OPTS. + --daemon + --no-configuration-cache + -Pcoverage + -PcentralRepo='http://localhost:8081/nexus/content/repositories/central/' + -PgoogleRepo='http://localhost:8081/nexus/content/repositories/google/' + -PpluginRepo='http://localhost:8081/nexus/content/repositories/gradle-plugins/' +) + +# Enumerates and downloads the dependencies of every resolvable configuration +# without running the requested tasks. +./gradlew "${GRADLE_FLAGS[@]}" --write-verification-metadata sha256 --dry-run detekt ktlint lint buildHealth build + +# AGP resolves the aapt2 binary while its tasks run, so the pass above misses it. +./gradlew "${GRADLE_FLAGS[@]}" :app:processDebugResources + +# Don't leave a 4GB heap sitting there while `after.sh` packages everything up. +./gradlew --stop . taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh diff --git a/taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh b/taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh index e2e52b51c9..477df7bc0e 100644 --- a/taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh +++ b/taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh @@ -6,7 +6,7 @@ # This is copy of # https://searchfox.org/mozilla-central/rev/2cd2d511c0d94a34fb7fa3b746f54170ee759e35/taskcluster/scripts/misc/android-gradle-dependencies/after.sh. -# gradle-plugins was removed because it's not used in this project. +# Later changes to that file have been picked up piecemeal. set -x -e @@ -16,13 +16,30 @@ echo "running as" $(id) set -v +# Resolved before the `pushd` below leaves the project directory. +DEPENDENCY_INVENTORY="$PWD/gradle/verification-metadata.dryrun.xml" +VERIFY_DEPENDENCIES="$PWD/taskcluster/scripts/toolchain/android-gradle-dependencies/verify_dependencies.py" + # Package everything up. pushd $WORKSPACE mkdir -p android-gradle-dependencies /builds/worker/artifacts cp -R ${NEXUS_WORK}/storage/central android-gradle-dependencies cp -R ${NEXUS_WORK}/storage/google android-gradle-dependencies - -tar cf - android-gradle-dependencies | xz > /builds/worker/artifacts/android-gradle-dependencies.tar.xz +cp -R ${NEXUS_WORK}/storage/gradle-plugins android-gradle-dependencies || { + echo "FATAL ERROR: no gradle-plugins storage. Did plugin resolution reach the proxy?" + exit 1 +} + +# Bug 1953671: catch intermittently incomplete artifacts here rather than +# downstream. +# The Mozilla repositories in build.gradle are not proxied, so what they serve +# is legitimately absent from the packaged tree. +python3 "$VERIFY_DEPENDENCIES" --inventory "$DEPENDENCY_INVENTORY" \ + --unproxied org/mozilla \ + android-gradle-dependencies/central android-gradle-dependencies/google \ + android-gradle-dependencies/gradle-plugins + +tar cavf /builds/worker/artifacts/android-gradle-dependencies.tar.zst android-gradle-dependencies popd diff --git a/taskcluster/scripts/toolchain/android-gradle-dependencies/verify_dependencies.py b/taskcluster/scripts/toolchain/android-gradle-dependencies/verify_dependencies.py new file mode 100644 index 0000000000..9a70f6de70 --- /dev/null +++ b/taskcluster/scripts/toolchain/android-gradle-dependencies/verify_dependencies.py @@ -0,0 +1,170 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Check the packaged Gradle dependency cache against the inventory that +`--write-verification-metadata` leaves behind, so that an incomplete artifact +fails here rather than in a downstream task (bug 1953671). + +Some of what the inventory lists comes from repositories that aren't proxied +through Nexus and so is expected to be absent from the packaged tree; --unproxied +says which paths those are. Anything else absent, or any component packaged with +one of its files missing, is a real fault. +""" + +import argparse +import hashlib +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +def parse_inventory(path): + """Yield (relative_directory, {filename: sha256}) for each component.""" + root = ET.parse(path).getroot() + for component in root.iter(): + if not component.tag.endswith("component"): + continue + group = component.get("group") + name = component.get("name") + version = component.get("version") + if not (group and name and version): + continue + + artifacts = {} + for artifact in component: + if not artifact.tag.endswith("artifact"): + continue + checksum = next( + ( + child.get("value") + for child in artifact + if child.tag.endswith("sha256") + ), + None, + ) + artifacts[artifact.get("name")] = checksum + + if artifacts: + yield Path(*group.split("."), name, version), artifacts + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as fh: + for block in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def verify(inventory, repositories, unproxied): + problems = [] + unpackaged = [] + components = checked = artifacts = 0 + + for relative_dir, expected in inventory: + components += 1 + + # Downstream tasks are given every one of these as a repository, so an + # artifact only has to be in one of them. + directories = [ + repository / relative_dir + for repository in repositories + if (repository / relative_dir).is_dir() + ] + if not directories: + if str(relative_dir).startswith(unproxied): + unpackaged.append(str(relative_dir)) + else: + problems.append(f"absent: {relative_dir}") + continue + + checked += 1 + checksums = None + for filename, checksum in sorted(expected.items()): + found = next( + (d / filename for d in directories if (d / filename).is_file()), None + ) + if found: + if checksum and sha256(found) != checksum: + problems.append(f"corrupt: {found}") + else: + artifacts += 1 + continue + + # The inventory records the file name from the module metadata, + # which for Kotlin Multiplatform is not the name the repository + # publishes it under. Fall back to matching the contents. + if checksums is None: + checksums = { + sha256(path) + for directory in directories + for path in directory.iterdir() + if path.is_file() + } + if checksum and checksum in checksums: + artifacts += 1 + else: + problems.append(f"missing: {relative_dir}/{filename}") + + return components, checked, artifacts, unpackaged, problems + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--inventory", + required=True, + type=Path, + help="verification-metadata.dryrun.xml written by the enumeration pass", + ) + parser.add_argument( + "--unproxied", + action="append", + default=[], + metavar="PREFIX", + help="path prefix served by a repository that isn't proxied through " + "Nexus, so is expected to be absent from the packaged tree; repeatable", + ) + parser.add_argument( + "repository", + nargs="+", + type=Path, + help="packaged repository directories to check, e.g. central google", + ) + args = parser.parse_args() + + if not args.inventory.is_file(): + print( + f"FATAL ERROR: no inventory at {args.inventory}. Did the " + "enumeration pass run?" + ) + return 1 + + components, checked, artifacts, unpackaged, problems = verify( + parse_inventory(args.inventory), args.repository, tuple(args.unproxied) + ) + print( + f"verified {artifacts} artifacts across {checked} of {components} " + f"components; {len(unpackaged)} are not packaged, coming from a " + "repository we don't proxy" + ) + for path in unpackaged: + print(f" not packaged: {path}") + + if not components: + print(f"FATAL ERROR: {args.inventory} lists no components at all.") + return 1 + + if problems: + print(f"FATAL ERROR: {len(problems)} absent, missing or corrupt:") + for problem in problems: + print(f" {problem}") + print("The dependency cache is incomplete. Try re-running this task.") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main())