diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index d3c48b6e259..76b2f03e7f3 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -49,75 +49,57 @@ # a handler classes MUST be added to this list to be active APPLICATION_PACKAGE_DATAFILE_HANDLERS = [ about.AboutFileHandler, - alpine.AlpineApkArchiveHandler, alpine.AlpineApkbuildHandler, - bower.BowerJsonHandler, - build_gradle.BuildGradleHandler, - build.AutotoolsConfigureHandler, build.BazelBuildHandler, + build.BazelModuleHandler, + build.BazelModuleLockHandler, build.BuckMetadataBzlHandler, build.BuckPackageHandler, - cargo.CargoLockHandler, cargo.CargoTomlHandler, - chef.ChefMetadataJsonHandler, chef.ChefMetadataRbHandler, - cocoapods.PodspecHandler, cocoapods.PodspecJsonHandler, cocoapods.PodfileLockHandler, cocoapods.PodfileHandler, - conda.CondaMetaJsonHandler, conda.CondaMetaYamlHandler, conda.CondaYamlHandler, - conan.ConanFileHandler, conan.ConanDataHandler, - cran.CranDescriptionFileHandler, - debian_copyright.DebianCopyrightFileInPackageHandler, debian_copyright.StandaloneDebianCopyrightFileHandler, debian.DebianDscFileHandler, - debian.DebianControlFileInExtractedDebHandler, debian.DebianControlFileInSourceHandler, - debian.DebianDebPackageHandler, debian.DebianMd5sumFilelistInPackageHandler, - debian.DebianSourcePackageMetadataTarballHandler, debian.DebianSourcePackageTarballHandler, - distro.EtcOsReleaseHandler, - freebsd.CompactManifestHandler, - godeps.GodepsHandler, golang.GoModHandler, golang.GoSumHandler, - haxe.HaxelibJsonHandler, - maven.MavenPomXmlHandler, maven.MavenPomPropertiesHandler, maven.JavaJarManifestHandler, maven.JavaOSGiManifestHandler, - misc.AndroidAppArchiveHandler, misc.AndroidLibraryHandler, misc.AppleDmgHandler, - misc.Axis2MarArchiveHandler , - misc.Axis2MarModuleXmlHandler , + misc.Axis2MarArchiveHandler, + misc.Axis2MarModuleXmlHandler, misc.CabArchiveHandler, misc.ChromeExtensionHandler, - misc.CpanDistIniHandler , + misc.CpanDistIniHandler, misc.CpanMakefilePlHandler, misc.CpanManifestHandler, misc.CpanMetaJsonHandler, @@ -126,19 +108,14 @@ misc.IosAppIpaHandler, misc.IsoImageHandler, misc.IvyXmlHandler, - - misc.JavaEarAppXmlHandler , - misc.JavaEarHandler , - + misc.JavaEarAppXmlHandler, + misc.JavaEarHandler, # is this redundant with Jar manifest? misc.JavaJarHandler, - misc.JavaWarHandler, misc.JavaWarWebXmlHandler, - - misc.JBossSarHandler , - misc.JBossServiceXmlHandler , - + misc.JBossSarHandler, + misc.JBossServiceXmlHandler, misc.MeteorPackageHandler, misc.MozillaExtensionHandler, misc.NsisInstallerHandler, @@ -152,19 +129,14 @@ npm.PnpmShrinkwrapYamlHandler, npm.PnpmLockYamlHandler, npm.PnpmWorkspaceYamlHandler, - nuget.NugetNupkgHandler, nuget.NugetNuspecHandler, nuget.NugetPackagesLockHandler, - opam.OpamFileHandler, - phpcomposer.PhpComposerJsonHandler, phpcomposer.PhpComposerLockHandler, - pubspec.DartPubspecYamlHandler, pubspec.DartPubspecLockHandler, - pypi.PipfileHandler, pypi.PipfileLockHandler, pypi.PipRequirementsFileHandler, @@ -180,36 +152,26 @@ pypi.PythonSdistPkgInfoFile, pypi.PythonSetupPyHandler, pypi.SetupCfgHandler, - readme.ReadmeHandler, - rpm.RpmArchiveHandler, rpm.RpmSpecfileHandler, - rubygems.GemMetadataArchiveExtractedHandler, rubygems.GemArchiveHandler, - # the order of these handlers matter rubygems.GemfileInExtractedGemHandler, rubygems.GemfileHandler, - # the order of these handlers matter rubygems.GemfileLockInExtractedGemHandler, rubygems.GemfileLockHandler, - # the order of these handlers matter rubygems.GemspecInInstalledVendorBundleSpecificationsHandler, rubygems.GemspecInExtractedGemHandler, rubygems.GemspecHandler, - swift.SwiftManifestJsonHandler, swift.SwiftPackageResolvedHandler, swift.SwiftShowDependenciesDepLockHandler, - windows.MicrosoftUpdateManifestHandler, - win_pe.WindowsExecutableHandler, - # These are handlers for deplock generated files pypi.PipInspectDeplockHandler, ] @@ -221,18 +183,14 @@ SYSTEM_PACKAGE_DATAFILE_HANDLERS = [ alpine.AlpineInstalledDatabaseHandler, - debian_copyright.DebianCopyrightFileInPackageHandler, debian_copyright.DebianCopyrightFileInSourceHandler, - debian.DebianDistrolessInstalledDatabaseHandler, - debian.DebianInstalledFilelistHandler, debian.DebianInstalledMd5sumFilelistHandler, debian.DebianInstalledStatusDatabaseHandler, - rpm.RpmLicenseFilesHandler, - rpm.RpmMarinerContainerManifestHandler + rpm.RpmMarinerContainerManifestHandler, ] if on_linux: @@ -240,7 +198,6 @@ rpm.RpmInstalledBdbDatabaseHandler, rpm.RpmInstalledSqliteDatabaseHandler, rpm.RpmInstalledNdbDatabaseHandler, - win_reg.InstalledProgramFromDockerSoftwareDeltaHandler, win_reg.InstalledProgramFromDockerFilesSoftwareHandler, win_reg.InstalledProgramFromDockerUtilityvmSoftwareHandler, @@ -255,6 +212,7 @@ try: from go_inspector.binary import get_go_binary_handler + handler = get_go_binary_handler() PACKAGE_IN_COMPILED_DATAFILE_HANDLERS.append(handler) except ImportError: @@ -262,16 +220,20 @@ try: from rust_inspector.packages import get_rust_binary_handler + handler = get_rust_binary_handler() PACKAGE_IN_COMPILED_DATAFILE_HANDLERS.append(handler) except ImportError: pass ALL_DATAFILE_HANDLERS = ( - APPLICATION_PACKAGE_DATAFILE_HANDLERS + [ - p for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS + APPLICATION_PACKAGE_DATAFILE_HANDLERS + + [ + p + for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS if p not in APPLICATION_PACKAGE_DATAFILE_HANDLERS - ] + PACKAGE_IN_COMPILED_DATAFILE_HANDLERS + ] + + PACKAGE_IN_COMPILED_DATAFILE_HANDLERS ) # registry of all handler classes keyed by datasource_id @@ -294,6 +256,4 @@ def get_package_handler(package_data): return ppc -PACKAGE_DATA_CLASS_BY_DATASOURCE_ID = { - maven.MavenPackageData.datasource_id: maven.MavenPackageData -} +PACKAGE_DATA_CLASS_BY_DATASOURCE_ID = {maven.MavenPackageData.datasource_id: maven.MavenPackageData} diff --git a/src/packagedcode/build.py b/src/packagedcode/build.py index 8044f8c2459..74add0ea344 100644 --- a/src/packagedcode/build.py +++ b/src/packagedcode/build.py @@ -7,10 +7,25 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import os -import logging import ast +import json +import logging +import os from collections import defaultdict +import re + + +def _extract_starlark_kwarg(block, key): + """ + Extract a string value for a named keyword argument from a + Starlark (Bazel) function call body. + + Example block: 'name = "foo", version = "1.0"' + _extract_starlark_kwarg(block, 'name') -> 'foo' + """ + match = re.search(r"\b" + re.escape(key) + r'\s*=\s*["\']([^"\']+)["\']', block) + return match.group(1) if match else None + from commoncode import fileutils from packageurl import PackageURL @@ -28,7 +43,7 @@ Buck, Bazel, Pants, etc. """ -TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False) +TRACE = os.environ.get("SCANCODE_DEBUG_PACKAGE", False) def logger_debug(*args): @@ -39,21 +54,23 @@ def logger_debug(*args): if TRACE: import sys + logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug( - ' '.join(isinstance(a, str) and a or repr(a) for a in args) - ) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) class AutotoolsConfigureHandler(models.NonAssemblableDatafileHandler): - datasource_id = 'autotools_configure' - path_patterns = ('*/configure', '*/configure.ac',) - default_package_type = 'autotools' - description = 'Autotools configure script' - documentation_url = 'https://www.gnu.org/software/automake/' + datasource_id = "autotools_configure" + path_patterns = ( + "*/configure", + "*/configure.ac", + ) + default_package_type = "autotools" + description = "Autotools configure script" + documentation_url = "https://www.gnu.org/software/automake/" @classmethod def parse(cls, location, package_only=False): @@ -77,8 +94,7 @@ def parse(cls, location, package_only=False): yield models.PackageData.from_data(package_data, package_only) - -def check_rule_name_ending(rule_name, starlark_rule_types=('binary', 'library')): +def check_rule_name_ending(rule_name, starlark_rule_types=("binary", "library")): """ Return True if `rule_name` ends with a rule type from `starlark_rule_types` Return False otherwise @@ -110,37 +126,38 @@ def assemble(cls, package_data, resource, codebase, package_adder): ) if TRACE: - logger_debug(f"BaseStarlarkManifestHandler.assemble: package_data: {package_data.to_dict()}") + logger_debug( + f"BaseStarlarkManifestHandler.assemble: package_data: {package_data.to_dict()}" + ) - package.license_detections, package.declared_license_expression = \ + package.license_detections, package.declared_license_expression = ( get_license_detections_and_expression( package=package_data, resource=resource, codebase=codebase, ) + ) if package.declared_license_expression: - package.declared_license_expression_spdx = str(build_spdx_license_expression( - license_expression=package.declared_license_expression, - licensing=get_cache().licensing, - )) + package.declared_license_expression_spdx = str( + build_spdx_license_expression( + license_expression=package.declared_license_expression, + licensing=get_cache().licensing, + ) + ) cls.assign_package_to_resources( - package=package, - resource=resource, - codebase=codebase, - package_adder=package_adder + package=package, resource=resource, codebase=codebase, package_adder=package_adder ) yield package - # we yield this as we do not want this further processed yield resource @classmethod def parse(cls, location, package_only=False): # Thanks to Starlark being a Python dialect, we can use `ast` to parse it - with open(location, 'rb') as f: + with open(location, "rb") as f: tree = ast.parse(f.read()) build_rules = defaultdict(list) @@ -170,8 +187,7 @@ def parse(cls, location, package_only=False): # We collect the elements of a list if the element is # not a function call args[arg_name] = [ - elt.value for elt in kw.value.elts - if not isinstance(elt, ast.Call) + elt.value for elt in kw.value.elts if not isinstance(elt, ast.Call) ] if args: build_rules[rule_name].append(args) @@ -179,13 +195,13 @@ def parse(cls, location, package_only=False): if build_rules: for rule_name, rule_instances_args in build_rules.items(): for args in rule_instances_args: - name = args.get('name') + name = args.get("name") # FIXME: we could still return partial package data if not name: continue - license_files = args.get('licenses') + license_files = args.get("licenses") if TRACE: logger_debug(f"build: parse: license_files: {license_files}") @@ -209,12 +225,14 @@ def parse(cls, location, package_only=False): package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, - name=fileutils.file_name(fileutils.parent_directory(location)) + name=fileutils.file_name(fileutils.parent_directory(location)), ) yield models.PackageData.from_data(package_data, package_only) @classmethod - def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name=None): + def assign_package_to_resources( + cls, package, resource, codebase, package_adder, skip_name=None + ): package_uid = package.package_uid if not package_uid: return @@ -258,8 +276,7 @@ def get_license_detections_and_expression(package, resource, codebase): if TRACE: logger_debug( - f"build: get_license_detections_and_expression:" - f"declared_licenses: {declared_licenses}" + f"build: get_license_detections_and_expression:declared_licenses: {declared_licenses}" ) logger_debug( f"build: get_license_detections_and_expression:" @@ -273,31 +290,28 @@ def get_license_detections_and_expression(package, resource, codebase): detections = detect_licenses(location=child.location) if TRACE: logger_debug( - f"build: get_license_detections_and_expression:" - f"detections: {detections}" + f"build: get_license_detections_and_expression:detections: {detections}" ) if not detections: - license_detections.append( - get_unknown_license_detection(declared_licenses) - ) + license_detections.append(get_unknown_license_detection(declared_licenses)) else: license_detections.extend(detections) - return get_mapping_and_expression_from_detections( - license_detections=license_detections - ) + return get_mapping_and_expression_from_detections(license_detections=license_detections) class BazelBuildHandler(BaseStarlarkManifestHandler): - datasource_id = 'bazel_build' - path_patterns = ('*/BUILD',) - default_package_type = 'bazel' - description = 'Bazel BUILD' - documentation_url = 'https://bazel.build/' + datasource_id = "bazel_build" + path_patterns = ("*/BUILD",) + default_package_type = "bazel" + description = "Bazel BUILD" + documentation_url = "https://bazel.build/" @classmethod - def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name='BUILD'): + def assign_package_to_resources( + cls, package, resource, codebase, package_adder, skip_name="BUILD" + ): return super().assign_package_to_resources( package=package, resource=resource, @@ -308,14 +322,16 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder, class BuckPackageHandler(BaseStarlarkManifestHandler): - datasource_id = 'buck_file' - path_patterns = ('*/BUCK',) - default_package_type = 'buck' - description = 'Buck file' - documentation_url = 'https://buck.build/' + datasource_id = "buck_file" + path_patterns = ("*/BUCK",) + default_package_type = "buck" + description = "Buck file" + documentation_url = "https://buck.build/" @classmethod - def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name='BUCK'): + def assign_package_to_resources( + cls, package, resource, codebase, package_adder, skip_name="BUCK" + ): return super().assign_package_to_resources( package=package, resource=resource, @@ -326,26 +342,26 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder, class BuckMetadataBzlHandler(BaseStarlarkManifestHandler): - datasource_id = 'buck_metadata' - path_patterns = ('*/METADATA.bzl',) - default_package_type = 'buck' - description = 'Buck metadata file' - documentation_url = 'https://buck.build/' + datasource_id = "buck_metadata" + path_patterns = ("*/METADATA.bzl",) + default_package_type = "buck" + description = "Buck metadata file" + documentation_url = "https://buck.build/" @classmethod def parse(cls, location, package_only=True): - with open(location, 'rb') as f: + with open(location, "rb") as f: tree = ast.parse(f.read()) metadata_fields = {} for statement in tree.body: - if not (hasattr(statement, 'targets') and isinstance(statement, ast.Assign)): + if not (hasattr(statement, "targets") and isinstance(statement, ast.Assign)): continue # We are looking for a dictionary assigned to the variable `METADATA` for target in statement.targets: - if not (target.id == 'METADATA' and isinstance(statement.value, ast.Dict)): + if not (target.id == "METADATA" and isinstance(statement.value, ast.Dict)): continue # Once we find the dictionary assignment, get and store its contents statement_keys = statement.value.keys @@ -365,61 +381,59 @@ def parse(cls, location, package_only=True): metadata_fields[key_name] = value parties = [] - maintainers = metadata_fields.get('maintainers', []) or [] + maintainers = metadata_fields.get("maintainers", []) or [] for maintainer in maintainers: parties.append( models.Party( type=models.party_org, name=maintainer, - role='maintainer', + role="maintainer", ) ) # TODO: Create function that determines package type from download URL, # then create a package of that package type from the metadata info - - if 'upstream_type' in metadata_fields: - package_type = metadata_fields['upstream_type'] - elif 'package_type' in metadata_fields: - package_type = metadata_fields['package_type'] + + if "upstream_type" in metadata_fields: + package_type = metadata_fields["upstream_type"] + elif "package_type" in metadata_fields: + package_type = metadata_fields["package_type"] else: package_type = cls.default_package_type - if 'licenses' in metadata_fields: - extracted_license_statement = metadata_fields['licenses'] + if "licenses" in metadata_fields: + extracted_license_statement = metadata_fields["licenses"] else: - extracted_license_statement = metadata_fields.get('license_expression') + extracted_license_statement = metadata_fields.get("license_expression") - if 'upstream_address' in metadata_fields: - homepage_url = metadata_fields['upstream_address'] + if "upstream_address" in metadata_fields: + homepage_url = metadata_fields["upstream_address"] else: - homepage_url = metadata_fields.get('homepage_url') - + homepage_url = metadata_fields.get("homepage_url") extra_data = {} - if 'vcs_commit_hash' in metadata_fields: - extra_data['vcs_commit_hash'] = metadata_fields['vcs_commit_hash'] - if 'upstream_hash' in metadata_fields: - extra_data['upstream_hash'] = metadata_fields['upstream_hash'] + if "vcs_commit_hash" in metadata_fields: + extra_data["vcs_commit_hash"] = metadata_fields["vcs_commit_hash"] + if "upstream_hash" in metadata_fields: + extra_data["upstream_hash"] = metadata_fields["upstream_hash"] package_data = dict( datasource_id=cls.datasource_id, type=package_type, - name=metadata_fields.get('name'), - version=metadata_fields.get('version'), + name=metadata_fields.get("name"), + version=metadata_fields.get("version"), extracted_license_statement=extracted_license_statement, parties=parties, homepage_url=homepage_url, - download_url=metadata_fields.get('download_url'), - vcs_url=metadata_fields.get('vcs_url'), - sha1=metadata_fields.get('download_archive_sha1'), - extra_data=extra_data + download_url=metadata_fields.get("download_url"), + vcs_url=metadata_fields.get("vcs_url"), + sha1=metadata_fields.get("download_archive_sha1"), + extra_data=extra_data, ) - if 'package_url' in metadata_fields: - package_data.update(PackageURL.from_string(metadata_fields['package_url']).to_dict()) - - yield models.PackageData.from_data(package_data, package_only=True) + if "package_url" in metadata_fields: + package_data.update(PackageURL.from_string(metadata_fields["package_url"]).to_dict()) + yield models.PackageData.from_data(package_data, package_only=True) @classmethod def assign_package_to_resources(cls, package, resource, codebase, package_adder): @@ -429,3 +443,161 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder) codebase=codebase, package_adder=package_adder, ) + + +class BaseBazelModuleHandler(models.DatafileHandler): + @classmethod + def assemble(cls, package_data, resource, codebase, package_adder): + if resource.has_parent(): + directory = resource.parent(codebase) + else: + directory = resource + + if not directory: + yield from super().assemble( + package_data=package_data, + resource=resource, + codebase=codebase, + package_adder=package_adder, + ) + return + + if not codebase.has_single_resource: + siblings = list(directory.children(codebase)) + else: + siblings = [directory] + + pkgdata_resources = [] + for datafile_name in ("MODULE.bazel", "MODULE.bazel.lock"): + for sibling in siblings: + if sibling.name != datafile_name: + continue + + for sibling_package_data in sibling.package_data: + pkgdata_resources.append( + (models.PackageData.from_dict(sibling_package_data), sibling) + ) + + if pkgdata_resources: + yield from cls.assemble_from_many( + pkgdata_resources=pkgdata_resources, + codebase=codebase, + package_adder=package_adder, + ignore_name_check=True, + parent_resource=directory, + ) + return + + yield from super().assemble( + package_data=package_data, + resource=resource, + codebase=codebase, + package_adder=package_adder, + ) + + +class BazelModuleHandler(BaseBazelModuleHandler): + """ + Handle Bazel MODULE.bazel module manifest files used by Bzlmod. + See: https://bazel.build/external/module + """ + + datasource_id = "bazel_module" + path_patterns = ("*/MODULE.bazel",) + default_package_type = "bazel" + description = "Bazel MODULE.bazel module manifest (Bzlmod)" + documentation_url = "https://bazel.build/external/module" + + @classmethod + def parse(cls, location, package_only=False): + with open(location, encoding="utf-8", errors="replace") as f: + content = f.read() + + # --- Extract module() declaration --- + name = None + version = None + module_match = re.search( + r"\bmodule\s*\(([^)]+)\)", + content, + re.DOTALL, + ) + if module_match: + block = module_match.group(1) + name = _extract_starlark_kwarg(block, "name") + version = _extract_starlark_kwarg(block, "version") + + # Root Bazel modules can omit module(), so fall back to the enclosing + # directory name to keep the manifest assemblable. + if not name: + name = fileutils.file_name(fileutils.parent_directory(location)) + + # --- Extract bazel_dep() declarations --- + dependencies = [] + for dep_match in re.finditer( + r"\bbazel_dep\s*\(([^)]+)\)", + content, + re.DOTALL, + ): + block = dep_match.group(1) + dep_name = _extract_starlark_kwarg(block, "name") + dep_version = _extract_starlark_kwarg(block, "version") + is_dev = bool(re.search(r"\bdev_dependency\s*=\s*True\b", block)) + + if not dep_name: + continue + + purl = f"pkg:bazel/{dep_name}" + if dep_version: + purl = f"{purl}@{dep_version}" + + dependencies.append( + models.DependentPackage( + purl=purl, + extracted_requirement=dep_version, + scope="dev" if is_dev else "dependencies", + is_runtime=not is_dev, + is_optional=is_dev, + ) + ) + + yield models.PackageData( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + name=name, + version=version, + dependencies=dependencies, + ) + + +class BazelModuleLockHandler(BaseBazelModuleHandler): + """ + Handle Bazel MODULE.bazel.lock files and merge their metadata with a + sibling MODULE.bazel manifest during assembly. + """ + + datasource_id = "bazel_module_lock" + path_patterns = ("*/MODULE.bazel.lock",) + default_package_type = "bazel" + is_lockfile = True + description = "Bazel MODULE.bazel lockfile" + documentation_url = "https://bazel.build/external/lockfile" + + @classmethod + def parse(cls, location, package_only=False): + with open(location, encoding="utf-8", errors="replace") as f: + parsed = json.load(f) + + registry_file_hashes = parsed.get("registryFileHashes") or {} + module_extensions = parsed.get("moduleExtensions") or {} + + yield models.PackageData( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + extra_data={ + "bazel_lockfile_version": parsed.get("lockFileVersion"), + "bazel_registry_file_hashes_count": len(registry_file_hashes), + "bazel_module_extensions": sorted(module_extensions), + "bazel_selected_yanked_versions": parsed.get("selectedYankedVersions") + or {}, + }, + ) diff --git a/tests/packagedcode/data/bazel_module/MODULE.bazel b/tests/packagedcode/data/bazel_module/MODULE.bazel new file mode 100644 index 00000000000..378b60b7fa4 --- /dev/null +++ b/tests/packagedcode/data/bazel_module/MODULE.bazel @@ -0,0 +1,11 @@ +# Sample Bazel module manifest +module( + name = "my_sample_project", + version = "0.5.0", + compatibility_level = 1, +) + +bazel_dep(name = "rules_python", version = "0.24.0") +bazel_dep(name = "rules_go", version = "0.41.0") +bazel_dep(name = "googletest", version = "1.14.0", dev_dependency = True) +bazel_dep(name = "abseil-cpp", version = "20230802.0", repo_name = "com_google_absl") diff --git a/tests/packagedcode/data/bazel_module/MODULE_no_version.bazel b/tests/packagedcode/data/bazel_module/MODULE_no_version.bazel new file mode 100644 index 00000000000..327ef2258e2 --- /dev/null +++ b/tests/packagedcode/data/bazel_module/MODULE_no_version.bazel @@ -0,0 +1,5 @@ +module( + name = "minimal_module", +) + +bazel_dep(name = "rules_python", version = "0.24.0") diff --git a/tests/packagedcode/data/build/bazel/allocation-instrumenter-expected.json b/tests/packagedcode/data/build/bazel/allocation-instrumenter-expected.json new file mode 100644 index 00000000000..7d3bb8b3872 --- /dev/null +++ b/tests/packagedcode/data/build/bazel/allocation-instrumenter-expected.json @@ -0,0 +1,409 @@ +{ + "packages": [ + { + "type": "bazel", + "namespace": null, + "name": "builddefs_bzl", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "package_uid": "pkg:bazel/builddefs_bzl?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "allocation-instrumenter/BUILD" + ], + "datasource_ids": [ + "bazel_build" + ], + "purl": "pkg:bazel/builddefs_bzl" + }, + { + "type": "bazel", + "namespace": null, + "name": "allocation-instrumenter", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "bazel_lockfile_version": 18, + "bazel_registry_file_hashes_count": 7, + "bazel_module_extensions": [ + "@@rules_python+//python/private/pypi:pip.bzl%pip_internal" + ], + "bazel_selected_yanked_versions": {} + }, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "package_uid": "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "allocation-instrumenter/MODULE.bazel", + "allocation-instrumenter/MODULE.bazel.lock" + ], + "datasource_ids": [ + "bazel_module", + "bazel_module_lock" + ], + "purl": "pkg:bazel/allocation-instrumenter" + } + ], + "dependencies": [ + { + "purl": "pkg:bazel/bazel_skylib@1.8.1", + "extracted_requirement": "1.8.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:bazel/bazel_skylib@1.8.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "allocation-instrumenter/MODULE.bazel", + "datasource_id": "bazel_module" + }, + { + "purl": "pkg:bazel/rules_java@8.15.2", + "extracted_requirement": "8.15.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:bazel/rules_java@8.15.2?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "allocation-instrumenter/MODULE.bazel", + "datasource_id": "bazel_module" + }, + { + "purl": "pkg:bazel/rules_shell@0.6.1", + "extracted_requirement": "0.6.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:bazel/rules_shell@0.6.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "allocation-instrumenter/MODULE.bazel", + "datasource_id": "bazel_module" + }, + { + "purl": "pkg:bazel/google_bazel_common", + "extracted_requirement": null, + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:bazel/google_bazel_common?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "allocation-instrumenter/MODULE.bazel", + "datasource_id": "bazel_module" + } + ], + "files": [ + { + "path": "allocation-instrumenter", + "type": "directory", + "package_data": [], + "for_packages": [ + "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "allocation-instrumenter/BUILD", + "type": "file", + "package_data": [ + { + "type": "bazel", + "namespace": null, + "name": "builddefs_bzl", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "bazel_build", + "purl": "pkg:bazel/builddefs_bzl" + } + ], + "for_packages": [ + "pkg:bazel/builddefs_bzl?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "allocation-instrumenter/MODULE.bazel", + "type": "file", + "package_data": [ + { + "type": "bazel", + "namespace": null, + "name": "allocation-instrumenter", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:bazel/bazel_skylib@1.8.1", + "extracted_requirement": "1.8.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:bazel/rules_java@8.15.2", + "extracted_requirement": "8.15.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:bazel/rules_shell@0.6.1", + "extracted_requirement": "0.6.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:bazel/google_bazel_common", + "extracted_requirement": null, + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "bazel_module", + "purl": "pkg:bazel/allocation-instrumenter" + } + ], + "for_packages": [ + "pkg:bazel/builddefs_bzl?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "allocation-instrumenter/MODULE.bazel.lock", + "type": "file", + "package_data": [ + { + "type": "bazel", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "bazel_lockfile_version": 18, + "bazel_registry_file_hashes_count": 7, + "bazel_module_extensions": [ + "@@rules_python+//python/private/pypi:pip.bzl%pip_internal" + ], + "bazel_selected_yanked_versions": {} + }, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "bazel_module_lock", + "purl": null + } + ], + "for_packages": [ + "pkg:bazel/builddefs_bzl?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "allocation-instrumenter/builddefs.bzl", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:bazel/builddefs_bzl?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:bazel/allocation-instrumenter?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/packagedcode/data/build/bazel/allocation-instrumenter.ABOUT b/tests/packagedcode/data/build/bazel/allocation-instrumenter.ABOUT new file mode 100644 index 00000000000..9a375498b93 --- /dev/null +++ b/tests/packagedcode/data/build/bazel/allocation-instrumenter.ABOUT @@ -0,0 +1,8 @@ +about_resource: allocation-instrumenter/ +name: allocation-instrumenter +version: 59d6bc4f8753e7cf76204be043075b5aa32487c8 +download_url: https://github.com/google/allocation-instrumenter/tree/59d6bc4f8753e7cf76204be043075b5aa32487c8 +vcs_repository: https://github.com/google/allocation-instrumenter.git +license: apache-2.0 +description: Trimmed BUILD, MODULE.bazel, and MODULE.bazel.lock files used to test Bazel module assembly with sibling BUILD manifests. +notes: MODULE.bazel.lock is a minimized subset of the upstream lockfile that preserves the fields needed for ScanCode tests. diff --git a/tests/packagedcode/data/build/bazel/allocation-instrumenter/BUILD b/tests/packagedcode/data/build/bazel/allocation-instrumenter/BUILD new file mode 100644 index 00000000000..8fc719f689b --- /dev/null +++ b/tests/packagedcode/data/build/bazel/allocation-instrumenter/BUILD @@ -0,0 +1,9 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +licenses(["notice"]) + +bzl_library( + name = "builddefs_bzl", + srcs = ["builddefs.bzl"], + deps = ["@rules_java//java:core_rules"], +) diff --git a/tests/packagedcode/data/build/bazel/allocation-instrumenter/MODULE.bazel b/tests/packagedcode/data/build/bazel/allocation-instrumenter/MODULE.bazel new file mode 100644 index 00000000000..1df918e33d8 --- /dev/null +++ b/tests/packagedcode/data/build/bazel/allocation-instrumenter/MODULE.bazel @@ -0,0 +1,10 @@ +bazel_dep(name = "bazel_skylib", version = "1.8.1") +bazel_dep(name = "rules_java", version = "8.15.2") +bazel_dep(name = "rules_shell", version = "0.6.1") +bazel_dep(name = "google_bazel_common") + +git_override( + module_name = "google_bazel_common", + commit = "8c50e017875b3fa93729cb75d68c345c4ef8b5d1", + remote = "https://github.com/google/bazel-common", +) diff --git a/tests/packagedcode/data/build/bazel/allocation-instrumenter/MODULE.bazel.lock b/tests/packagedcode/data/build/bazel/allocation-instrumenter/MODULE.bazel.lock new file mode 100644 index 00000000000..da8279e0895 --- /dev/null +++ b/tests/packagedcode/data/build/bazel/allocation-instrumenter/MODULE.bazel.lock @@ -0,0 +1,30 @@ +{ + "lockFileVersion": 18, + "registryFileHashes": { + "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/source.json": "7ebaefba0b03efe59cac88ed5bbc67bcf59a3eff33af937345ede2a38b2d368a", + "https://bcr.bazel.build/modules/rules_java/8.15.2/MODULE.bazel": "5cc6698c822b2f9ef90ca5558599851bed8c3b13f1f8eb140d9bfec638d2acb4", + "https://bcr.bazel.build/modules/rules_java/8.15.2/source.json": "352984cbe6d32fac3bf76449e581ed5bcd54a2da2137fca1559aaf04756b7bfa", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c" + }, + "selectedYankedVersions": {}, + "moduleExtensions": { + "@@rules_python+//python/private/pypi:pip.bzl%pip_internal": { + "general": { + "bzlTransitiveDigest": "Yq7o46BUXUk7CJz/OHGoo/nvjKqnnS+Elj2k8iCP8sk=", + "usagesDigest": "OLoIStnzNObNalKEMRq99FqenhPGLFZ5utVLV4sz7OI=", + "recordedFileInputs": { + "@@rules_python+//tools/publish/requirements_darwin.txt": "2994136eab7e57b083c3de76faf46f70fad130bc8e7360a7fed2b288b69e79dc" + }, + "recordedDirentsInputs": {}, + "envVariables": { + "RULES_PYTHON_REPO_DEBUG": null + }, + "generatedRepoSpecs": {}, + "recordedRepoMappingEntries": [] + } + } + } +} diff --git a/tests/packagedcode/data/build/bazel/allocation-instrumenter/builddefs.bzl b/tests/packagedcode/data/build/bazel/allocation-instrumenter/builddefs.bzl new file mode 100644 index 00000000000..6c2044adfbb --- /dev/null +++ b/tests/packagedcode/data/build/bazel/allocation-instrumenter/builddefs.bzl @@ -0,0 +1,2 @@ +def java_agent_binary(**kwargs): + pass diff --git a/tests/packagedcode/data/plugin/plugins_list_linux.txt b/tests/packagedcode/data/plugin/plugins_list_linux.txt index eb4763d6c7e..d848b091d0f 100755 --- a/tests/packagedcode/data/plugin/plugins_list_linux.txt +++ b/tests/packagedcode/data/plugin/plugins_list_linux.txt @@ -69,6 +69,20 @@ Package type: bazel description: Bazel BUILD path_patterns: '*/BUILD' -------------------------------------------- +Package type: bazel + datasource_id: bazel_module + documentation URL: https://bazel.build/external/module + primary language: None + description: Bazel MODULE.bazel module manifest (Bzlmod) + path_patterns: '*/MODULE.bazel' +-------------------------------------------- +Package type: bazel + datasource_id: bazel_module_lock + documentation URL: https://bazel.build/external/lockfile + primary language: None + description: Bazel MODULE.bazel lockfile + path_patterns: '*/MODULE.bazel.lock' +-------------------------------------------- Package type: bower datasource_id: bower_json documentation URL: https://bower.io diff --git a/tests/packagedcode/test_build.py b/tests/packagedcode/test_build.py index 11b2c8cb5ff..ede09588695 100644 --- a/tests/packagedcode/test_build.py +++ b/tests/packagedcode/test_build.py @@ -20,52 +20,63 @@ from scancode_config import REGEN_TEST_FIXTURES - class TestBuild(PackageTester): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data/build') + test_data_dir = os.path.join(os.path.dirname(__file__), "data/build") def test_end2end_scan_can_detect_bazel(self): - test_file = self.get_test_loc('bazel/end2end') - expected_file = self.get_test_loc('bazel/end2end-expected.json') - result_file = self.get_temp_file('results.json') - run_scan_click(['--package', test_file, '--json-pp', result_file]) + test_file = self.get_test_loc("bazel/end2end") + expected_file = self.get_test_loc("bazel/end2end-expected.json") + result_file = self.get_temp_file("results.json") + run_scan_click(["--package", test_file, "--json-pp", result_file]) check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES) + def test_end2end_scan_can_detect_bazel_build_and_module_together(self): + test_file = self.get_test_loc("bazel/allocation-instrumenter") + expected_file = self.get_test_loc("bazel/allocation-instrumenter-expected.json") + result_file = self.get_temp_file("results.json") + run_scan_click(["--package", test_file, "--json-pp", result_file]) + check_json_scan( + expected_file, + result_file, + remove_uuid=True, + regen=REGEN_TEST_FIXTURES, + ) + def test_end2end_scan_can_detect_buck(self): - test_file = self.get_test_loc('buck/end2end') - expected_file = self.get_test_loc('buck/end2end-expected.json') - result_file = self.get_temp_file('results.json') - run_scan_click(['--package', test_file, '--json-pp', result_file]) + test_file = self.get_test_loc("buck/end2end") + expected_file = self.get_test_loc("buck/end2end-expected.json") + result_file = self.get_temp_file("results.json") + run_scan_click(["--package", test_file, "--json-pp", result_file]) check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES) def test_BazelPackage_parse(self): - test_file = self.get_test_loc('bazel/parse/BUILD') + test_file = self.get_test_loc("bazel/parse/BUILD") result_packages = build.BazelBuildHandler.parse(test_file) expected_packages = [ models.PackageData( - name='hello-greet', + name="hello-greet", type=build.BazelBuildHandler.default_package_type, datasource_id=build.BazelBuildHandler.datasource_id, ), models.PackageData( - name='hello-world', + name="hello-world", type=build.BazelBuildHandler.default_package_type, datasource_id=build.BazelBuildHandler.datasource_id, - ) + ), ] compare_package_results(expected_packages, result_packages) def test_BuckPackage_parse(self): - test_file = self.get_test_loc('buck/parse/BUCK') + test_file = self.get_test_loc("buck/parse/BUCK") result_packages = build.BuckPackageHandler.parse(test_file) expected_packages = [ models.PackageData( - name='app', + name="app", type=build.BuckPackageHandler.default_package_type, datasource_id=build.BuckPackageHandler.datasource_id, ), models.PackageData( - name='app2', + name="app2", type=build.BuckPackageHandler.default_package_type, datasource_id=build.BuckPackageHandler.datasource_id, ), @@ -73,81 +84,155 @@ def test_BuckPackage_parse(self): compare_package_results(expected_packages, result_packages) def test_BuckPackage_recognize_with_license(self): - test_file = self.get_test_loc('buck/parse/license/BUCK') - test_loc = self.get_test_loc('buck/parse/license/') + test_file = self.get_test_loc("buck/parse/license/BUCK") + test_loc = self.get_test_loc("buck/parse/license/") result_package = list(build.BuckPackageHandler.parse(test_file))[0] codebase = Codebase(test_loc) - resource = codebase.get_resource('license/BUCK') + resource = codebase.get_resource("license/BUCK") _detections, license_expression = build.get_license_detections_and_expression( result_package, resource, codebase ) - assert license_expression == 'apache-2.0' + assert license_expression == "apache-2.0" def test_MetadataBzl_parse(self): - test_file = self.get_test_loc('metadatabzl/METADATA.bzl') + test_file = self.get_test_loc("metadatabzl/METADATA.bzl") result_packages = build.BuckMetadataBzlHandler.parse(test_file, package_only=True) package_data = dict( datasource_id=build.BuckMetadataBzlHandler.datasource_id, - type='github', - name='example', - version='0.0.1', - extracted_license_statement=['BSD-3-Clause'], - parties=[ - models.Party( - type=models.party_org, - name='oss_foundation', - role='maintainer' - ) - ], - extra_data=dict(upstream_hash='deadbeef'), - homepage_url='https://github.com/example/example', + type="github", + name="example", + version="0.0.1", + extracted_license_statement=["BSD-3-Clause"], + parties=[models.Party(type=models.party_org, name="oss_foundation", role="maintainer")], + extra_data=dict(upstream_hash="deadbeef"), + homepage_url="https://github.com/example/example", ) - expected_packages = [models.PackageData.from_data(package_data=package_data, package_only=True)] + expected_packages = [ + models.PackageData.from_data(package_data=package_data, package_only=True) + ] compare_package_results(expected_packages, result_packages) - + def test_MetadataBzl_parse_with_package_url(self): - test_file = self.get_test_loc('metadatabzl/with-package-url/METADATA.bzl') + test_file = self.get_test_loc("metadatabzl/with-package-url/METADATA.bzl") result_packages = build.BuckMetadataBzlHandler.parse(test_file, package_only=True) package_data = dict( datasource_id=build.BuckMetadataBzlHandler.datasource_id, - name='animation', - namespace='androidx.compose.animation', - type='maven', - version='0.0.1', - extracted_license_statement=['BSD-3-Clause'], - parties=[ - models.Party( - type=models.party_org, - name='oss_foundation', - role='maintainer' - ) - ], - homepage_url='https://developer.android.com/jetpack/androidx/releases/compose-animation#0.0.1', + name="animation", + namespace="androidx.compose.animation", + type="maven", + version="0.0.1", + extracted_license_statement=["BSD-3-Clause"], + parties=[models.Party(type=models.party_org, name="oss_foundation", role="maintainer")], + homepage_url="https://developer.android.com/jetpack/androidx/releases/compose-animation#0.0.1", ) - expected_packages = [models.PackageData.from_data(package_data=package_data, package_only=True)] + expected_packages = [ + models.PackageData.from_data(package_data=package_data, package_only=True) + ] compare_package_results(expected_packages, result_packages) def test_MetadataBzl_recognize_new_format(self): - test_file = self.get_test_loc('metadatabzl/new-format/METADATA.bzl') + test_file = self.get_test_loc("metadatabzl/new-format/METADATA.bzl") result_packages = build.BuckMetadataBzlHandler.parse(test_file, package_only=True) package_data = dict( datasource_id=build.BuckMetadataBzlHandler.datasource_id, - type='github', - name='example/example', - version='0.0.1', - extracted_license_statement='BSD-3-Clause', - parties=[ - models.Party( - type=models.party_org, - name='example_org', - role='maintainer' - ) - ], - download_url='', - sha1='', - homepage_url='https://github.com/example/example', - vcs_url='https://github.com/example/example.git', - extra_data=dict(vcs_commit_hash="deadbeef") + type="github", + name="example/example", + version="0.0.1", + extracted_license_statement="BSD-3-Clause", + parties=[models.Party(type=models.party_org, name="example_org", role="maintainer")], + download_url="", + sha1="", + homepage_url="https://github.com/example/example", + vcs_url="https://github.com/example/example.git", + extra_data=dict(vcs_commit_hash="deadbeef"), ) - expected_packages = [models.PackageData.from_data(package_data=package_data, package_only=True)] + expected_packages = [ + models.PackageData.from_data(package_data=package_data, package_only=True) + ] compare_package_results(expected_packages, result_packages) + + +class TestBazelModuleHandler(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), "data") + + def test_parse_basic_module(self): + location = self.get_test_loc("bazel_module/MODULE.bazel") + results = list(build.BazelModuleHandler.parse(location)) + + assert len(results) == 1 + pkg = results[0] + + # Package identity + assert pkg.name == "my_sample_project" + assert pkg.version == "0.5.0" + assert pkg.type == "bazel" + assert pkg.datasource_id == "bazel_module" + + # Total deps: 3 runtime + 1 dev = 4 + assert len(pkg.dependencies) == 4 + + def test_parse_runtime_dependencies(self): + location = self.get_test_loc("bazel_module/MODULE.bazel") + results = list(build.BazelModuleHandler.parse(location)) + pkg = results[0] + + runtime_deps = [d for d in pkg.dependencies if d.scope == "dependencies"] + assert len(runtime_deps) == 3 + + dep_names = [d.purl for d in runtime_deps] + assert "pkg:bazel/rules_python@0.24.0" in dep_names + assert "pkg:bazel/rules_go@0.41.0" in dep_names + + def test_parse_dev_dependency(self): + location = self.get_test_loc("bazel_module/MODULE.bazel") + results = list(build.BazelModuleHandler.parse(location)) + pkg = results[0] + + dev_deps = [d for d in pkg.dependencies if d.scope == "dev"] + assert len(dev_deps) == 1 + assert dev_deps[0].purl == "pkg:bazel/googletest@1.14.0" + assert dev_deps[0].is_optional is True + assert dev_deps[0].is_runtime is False + + def test_parse_module_without_version(self): + location = self.get_test_loc("bazel_module/MODULE_no_version.bazel") + results = list(build.BazelModuleHandler.parse(location)) + + assert len(results) == 1 + pkg = results[0] + assert pkg.name == "minimal_module" + assert pkg.version is None + + def test_parse_module_without_module_declaration_uses_directory_name(self): + location = self.get_test_loc("build/bazel/allocation-instrumenter/MODULE.bazel") + results = list(build.BazelModuleHandler.parse(location)) + + assert len(results) == 1 + pkg = results[0] + assert pkg.name == "allocation-instrumenter" + assert pkg.version is None + + def test_parse_module_lock_metadata(self): + location = self.get_test_loc("build/bazel/allocation-instrumenter/MODULE.bazel.lock") + results = list(build.BazelModuleLockHandler.parse(location)) + + assert len(results) == 1 + pkg = results[0] + assert pkg.datasource_id == "bazel_module_lock" + assert pkg.extra_data["bazel_lockfile_version"] == 18 + assert pkg.extra_data["bazel_registry_file_hashes_count"] == 7 + assert pkg.extra_data["bazel_module_extensions"] == [ + "@@rules_python+//python/private/pypi:pip.bzl%pip_internal" + ] + + def test_path_pattern_matches(self): + handler = build.BazelModuleHandler + assert handler.is_datafile("some/path/MODULE.bazel", _bare_filename=True) + assert not handler.is_datafile("some/path/notMODULE.bazel", _bare_filename=True) + assert not handler.is_datafile("some/path/WORKSPACE", _bare_filename=True) + assert not handler.is_datafile("some/path/BUILD", _bare_filename=True) + + def test_lock_path_pattern_matches(self): + handler = build.BazelModuleLockHandler + assert handler.is_datafile("some/path/MODULE.bazel.lock", _bare_filename=True) + assert not handler.is_datafile("some/path/MODULE.bazel", _bare_filename=True)