From 1139ba9e01d98b41127bdb6c9d457be51543bcaf Mon Sep 17 00:00:00 2001 From: Enji Cooper Date: Wed, 15 Jul 2026 18:42:42 -0700 Subject: [PATCH 1/3] identify.license_id: modify to match other identify APIs Prior to this change, the `license_id` API would return a license or `None` in the event that the file had an identifiable license or does not have a license that could be identified. This change modifies the `license_id` API to better match the other identify APIs: after this change a set of 0 or more licenses should be returned. Signed-off-by: Enji Cooper --- identify/identify.py | 21 ++++++++++++--------- tests/identify_test.py | 6 +++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/identify/identify.py b/identify/identify.py index 0279ba8e..76cfb3ac 100644 --- a/identify/identify.py +++ b/identify/identify.py @@ -229,11 +229,11 @@ def _norm_license(s: str) -> str: return s.strip() -def license_id(filename: str) -> str | None: - """Return the spdx id for the license contained in `filename`. If no - license is detected, returns `None`. +def license_id(path: str) -> set[str]: + """Return the SPDX IDs for the license(s) contained in `path`. If no + licenses are detected, return an empty set. - spdx: https://spdx.org/licenses/ + SPDX: https://spdx.org/licenses/ licenses from choosealicense.com: https://github.com/choosealicense.com Approximate algorithm: @@ -245,8 +245,11 @@ def license_id(filename: str) -> str | None: """ import ukkonen # `pip install identify[license]` - with open(filename, encoding='UTF-8') as f: - contents = f.read() + try: + with open(path, encoding='UTF-8') as f: + contents = f.read() + except OSError: + raise ValueError(f'{path} does not exist.') norm = _norm_license(contents) @@ -259,7 +262,7 @@ def license_id(filename: str) -> str | None: for spdx, text in licenses.LICENSES: norm_license = _norm_license(text) if norm == norm_license: - return spdx + return {spdx} # skip the slow calculation if the lengths are very different if norm and abs(len(norm) - len(norm_license)) / len(norm) > .05: @@ -272,7 +275,7 @@ def license_id(filename: str) -> str | None: # if there's less than 5% edited from the license, we found our match if norm and min_edit_dist < cutoff: - return min_edit_dist_spdx + return {min_edit_dist_spdx} else: # no matches :'( - return None + return set() diff --git a/tests/identify_test.py b/tests/identify_test.py index fb372bab..7e6301e2 100644 --- a/tests/identify_test.py +++ b/tests/identify_test.py @@ -376,7 +376,7 @@ def make_executable(filename): def test_license_identification(): - assert identify.license_id('LICENSE') == 'MIT' + assert identify.license_id('LICENSE') == {'MIT'} def test_license_exact_identification(tmpdir): @@ -397,8 +397,8 @@ def test_license_exact_identification(tmpdir): ''' f = tmpdir.join('LICENSE') f.write(wtfpl) - assert identify.license_id(f.strpath) == 'WTFPL' + assert identify.license_id(f.strpath) == {'WTFPL'} def test_license_not_identified(): - assert identify.license_id(os.devnull) is None + assert identify.license_id(os.devnull) == set() From 94232ed9a3c24279a1e44b75c867ce3d848cabfb Mon Sep 17 00:00:00 2001 From: Enji Cooper Date: Wed, 15 Jul 2026 22:06:08 -0700 Subject: [PATCH 2/3] Support SPDX markers instead of just full license text A lot of source code uses SPDX IDs instead of full license text for brevity. Support the shorter ID format with the license_id* API. This change renames the `license_id` API to `license_ids` for clarity. Signed-off-by: Enji Cooper --- identify/identify.py | 45 ++++++++++++++++++------- tests/identify_test.py | 74 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 14 deletions(-) diff --git a/identify/identify.py b/identify/identify.py index 76cfb3ac..be5006c5 100644 --- a/identify/identify.py +++ b/identify/identify.py @@ -95,7 +95,7 @@ def tags_from_filename(path: str) -> set[str]: ret.update(extensions.NAMES[part]) break - if len(ext) > 0: + if ext: ext = ext[1:].lower() if ext in extensions.EXTENSIONS: ret.update(extensions.EXTENSIONS[ext]) @@ -220,6 +220,13 @@ def parse_shebang_from_file(path: str) -> tuple[str, ...]: COPYRIGHT_RE = re.compile(r'^\s*(Copyright|\(C\)) .*$', re.I | re.MULTILINE) +SPDX_CUSTOM_LICENSE_PREFIX = 'LicenseRef-' +SPDX_LICENSE_ID_PREFIX = 'SPDX-License-Identifier:' +SPDX_LICENSE_ID_RE = re.compile(rf'{SPDX_LICENSE_ID_PREFIX}\s+(.+)') +# NB: the following intentionally ignores the `WITH` licensing clause +# separator. +SPDX_LICENSE_SEPARATORS = ['OR', 'AND'] +SPDX_LICENSE_SEPARATORS_RE = re.compile(r'\s+(OR|AND)\s+') WS_RE = re.compile(r'\s+') @@ -229,7 +236,16 @@ def _norm_license(s: str) -> str: return s.strip() -def license_id(path: str) -> set[str]: +def _parse_spdx_matches(spdx_matches: list[str]) -> set[str]: + return { + token.replace(SPDX_CUSTOM_LICENSE_PREFIX, '', 1) + for spdx_match in spdx_matches + for token in SPDX_LICENSE_SEPARATORS_RE.split(spdx_match) + if token not in SPDX_LICENSE_SEPARATORS + } + + +def license_ids(path: str) -> set[str]: """Return the SPDX IDs for the license(s) contained in `path`. If no licenses are detected, return an empty set. @@ -238,10 +254,13 @@ def license_id(path: str) -> set[str]: Approximate algorithm: - 1. strip copyright line - 2. normalize whitespace (replace all whitespace with a single space) - 3. check exact text match with existing licenses - 4. failing that use edit distance + 1. scan for SPDX licenses. If any are found, parse them and return + immediately. + 2. Else, do a more in-depth license scan like so: + a. strip copyright line + b. normalize whitespace (replace all whitespace with a single space) + c. check exact text match with existing licenses + d. failing that use edit distance """ import ukkonen # `pip install identify[license]` @@ -251,6 +270,10 @@ def license_id(path: str) -> set[str]: except OSError: raise ValueError(f'{path} does not exist.') + spdx_matches = SPDX_LICENSE_ID_RE.findall(contents) + if spdx_matches: + return _parse_spdx_matches(spdx_matches) + norm = _norm_license(contents) min_edit_dist = sys.maxsize @@ -258,11 +281,13 @@ def license_id(path: str) -> set[str]: cutoff = math.ceil(.05 * len(norm)) + spdx_license_ids = set() + # try exact matches for spdx, text in licenses.LICENSES: norm_license = _norm_license(text) if norm == norm_license: - return {spdx} + spdx_license_ids |= {spdx} # skip the slow calculation if the lengths are very different if norm and abs(len(norm) - len(norm_license)) / len(norm) > .05: @@ -275,7 +300,5 @@ def license_id(path: str) -> set[str]: # if there's less than 5% edited from the license, we found our match if norm and min_edit_dist < cutoff: - return {min_edit_dist_spdx} - else: - # no matches :'( - return set() + spdx_license_ids |= {min_edit_dist_spdx} + return spdx_license_ids diff --git a/tests/identify_test.py b/tests/identify_test.py index 7e6301e2..277159c6 100644 --- a/tests/identify_test.py +++ b/tests/identify_test.py @@ -4,6 +4,7 @@ import errno import io import os +import shutil import socket import stat from tempfile import TemporaryDirectory @@ -12,6 +13,8 @@ import pytest from identify import identify +from identify.identify import SPDX_CUSTOM_LICENSE_PREFIX +from identify.identify import SPDX_LICENSE_ID_PREFIX def test_all_tags_includes_basic_ones(): @@ -376,7 +379,7 @@ def make_executable(filename): def test_license_identification(): - assert identify.license_id('LICENSE') == {'MIT'} + assert identify.license_ids('LICENSE') == {'MIT'} def test_license_exact_identification(tmpdir): @@ -397,8 +400,73 @@ def test_license_exact_identification(tmpdir): ''' f = tmpdir.join('LICENSE') f.write(wtfpl) - assert identify.license_id(f.strpath) == {'WTFPL'} + assert identify.license_ids(f.strpath) == {'WTFPL'} def test_license_not_identified(): - assert identify.license_id(os.devnull) == set() + assert identify.license_ids(os.devnull) == set() + + +@pytest.fixture +def tmp_license(tmpdir): + tmp_lic = tmpdir / 'lic.txt' + yield tmp_lic + shutil.rmtree(tmpdir) + + +@pytest.mark.parametrize( + 'input_s,expected_output', + [ + ( + f'{SPDX_LICENSE_ID_PREFIX} MyBogusLicense', + {'MyBogusLicense'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} Lic1\n{SPDX_LICENSE_ID_PREFIX}\tLic2', + {'Lic1', 'Lic2'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} Lic1 AND Lic2', + {'Lic1', 'Lic2'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} Lic1 AND Lic2\n{SPDX_LICENSE_ID_PREFIX} Lic3', # noqa: E501 + {'Lic1', 'Lic2', 'Lic3'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} Lic1 OR Lic2\n{SPDX_LICENSE_ID_PREFIX} Lic3', # noqa: E501 + {'Lic1', 'Lic2', 'Lic3'}, + ), + ( + f'Lic1 OR Lic2{SPDX_LICENSE_ID_PREFIX}', + set(), + ), + ( + SPDX_LICENSE_ID_PREFIX, + set(), + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} AND', + set(), + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} OR', + set(), + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} A OR', + {'A OR'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} AND B', + {'AND B'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} AND {SPDX_CUSTOM_LICENSE_PREFIX}B', + {'AND B'}, + ), + ], +) +def test_license_spdx(tmp_license, input_s, expected_output): + tmp_license.write_text(input_s, encoding='UTF-8') + assert identify.license_ids(str(tmp_license)) == expected_output From d0f15b79abf178d7aee9c761e6faddd128e40d84 Mon Sep 17 00:00:00 2001 From: Enji Cooper Date: Wed, 15 Jul 2026 22:12:54 -0700 Subject: [PATCH 3/3] Add support for license ID'ing files This change adds a new option, `--tag-type`, which allows one to identify by either file type or license type. This exposes the `identify.license_ids(..)` API for public consumption on the CLI. Signed-off-by: Enji Cooper --- identify/cli.py | 35 +++++++++++++++++++++++++++++------ setup.cfg | 2 ++ tests/cli_test.py | 21 +++++++++++++++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/identify/cli.py b/identify/cli.py index 98c77bd6..d62f1d75 100644 --- a/identify/cli.py +++ b/identify/cli.py @@ -1,22 +1,46 @@ from __future__ import annotations import argparse +try: + from enum import StrEnum +except ImportError: # pragma: no cover + from strenum import StrEnum # type: ignore import json from collections.abc import Sequence from identify import identify +class TagType(StrEnum): + FILE = 'file' + LICENSE = 'license' + + +TAG_TYPE_CHOICES = [TagType.FILE.value] +try: + import ukkonen # noqa: F401 +except ImportError: # pragma: no cover + pass +else: + TAG_TYPE_CHOICES.append(TagType.LICENSE.value) + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('--filename-only', action='store_true') + parser.add_argument( + '--tag-type', choices=TAG_TYPE_CHOICES, default=TAG_TYPE_CHOICES[0], + ) parser.add_argument('path') args = parser.parse_args(argv) - if args.filename_only: - func = identify.tags_from_filename + if args.tag_type == TagType.FILE.value: + func = ( + identify.tags_from_filename + if args.filename_only else identify.tags_from_path + ) else: - func = identify.tags_from_path + func = identify.license_ids try: tags = sorted(func(args.path)) @@ -26,9 +50,8 @@ def main(argv: Sequence[str] | None = None) -> int: if not tags: return 1 - else: - print(json.dumps(tags)) - return 0 + print(json.dumps(tags)) + return 0 if __name__ == '__main__': diff --git a/setup.cfg b/setup.cfg index 39284d93..dc23d225 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,8 @@ classifiers = [options] packages = find: +install_requires = + StrEnum;python_version<"3.11" python_requires = >=3.10 [options.packages.find] diff --git a/tests/cli_test.py b/tests/cli_test.py index 0c954cf0..13dff249 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from identify import cli @@ -29,3 +31,22 @@ def test_file_not_found(capsys): out, _ = capsys.readouterr() assert ret == 1 assert out == 'x.unknown does not exist.\n' + + +@pytest.fixture +def requires_licensing_support(): + _ = pytest.importorskip('ukkonen') + + +def test_copyright_has_copyright(capsys, requires_licensing_support): + ret = cli.main(('LICENSE', '--tag-type=license')) + out, _ = capsys.readouterr() + assert ret == 0 + assert out == '["MIT"]\n' + + +def test_copyright_full_copyright(capsys, requires_licensing_support): + ret = cli.main(('x.unknown', '--tag-type=license')) + out, _ = capsys.readouterr() + assert ret == 1 + assert out == 'x.unknown does not exist.\n'