diff --git a/identify/cli.py b/identify/cli.py index 98c77bd..d62f1d7 100644 --- a/identify/cli.py +++ b/identify/cli.py @@ -1,22 +1,46 @@ from __future__ import annotations import argparse +try: + from enum import StrEnum +except ImportError: # pragma: no cover + from strenum import StrEnum # type: ignore import json from collections.abc import Sequence from identify import identify +class TagType(StrEnum): + FILE = 'file' + LICENSE = 'license' + + +TAG_TYPE_CHOICES = [TagType.FILE.value] +try: + import ukkonen # noqa: F401 +except ImportError: # pragma: no cover + pass +else: + TAG_TYPE_CHOICES.append(TagType.LICENSE.value) + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('--filename-only', action='store_true') + parser.add_argument( + '--tag-type', choices=TAG_TYPE_CHOICES, default=TAG_TYPE_CHOICES[0], + ) parser.add_argument('path') args = parser.parse_args(argv) - if args.filename_only: - func = identify.tags_from_filename + if args.tag_type == TagType.FILE.value: + func = ( + identify.tags_from_filename + if args.filename_only else identify.tags_from_path + ) else: - func = identify.tags_from_path + func = identify.license_ids try: tags = sorted(func(args.path)) @@ -26,9 +50,8 @@ def main(argv: Sequence[str] | None = None) -> int: if not tags: return 1 - else: - print(json.dumps(tags)) - return 0 + print(json.dumps(tags)) + return 0 if __name__ == '__main__': diff --git a/identify/identify.py b/identify/identify.py index 0279ba8..be5006c 100644 --- a/identify/identify.py +++ b/identify/identify.py @@ -95,7 +95,7 @@ def tags_from_filename(path: str) -> set[str]: ret.update(extensions.NAMES[part]) break - if len(ext) > 0: + if ext: ext = ext[1:].lower() if ext in extensions.EXTENSIONS: ret.update(extensions.EXTENSIONS[ext]) @@ -220,6 +220,13 @@ def parse_shebang_from_file(path: str) -> tuple[str, ...]: COPYRIGHT_RE = re.compile(r'^\s*(Copyright|\(C\)) .*$', re.I | re.MULTILINE) +SPDX_CUSTOM_LICENSE_PREFIX = 'LicenseRef-' +SPDX_LICENSE_ID_PREFIX = 'SPDX-License-Identifier:' +SPDX_LICENSE_ID_RE = re.compile(rf'{SPDX_LICENSE_ID_PREFIX}\s+(.+)') +# NB: the following intentionally ignores the `WITH` licensing clause +# separator. +SPDX_LICENSE_SEPARATORS = ['OR', 'AND'] +SPDX_LICENSE_SEPARATORS_RE = re.compile(r'\s+(OR|AND)\s+') WS_RE = re.compile(r'\s+') @@ -229,24 +236,43 @@ def _norm_license(s: str) -> str: return s.strip() -def license_id(filename: str) -> str | None: - """Return the spdx id for the license contained in `filename`. If no - license is detected, returns `None`. +def _parse_spdx_matches(spdx_matches: list[str]) -> set[str]: + return { + token.replace(SPDX_CUSTOM_LICENSE_PREFIX, '', 1) + for spdx_match in spdx_matches + for token in SPDX_LICENSE_SEPARATORS_RE.split(spdx_match) + if token not in SPDX_LICENSE_SEPARATORS + } - spdx: https://spdx.org/licenses/ + +def license_ids(path: str) -> set[str]: + """Return the SPDX IDs for the license(s) contained in `path`. If no + licenses are detected, return an empty set. + + SPDX: https://spdx.org/licenses/ licenses from choosealicense.com: https://github.com/choosealicense.com Approximate algorithm: - 1. strip copyright line - 2. normalize whitespace (replace all whitespace with a single space) - 3. check exact text match with existing licenses - 4. failing that use edit distance + 1. scan for SPDX licenses. If any are found, parse them and return + immediately. + 2. Else, do a more in-depth license scan like so: + a. strip copyright line + b. normalize whitespace (replace all whitespace with a single space) + c. check exact text match with existing licenses + d. failing that use edit distance """ import ukkonen # `pip install identify[license]` - with open(filename, encoding='UTF-8') as f: - contents = f.read() + try: + with open(path, encoding='UTF-8') as f: + contents = f.read() + except OSError: + raise ValueError(f'{path} does not exist.') + + spdx_matches = SPDX_LICENSE_ID_RE.findall(contents) + if spdx_matches: + return _parse_spdx_matches(spdx_matches) norm = _norm_license(contents) @@ -255,11 +281,13 @@ def license_id(filename: str) -> str | None: cutoff = math.ceil(.05 * len(norm)) + spdx_license_ids = set() + # try exact matches for spdx, text in licenses.LICENSES: norm_license = _norm_license(text) if norm == norm_license: - return spdx + spdx_license_ids |= {spdx} # skip the slow calculation if the lengths are very different if norm and abs(len(norm) - len(norm_license)) / len(norm) > .05: @@ -272,7 +300,5 @@ def license_id(filename: str) -> str | None: # if there's less than 5% edited from the license, we found our match if norm and min_edit_dist < cutoff: - return min_edit_dist_spdx - else: - # no matches :'( - return None + spdx_license_ids |= {min_edit_dist_spdx} + return spdx_license_ids diff --git a/setup.cfg b/setup.cfg index 39284d9..dc23d22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,8 @@ classifiers = [options] packages = find: +install_requires = + StrEnum;python_version<"3.11" python_requires = >=3.10 [options.packages.find] diff --git a/tests/cli_test.py b/tests/cli_test.py index 0c954cf..13dff24 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from identify import cli @@ -29,3 +31,22 @@ def test_file_not_found(capsys): out, _ = capsys.readouterr() assert ret == 1 assert out == 'x.unknown does not exist.\n' + + +@pytest.fixture +def requires_licensing_support(): + _ = pytest.importorskip('ukkonen') + + +def test_copyright_has_copyright(capsys, requires_licensing_support): + ret = cli.main(('LICENSE', '--tag-type=license')) + out, _ = capsys.readouterr() + assert ret == 0 + assert out == '["MIT"]\n' + + +def test_copyright_full_copyright(capsys, requires_licensing_support): + ret = cli.main(('x.unknown', '--tag-type=license')) + out, _ = capsys.readouterr() + assert ret == 1 + assert out == 'x.unknown does not exist.\n' diff --git a/tests/identify_test.py b/tests/identify_test.py index fb372ba..277159c 100644 --- a/tests/identify_test.py +++ b/tests/identify_test.py @@ -4,6 +4,7 @@ import errno import io import os +import shutil import socket import stat from tempfile import TemporaryDirectory @@ -12,6 +13,8 @@ import pytest from identify import identify +from identify.identify import SPDX_CUSTOM_LICENSE_PREFIX +from identify.identify import SPDX_LICENSE_ID_PREFIX def test_all_tags_includes_basic_ones(): @@ -376,7 +379,7 @@ def make_executable(filename): def test_license_identification(): - assert identify.license_id('LICENSE') == 'MIT' + assert identify.license_ids('LICENSE') == {'MIT'} def test_license_exact_identification(tmpdir): @@ -397,8 +400,73 @@ def test_license_exact_identification(tmpdir): ''' f = tmpdir.join('LICENSE') f.write(wtfpl) - assert identify.license_id(f.strpath) == 'WTFPL' + assert identify.license_ids(f.strpath) == {'WTFPL'} def test_license_not_identified(): - assert identify.license_id(os.devnull) is None + assert identify.license_ids(os.devnull) == set() + + +@pytest.fixture +def tmp_license(tmpdir): + tmp_lic = tmpdir / 'lic.txt' + yield tmp_lic + shutil.rmtree(tmpdir) + + +@pytest.mark.parametrize( + 'input_s,expected_output', + [ + ( + f'{SPDX_LICENSE_ID_PREFIX} MyBogusLicense', + {'MyBogusLicense'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} Lic1\n{SPDX_LICENSE_ID_PREFIX}\tLic2', + {'Lic1', 'Lic2'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} Lic1 AND Lic2', + {'Lic1', 'Lic2'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} Lic1 AND Lic2\n{SPDX_LICENSE_ID_PREFIX} Lic3', # noqa: E501 + {'Lic1', 'Lic2', 'Lic3'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} Lic1 OR Lic2\n{SPDX_LICENSE_ID_PREFIX} Lic3', # noqa: E501 + {'Lic1', 'Lic2', 'Lic3'}, + ), + ( + f'Lic1 OR Lic2{SPDX_LICENSE_ID_PREFIX}', + set(), + ), + ( + SPDX_LICENSE_ID_PREFIX, + set(), + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} AND', + set(), + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} OR', + set(), + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} A OR', + {'A OR'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} AND B', + {'AND B'}, + ), + ( + f'{SPDX_LICENSE_ID_PREFIX} AND {SPDX_CUSTOM_LICENSE_PREFIX}B', + {'AND B'}, + ), + ], +) +def test_license_spdx(tmp_license, input_s, expected_output): + tmp_license.write_text(input_s, encoding='UTF-8') + assert identify.license_ids(str(tmp_license)) == expected_output