Improve licensing support

identify/cli.py

@@ -1,22 +1,46 @@
 from __future__ import annotations
 
 import argparse
+try:
+    from enum import StrEnum
+except ImportError:  # pragma: no cover
+    from strenum import StrEnum  # type: ignore
 import json
 from collections.abc import Sequence
 
 from identify import identify
 
 
+class TagType(StrEnum):
+    FILE = 'file'
+    LICENSE = 'license'
+
+
+TAG_TYPE_CHOICES = [TagType.FILE.value]
+try:
+    import ukkonen  # noqa: F401
+except ImportError:  # pragma: no cover
+    pass
+else:
+    TAG_TYPE_CHOICES.append(TagType.LICENSE.value)
+
+
 def main(argv: Sequence[str] | None = None) -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument('--filename-only', action='store_true')
+    parser.add_argument(
+        '--tag-type', choices=TAG_TYPE_CHOICES, default=TAG_TYPE_CHOICES[0],
+    )
     parser.add_argument('path')
     args = parser.parse_args(argv)
 
-    if args.filename_only:
-        func = identify.tags_from_filename
+    if args.tag_type == TagType.FILE.value:
+        func = (
+            identify.tags_from_filename
+            if args.filename_only else identify.tags_from_path
+        )
     else:
-        func = identify.tags_from_path
+        func = identify.license_ids
 
     try:
         tags = sorted(func(args.path))
@@ -26,9 +50,8 @@ def main(argv: Sequence[str] | None = None) -> int:
 
     if not tags:
         return 1
-    else:
-        print(json.dumps(tags))
-        return 0
+    print(json.dumps(tags))
+    return 0
 
 
 if __name__ == '__main__':

identify/identify.py

@@ -95,7 +95,7 @@ def tags_from_filename(path: str) -> set[str]:
             ret.update(extensions.NAMES[part])
             break
 
-    if len(ext) > 0:
+    if ext:
         ext = ext[1:].lower()
         if ext in extensions.EXTENSIONS:
             ret.update(extensions.EXTENSIONS[ext])
@@ -220,6 +220,13 @@ def parse_shebang_from_file(path: str) -> tuple[str, ...]:
 
 
 COPYRIGHT_RE = re.compile(r'^\s*(Copyright|\(C\)) .*$', re.I | re.MULTILINE)
+SPDX_CUSTOM_LICENSE_PREFIX = 'LicenseRef-'
+SPDX_LICENSE_ID_PREFIX = 'SPDX-License-Identifier:'
+SPDX_LICENSE_ID_RE = re.compile(rf'{SPDX_LICENSE_ID_PREFIX}\s+(.+)')
+# NB: the following intentionally ignores the `WITH` licensing clause
+# separator.
+SPDX_LICENSE_SEPARATORS = ['OR', 'AND']
+SPDX_LICENSE_SEPARATORS_RE = re.compile(r'\s+(OR|AND)\s+')
 WS_RE = re.compile(r'\s+')
 
 
@@ -229,24 +236,43 @@ def _norm_license(s: str) -> str:
     return s.strip()
 
 
-def license_id(filename: str) -> str | None:
-    """Return the spdx id for the license contained in `filename`.  If no
-    license is detected, returns `None`.
+def _parse_spdx_matches(spdx_matches: list[str]) -> set[str]:
+    return {
+        token.replace(SPDX_CUSTOM_LICENSE_PREFIX, '', 1)
+        for spdx_match in spdx_matches
+        for token in SPDX_LICENSE_SEPARATORS_RE.split(spdx_match)
+        if token not in SPDX_LICENSE_SEPARATORS
+    }
 
-    spdx: https://spdx.org/licenses/
+
+def license_ids(path: str) -> set[str]:
+    """Return the SPDX IDs for the license(s) contained in `path`.  If no
+    licenses are detected, return an empty set.
+
+    SPDX: https://spdx.org/licenses/
     licenses from choosealicense.com: https://github.com/choosealicense.com
 
     Approximate algorithm:
 
-    1. strip copyright line
-    2. normalize whitespace (replace all whitespace with a single space)
-    3. check exact text match with existing licenses
-    4. failing that use edit distance
+    1. scan for SPDX licenses. If any are found, parse them and return
+       immediately.
+    2. Else, do a more in-depth license scan like so:
+        a. strip copyright line
+        b. normalize whitespace (replace all whitespace with a single space)
+        c. check exact text match with existing licenses
+        d. failing that use edit distance
     """
     import ukkonen  # `pip install identify[license]`
 
-    with open(filename, encoding='UTF-8') as f:
-        contents = f.read()
+    try:
+        with open(path, encoding='UTF-8') as f:
+            contents = f.read()
+    except OSError:
+        raise ValueError(f'{path} does not exist.')
+
+    spdx_matches = SPDX_LICENSE_ID_RE.findall(contents)
+    if spdx_matches:
+        return _parse_spdx_matches(spdx_matches)
 
     norm = _norm_license(contents)
 
@@ -255,11 +281,13 @@ def license_id(filename: str) -> str | None:
 
     cutoff = math.ceil(.05 * len(norm))
 
+    spdx_license_ids = set()
+
     # try exact matches
     for spdx, text in licenses.LICENSES:
         norm_license = _norm_license(text)
         if norm == norm_license:
-            return spdx
+            spdx_license_ids |= {spdx}
 
         # skip the slow calculation if the lengths are very different
         if norm and abs(len(norm) - len(norm_license)) / len(norm) > .05:
@@ -272,7 +300,5 @@ def license_id(filename: str) -> str | None:
 
     # if there's less than 5% edited from the license, we found our match
     if norm and min_edit_dist < cutoff:
-        return min_edit_dist_spdx
-    else:
-        # no matches :'(
-        return None
+        spdx_license_ids |= {min_edit_dist_spdx}
+    return spdx_license_ids

setup.cfg

@@ -17,6 +17,8 @@ classifiers =
 
 [options]
 packages = find:
+install_requires =
+    StrEnum;python_version<"3.11"
 python_requires = >=3.10
 
 [options.packages.find]

tests/cli_test.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pytest
+
 from identify import cli
 
 
@@ -29,3 +31,22 @@ def test_file_not_found(capsys):
     out, _ = capsys.readouterr()
     assert ret == 1
     assert out == 'x.unknown does not exist.\n'
+
+
+@pytest.fixture
+def requires_licensing_support():
+    _ = pytest.importorskip('ukkonen')
+
+
+def test_copyright_has_copyright(capsys, requires_licensing_support):
+    ret = cli.main(('LICENSE', '--tag-type=license'))
+    out, _ = capsys.readouterr()
+    assert ret == 0
+    assert out == '["MIT"]\n'
+
+
+def test_copyright_full_copyright(capsys, requires_licensing_support):
+    ret = cli.main(('x.unknown', '--tag-type=license'))
+    out, _ = capsys.readouterr()
+    assert ret == 1
+    assert out == 'x.unknown does not exist.\n'

tests/identify_test.py

@@ -4,6 +4,7 @@
 import errno
 import io
 import os
+import shutil
 import socket
 import stat
 from tempfile import TemporaryDirectory
@@ -12,6 +13,8 @@
 import pytest
 
 from identify import identify
+from identify.identify import SPDX_CUSTOM_LICENSE_PREFIX
+from identify.identify import SPDX_LICENSE_ID_PREFIX
 
 
 def test_all_tags_includes_basic_ones():
@@ -376,7 +379,7 @@ def make_executable(filename):
 
 
 def test_license_identification():
-    assert identify.license_id('LICENSE') == 'MIT'
+    assert identify.license_ids('LICENSE') == {'MIT'}
 
 
 def test_license_exact_identification(tmpdir):
@@ -397,8 +400,73 @@ def test_license_exact_identification(tmpdir):
 '''
     f = tmpdir.join('LICENSE')
     f.write(wtfpl)
-    assert identify.license_id(f.strpath) == 'WTFPL'
+    assert identify.license_ids(f.strpath) == {'WTFPL'}
 
 
 def test_license_not_identified():
-    assert identify.license_id(os.devnull) is None
+    assert identify.license_ids(os.devnull) == set()
+
+
+@pytest.fixture
+def tmp_license(tmpdir):
+    tmp_lic = tmpdir / 'lic.txt'
+    yield tmp_lic
+    shutil.rmtree(tmpdir)
+
+
+@pytest.mark.parametrize(
+    'input_s,expected_output',
+    [
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} MyBogusLicense',
+            {'MyBogusLicense'},
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} Lic1\n{SPDX_LICENSE_ID_PREFIX}\tLic2',
+            {'Lic1', 'Lic2'},
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} Lic1 AND Lic2',
+            {'Lic1', 'Lic2'},
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} Lic1 AND Lic2\n{SPDX_LICENSE_ID_PREFIX} Lic3',  # noqa: E501
+            {'Lic1', 'Lic2', 'Lic3'},
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} Lic1 OR Lic2\n{SPDX_LICENSE_ID_PREFIX} Lic3',  # noqa: E501
+            {'Lic1', 'Lic2', 'Lic3'},
+        ),
+        (
+            f'Lic1 OR Lic2{SPDX_LICENSE_ID_PREFIX}',
+            set(),
+        ),
+        (
+            SPDX_LICENSE_ID_PREFIX,
+            set(),
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} AND',
+            set(),
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} OR',
+            set(),
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} A OR',
+            {'A OR'},
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} AND B',
+            {'AND B'},
+        ),
+        (
+            f'{SPDX_LICENSE_ID_PREFIX} AND {SPDX_CUSTOM_LICENSE_PREFIX}B',
+            {'AND B'},
+        ),
+    ],
+)
+def test_license_spdx(tmp_license, input_s, expected_output):
+    tmp_license.write_text(input_s, encoding='UTF-8')
+    assert identify.license_ids(str(tmp_license)) == expected_output