Enable inclusion of "All rights reserved" into extracted copyrights

pyproject.toml

@@ -272,6 +272,7 @@ facet = "summarycode.facet:AddFacet"
 info = "scancode.plugin_info:InfoScanner"
 licenses = "licensedcode.plugin_license:LicenseScanner"
 copyrights = "cluecode.plugin_copyright:CopyrightScanner"
+allrights = "cluecode.plugin_allrights:AllrightsCopyrightScanner"
 packages = "packagedcode.plugin_package:PackageScanner"
 emails = "cluecode.plugin_email:EmailScanner"
 urls = "cluecode.plugin_url:UrlScanner"

src/cluecode/copyrights.py

@@ -336,20 +336,25 @@ def detect(self,
             tree_node_label = tree_node.label
 
             if (include_copyrights or include_holders) and 'COPYRIGHT' in tree_node_label:
-                copyrght = build_detection_from_node(
+                if include_copyright_allrights:
+                    refiner = refine_allrights
+                else:
+                    refiner = refine_copyright
+
+                copyright = build_detection_from_node(
                     node=tree_node,
                     cls=CopyrightDetection,
                     ignored_labels=non_copyright_labels,
                     include_copyright_allrights=include_copyright_allrights,
-                    refiner=refine_copyright,
+                    refiner=refiner,
                 )
 
                 if TRACE or TRACE_DEEP:
-                    logger_debug(f'CopyrightDetector: final copyright: {copyrght}')
+                    logger_debug(f'CopyrightDetector: final copyright: {copyright}')
 
-                if copyrght:
+                if copyright:
                     if include_copyrights:
-                        yield copyrght
+                        yield copyright
 
                     if include_holders:
                         # By default we strip email and urls from holders ....
@@ -3562,6 +3567,31 @@ def refine_copyright(c):
     return c.strip()
 
 
+def refine_allrights(c):
+    """
+    Refine a detected copyright string.
+    FIXME: the grammar should not allow this to happen.
+    """
+    if not c:
+        return
+    c = ' '.join(c.split())
+    c = strip_some_punct(c)
+    c = strip_solo_quotes(c)
+    # this catches trailing slashes in URL for consistency
+    c = c.strip('/ ~')
+    c = strip_all_unbalanced_parens(c)
+    c = remove_some_extra_words_and_punct(c)
+    c = ' '.join(c.split())
+    c = remove_dupe_copyright_words(c)
+    c = strip_prefixes(c, prefixes=set(['by', 'c']))
+    c = c.strip()
+    c = c.strip('+')
+    c = strip_balanced_edge_parens(c)
+    c = strip_suffixes(c, suffixes=COPYRIGHTS_SUFFIXES)
+    c = c.strip("'")
+    return c.strip()
+
+
 def remove_dupe_holder(h):
     """
     Remove duplicated holders
@@ -4452,7 +4482,7 @@ def collect_candidate_lines(numbered_lines):
 remove_weird_comment_markers = re.compile(r'^(rem|\@rem|dnl)\s+').sub
 
 # common comment line prefix in man pages
-remove_man_comment_markers = re.compile(r'\."').sub
+remove_man_comment_markers = re.compile(r'^\.\\"').sub
 
 
 def remove_code_comment_markers(s):

src/cluecode/plugin_allrights.py

@@ -0,0 +1,61 @@
+#
+# This code is heavily inspired by original code from ScanCode Toolkit.
+# The copyright header is therefore propagated. The content is primarily
+# taken from following packages in the ScanCode Toolkit:
+# - cluecode.plugin_copyright
+# - scancode.api
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import attr
+from plugincode.scan import ScanPlugin
+from plugincode.scan import scan_impl
+
+from commoncode.cliutils import PluggableCommandLineOption
+from commoncode.cliutils import SCAN_GROUP
+
+
+@scan_impl
+class AllrightsCopyrightScanner(ScanPlugin):
+    """Scan a Resource for copyrights with all rights reserved. This class is an adapted version of
+    scancodes cluecode.plugin_copyright.
+    """
+
+    resource_attributes = dict(
+        [
+            ("copyrights", attr.ib(default=attr.Factory(list))),
+            ("holders", attr.ib(default=attr.Factory(list))),
+            ("authors", attr.ib(default=attr.Factory(list))),
+        ]
+    )
+
+    run_order = 6
+    sort_order = 6
+
+    options = [
+        PluggableCommandLineOption(
+            (
+                "-a",
+                "--allrights",
+            ),
+            is_flag=True,
+            default=False,
+            help="Scan <input> for copyrights and all rights reserved.",
+            help_group=SCAN_GROUP,
+            sort_order=50,
+        ),
+    ]
+
+    def is_enabled(self, allrights, **kwargs):  # NOQA
+        return allrights
+
+    def get_scanner(self, **kwargs):
+        from scancode.api import allrights_scanner
+        return allrights_scanner
+

src/licensedcode/data/licenses/autodesk-3d-sft-3.0.LICENSE

@@ -7,11 +7,10 @@ owner: Autodesk
 homepage_url: https://github.com/Toksisitee/ALACNPopWorldEditor/blob/43d4119d698a7567ab4f32ddceb4076984dce136/_3ds/3dsftk3/license.txt
 spdx_license_key: LicenseRef-scancode-autodesk-3d-sft-3.0
 ignorable_copyrights:
-    - (c) Copyright 1995 by Autodesk, Inc
+    - (c) Copyright 1995 by Autodesk, Inc.
     - (c) Copyright 1997 by Autodesk, Inc.
     - copyrighted by Autodesk, Inc.
 ignorable_holders:
-    - Autodesk, Inc
     - Autodesk, Inc.
 ---
 

src/licensedcode/data/licenses/fsf-regex-gpl.LICENSE

@@ -7,9 +7,9 @@ owner: Free Software Foundation (FSF)
 homepage_url: https://sources.debian.org/src/xxgdb/1.12-9.4potato/regex.h/
 spdx_license_key: LicenseRef-scancode-fsf-regex-gpl
 ignorable_copyrights:
-    - Copyright (c) 1985 Free Software Foundation, Inc
+    - Copyright (c) 1985 Free Software Foundation, Inc.
 ignorable_holders:
-    - Free Software Foundation, Inc
+    - Free Software Foundation, Inc.
 ---
 
 NO WARRANTY

src/licensedcode/data/licenses/gnu-emacs-gpl-1988.LICENSE

@@ -10,9 +10,9 @@ text_urls:
     - https://www.cs.bham.ac.uk/research/projects/poplog/emacs/COPYING
 ignorable_copyrights:
     - Copyright (c) 1985, 1987, 1988 Richard M. Stallman
-    - Copyright (c) 1988 Free Software Foundation, Inc
+    - Copyright (c) 1988 Free Software Foundation, Inc.
 ignorable_holders:
-    - Free Software Foundation, Inc
+    - Free Software Foundation, Inc.
     - Richard M. Stallman
 ---
 

src/licensedcode/data/licenses/ricoh-1.0.LICENSE

@@ -17,7 +17,6 @@ other_urls:
 ignorable_copyrights:
     - Copyright (c) 1995-1999
 ignorable_authors:
-    - Ricoh Silicon Valley, Inc
     - Ricoh Silicon Valley, Inc.
 ignorable_urls:
     - http://www.risource.org/RPL

src/licensedcode/data/licenses/sun-sissl-1.2.LICENSE

@@ -10,10 +10,8 @@ text_urls:
     - http://gridscheduler.sourceforge.net/Gridengine_SISSL_license.html
 minimum_coverage: 50
 ignorable_copyrights:
-    - Copyright (c) 2001 Sun Microsystems, Inc
     - Copyright (c) 2001 Sun Microsystems, Inc.
 ignorable_holders:
-    - Sun Microsystems, Inc
     - Sun Microsystems, Inc.
 ---
 

src/licensedcode/data/rules/ricoh-1.0.SPDX.RULE

@@ -6,7 +6,6 @@ notes: license text as published by SPDX
 ignorable_copyrights:
     - Copyright (c) 1995-1999
 ignorable_authors:
-    - Ricoh Silicon Valley, Inc
     - Ricoh Silicon Valley, Inc.
 ignorable_urls:
     - http://www.risource.org/RPL

src/scancode/api.py

@@ -6,16 +6,17 @@
 # See https://github.com/nexB/scancode-toolkit for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-from itertools import islice
-from os.path import getsize
 import logging
 import os
 import sys
+from itertools import islice
+from os.path import getsize
+
+from typecode.contenttype import get_type
 
 from commoncode.filetype import get_last_modified_date
 from commoncode.hash import multi_checksums
 from scancode import ScancodeError
-from typecode.contenttype import get_type
 
 TRACE = os.environ.get('SCANCODE_DEBUG_API', False)
 
@@ -30,6 +31,7 @@ def logger_debug(*args):
     logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
 
+
     def logger_debug(*args):
         return logger.debug(
             ' '.join(isinstance(a, str) and a or repr(a) for a in args)
@@ -46,9 +48,9 @@ def logger_debug(*args):
 
 
 def get_copyrights(
-    location,
-    deadline=sys.maxsize,
-    **kwargs,
+        location,
+        deadline=sys.maxsize,
+        **kwargs,
 ):
     """
     Return a mapping with a single 'copyrights' key with a value that is a list
@@ -79,12 +81,45 @@ def get_copyrights(
     return results
 
 
+def allrights_scanner(
+        location,
+        deadline=sys.maxsize,
+        **kwargs,
+):
+    """Return a mapping with a single 'copyrights' key with a value that is a list
+    of mappings for copyright detected in the file at `location`.
+    Adapted copy of scancodes scancode.api.get_copyrights.
+    """
+    from cluecode.copyrights import detect_copyrights
+    from cluecode.copyrights import Detection
+
+    detections = detect_copyrights(
+        location,
+        include_copyrights=True,
+        include_holders=True,
+        include_authors=True,
+        include_copyright_years=True,
+        include_copyright_allrights=True,
+        deadline=deadline,
+    )
+
+    copyrights, holders, authors = Detection.split(detections, to_dict=True)
+
+    results = dict([
+        ('copyrights', copyrights),
+        ('holders', holders),
+        ('authors', authors),
+    ])
+
+    return results
+
+
 def get_emails(
-    location,
-    threshold=50,
-    test_slow_mode=False,
-    test_error_mode=False,
-    **kwargs,
+        location,
+        threshold=50,
+        test_slow_mode=False,
+        test_error_mode=False,
+        **kwargs,
 ):
     """
     Return a mapping with a single 'emails' key with a value that is a list of
@@ -148,14 +183,14 @@ def get_urls(location, threshold=50, **kwargs):
 
 
 def get_licenses(
-    location,
-    min_score=0,
-    include_text=False,
-    license_text_diagnostics=False,
-    license_diagnostics=False,
-    deadline=sys.maxsize,
-    unknown_licenses=False,
-    **kwargs,
+        location,
+        min_score=0,
+        include_text=False,
+        license_text_diagnostics=False,
+        license_diagnostics=False,
+        deadline=sys.maxsize,
+        unknown_licenses=False,
+        **kwargs,
 ):
     """
     Return a mapping or license_detections for licenses detected in the file at
@@ -257,12 +292,12 @@ def get_licenses(
 
 
 def _get_package_data(
-    location,
-    application=True,
-    system=False,
-    compiled=False,
-    package_only=False, 
-    **kwargs
+        location,
+        application=True,
+        system=False,
+        compiled=False,
+        package_only=False,
+        **kwargs
 ):
     """
     Return a mapping of package manifest information detected in the file at ``location``.
@@ -309,12 +344,12 @@ def get_package_info(location, **kwargs):
 
 
 def get_package_data(
-    location,
-    application=True,
-    system=False,
-    compiled=False,
-    package_only=False,
-    **kwargs
+        location,
+        application=True,
+        system=False,
+        compiled=False,
+        package_only=False,
+        **kwargs
 ):
     """
     Return a mapping of package manifest information detected in the file at