kbuild: parse device tree compiler errors

logspec/errors/kbuild.py

@@ -255,6 +255,130 @@ def _parse(self, text):
         return parse_end_pos
 
 
+class KbuildDtcError(Error):
+    """Models an error emitted by the Device Tree Compiler (DTC)."""
+
+    _source_diagnostic_re = re.compile(
+        rf'^{TIMESTAMP}(?P<kind>Lexical error|Error): '
+        r'(?P<src_file>.*?):'
+        r'(?P<location>\d+\.\d+(?:-\d+(?:\.\d+)?)?) '
+        r'(?P<message>.*)$',
+        flags=re.MULTILINE,
+    )
+    _check_diagnostic_re = re.compile(
+        rf'^{TIMESTAMP}(?P<src_file>.*?)'
+        r'(?::(?P<location>\d+\.\d+(?:-\d+(?:\.\d+)?)?))?'
+        r': ERROR \((?P<check>[^)]+)\): (?P<message>.*)$',
+        flags=re.MULTILINE,
+    )
+    _fatal_re = re.compile(
+        rf'^{TIMESTAMP}FATAL ERROR: (?P<message>.*)$',
+        flags=re.MULTILINE,
+    )
+    _check_terminal_re = re.compile(
+        rf'^{TIMESTAMP}ERROR: Input tree has errors,.*$',
+        flags=re.MULTILINE,
+    )
+
+    def __init__(self, script=None, target=None):
+        super().__init__()
+        self.script = script
+        self.target = target
+        self.src_file = ""
+        self.location = ""
+        self.check = ""
+
+    @classmethod
+    def has_diagnostic(cls, text):
+        """Return whether *text* contains a structured DTC error."""
+        return any(regex.search(text) for regex in (
+            cls._source_diagnostic_re,
+            cls._check_diagnostic_re,
+            cls._fatal_re,
+        ))
+
+    def _set_report_end(self, text, match, diagnostic_type):
+        """Find the terminal line belonging to a DTC diagnostic."""
+        report_end = match.end()
+        following_text = text[report_end:]
+
+        if diagnostic_type == "source":
+            terminal_match = re.match(
+                rf'\n{TIMESTAMP}FATAL ERROR: .*',
+                following_text,
+            )
+        elif diagnostic_type == "check":
+            terminal_match = self._check_terminal_re.search(
+                text, match.end()
+            )
+        else:
+            terminal_match = None
+
+        if terminal_match:
+            report_end = (
+                report_end + terminal_match.end()
+                if diagnostic_type == "source"
+                else terminal_match.end()
+            )
+        return report_end
+
+    def _parse(self, text):
+        """Extract the last DTC diagnostic preceding the Make failure."""
+        candidates = []
+        for diagnostic_type, regex in (
+            ("source", self._source_diagnostic_re),
+            ("check", self._check_diagnostic_re),
+        ):
+            matches = list(regex.finditer(text))
+            if matches:
+                candidates.append((diagnostic_type, matches[-1]))
+
+        if candidates:
+            diagnostic_type, match = max(
+                candidates, key=lambda candidate: candidate[1].start()
+            )
+        else:
+            fatal_matches = list(self._fatal_re.finditer(text))
+            if not fatal_matches:
+                return 0
+            diagnostic_type = "fatal"
+            match = fatal_matches[-1]
+
+        if diagnostic_type == "source":
+            kind = match.group('kind')
+            self.error_type = (
+                "kbuild.dtc.lexical_error"
+                if kind == "Lexical error"
+                else "kbuild.dtc.error"
+            )
+            self.src_file = match.group('src_file')
+            self.location = match.group('location')
+            self.error_summary = f"{kind}: {match.group('message')}"
+        elif diagnostic_type == "check":
+            self.error_type = "kbuild.dtc.check_error"
+            self.src_file = match.group('src_file')
+            self.location = match.group('location') or ""
+            self.check = match.group('check')
+            self.error_summary = match.group('message')
+        else:
+            self.error_type = "kbuild.dtc.fatal_error"
+            self.error_summary = match.group('message')
+            file_match = re.search(
+                r'(?:Couldn.t open|Error closing) "(?P<src_file>[^"]+)"',
+                self.error_summary,
+            )
+            if file_match:
+                self.src_file = file_match.group('src_file')
+
+        for field in ('src_file', 'location', 'check'):
+            if getattr(self, field):
+                self._signature_fields.append(field)
+
+        report_end = self._set_report_end(text, match, diagnostic_type)
+        self._report = text[match.start():report_end] + "\n"
+        return report_end
+
+
 class KbuildProcessError(Error):
     """Models the information extracted from a kbuild error caused by a
     script, configuration or other runtime error.
@@ -458,6 +582,14 @@ def _is_kbuild_target(target):
     return False
 
 
+def _is_dtc_target(script, target):
+    """Return whether a Make failure belongs to a DTC build target."""
+    return (
+        target.endswith(('.dtb', '.dtbo'))
+        or os.path.basename(script.split(':', maxsplit=1)[0]) == 'Makefile.dtbs'
+    )
+
+
 def _find_script_target(error_str, text):
     match = re.search(r'\[(?P<script>.*?): (?P<target>.*?)\] Error', error_str)
     if not match:
@@ -510,7 +642,13 @@ def find_kbuild_error(text):
         logging.debug(f"[find_kbuild_error] script: {script}, target: {target}")
         error = None
         # Kbuild error classification
-        if _is_object_file(target) or _is_other_compiler_target(target, text[:start]):
+        error_text = text[:start]
+        if (
+            _is_dtc_target(script, target)
+            and KbuildDtcError.has_diagnostic(error_text)
+        ):
+            error = KbuildDtcError(script=script, target=target)
+        elif _is_object_file(target) or _is_other_compiler_target(target, error_text):
             error = KbuildCompilerError(script=script, target=target)
         elif 'modpost' in script:
             error = KbuildModpostError(script=script, target=target)
@@ -519,8 +657,7 @@ def find_kbuild_error(text):
         else:
             # Catch-all condition for non-specific errors
             error = KbuildGenericError(script=script, target=target)
-        text = text[:start]
-        error.parse(text)
+        error.parse(error_text)
     else:
         # Unrecognized error, these are marked as unknown and not parsed
         error = KbuildUnknownError(error_str)

tests/logs/index.txt

@@ -116,6 +116,23 @@
   - kbuild_017.log: Modpost error: Section mismatches detected.
       FATAL: modpost: Section mismatches detected.
 
+  - kbuild_020.log: Device Tree Compiler lexical error
+      Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'
+      FATAL ERROR: Syntax error parsing input tree
+
+  - kbuild_021.log: Device Tree Compiler syntax error
+      Error: ../arch/arm64/boot/dts/intel/socfpga_agilex.dtsi:313.15-16 syntax error
+      FATAL ERROR: Unable to parse input tree
+
+  - kbuild_022.log: Device Tree Compiler phandle check error
+      arch/arm64/boot/dts/qcom/qcs8300.dtsi:714.22-851.5: ERROR (phandle_references): Reference to non-existent node or label
+
+  - kbuild_023.log: Device Tree Compiler overlay duplicate label error
+      arch/arm64/boot/dts/overlays/imx477_378.dtsi:26.20-31.3: ERROR (duplicate_label): Duplicate label
+
+  - kbuild_024.log: Device Tree Compiler fatal file error
+      FATAL ERROR: Couldn't open "arch/arm64/boot/dts/qcom/missing.dtsi": No such file or directory
+
 
 ./linux_boot
 

tests/logs/kbuild/kbuild_020.log

@@ -0,0 +1,7 @@
+make --silent --keep-going --jobs=16 O=/tmp/kci/artifacts/build ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu-
+Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'
+FATAL ERROR: Syntax error parsing input tree
+make[4]: *** [/tmp/kci/linux/scripts/Makefile.dtbs:140: arch/arm64/boot/dts/qcom/purwa-iot-evk.dtb] Error 1
+Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'
+FATAL ERROR: Syntax error parsing input tree
+make[4]: *** [/tmp/kci/linux/scripts/Makefile.dtbs:140: arch/arm64/boot/dts/qcom/x1p42100-crd.dtb] Error 1

tests/logs/kbuild/kbuild_021.log

@@ -0,0 +1,4 @@
+Error: ../arch/arm64/boot/dts/intel/socfpga_agilex.dtsi:313.15-16 syntax error
+FATAL ERROR: Unable to parse input tree
+make[3]: *** [scripts/Makefile.lib:314: arch/arm64/boot/dts/intel/socfpga_agilex_socdk.dtb] Error 1
+make[2]: *** [scripts/Makefile.build:497: arch/arm64/boot/dts/intel] Error 2

tests/logs/kbuild/kbuild_022.log

@@ -0,0 +1,5 @@
+arch/arm64/boot/dts/qcom/qcs8300.dtsi:714.22-851.5: ERROR (phandle_references): /soc@0/pci@1c00000: Reference to non-existent node or label "pcie_smmu"
+  also defined at arch/arm64/boot/dts/qcom/qcs8300-ride.dts:288.8-296.3
+ERROR: Input tree has errors, aborting (use -f to force output)
+make[3]: *** [scripts/Makefile.dtbs:131: arch/arm64/boot/dts/qcom/qcs8300-ride.dtb] Error 2
+make[2]: *** [scripts/Makefile.build:461: arch/arm64/boot/dts/qcom] Error 2

tests/logs/kbuild/kbuild_023.log

@@ -0,0 +1,4 @@
+arch/arm64/boot/dts/overlays/imx477_378.dtsi:26.20-31.3: ERROR (duplicate_label): /fragment@200/__overlay__/pca@70/i2c@1/cef168@d: Duplicate label 'vcm_node'
+ERROR: Input tree has errors, aborting (use -f to force output)
+make[3]: *** [scripts/Makefile.dtbs:142: arch/arm64/boot/dts/overlays/camera-mux-2port.dtbo] Error 2
+make[2]: *** [scripts/Makefile.build:544: arch/arm64/boot/dts/overlays] Error 2

tests/logs/kbuild/kbuild_024.log

@@ -0,0 +1,3 @@
+FATAL ERROR: Couldn't open "arch/arm64/boot/dts/qcom/missing.dtsi": No such file or directory
+make[3]: *** [scripts/Makefile.dtbs:142: arch/arm64/boot/dts/qcom/example.dtb] Error 1
+make[2]: *** [scripts/Makefile.build:544: arch/arm64/boot/dts/qcom] Error 2

tests/test_kbuild.py

@@ -476,6 +476,97 @@
                  "target": "vmlinux.unstripped"
              }
          ]
+     }),
+
+    # Device Tree Compiler lexical error.  The source file named by DTC may
+    # differ from the .dtb target reported by Make.
+    #
+    # Example:
+    #
+    # Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'
+    # FATAL ERROR: Syntax error parsing input tree
+    # make[4]: *** [/tmp/kci/linux/scripts/Makefile.dtbs:140: arch/arm64/boot/dts/qcom/purwa-iot-evk.dtb] Error 1
+    ('kbuild_020.log',
+     'kbuild',
+     {
+         "errors": [
+             {
+                 "error_summary": "Lexical error: Unexpected 'VIDEO_CC_MVS0_BSE_CLK'",
+                 "error_type": "kbuild.dtc.lexical_error",
+                 "location": "169.14-35",
+                 "script": "/tmp/kci/linux/scripts/Makefile.dtbs:140",
+                 "src_file": "/tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi",
+                 "target": "arch/arm64/boot/dts/qcom/purwa-iot-evk.dtb"
+             }
+         ]
+     }),
+
+    # Standard DTC parser error.  Older Makefile.lib-based DTB builds use the
+    # same source diagnostic format as current Makefile.dtbs builds.
+    ('kbuild_021.log',
+     'kbuild',
+     {
+         "errors": [
+             {
+                 "error_summary": "Error: syntax error",
+                 "error_type": "kbuild.dtc.error",
+                 "location": "313.15-16",
+                 "script": "scripts/Makefile.lib:314",
+                 "src_file": "../arch/arm64/boot/dts/intel/socfpga_agilex.dtsi",
+                 "target": "arch/arm64/boot/dts/intel/socfpga_agilex_socdk.dtb"
+             }
+         ]
+     }),
+
+    # A DTC semantic check error has a different prefix and terminates with
+    # "ERROR: Input tree has errors" rather than "FATAL ERROR".
+    ('kbuild_022.log',
+     'kbuild',
+     {
+         "errors": [
+             {
+                 "check": "phandle_references",
+                 "error_summary": "/soc@0/pci@1c00000: Reference to non-existent node or label \"pcie_smmu\"",
+                 "error_type": "kbuild.dtc.check_error",
+                 "location": "714.22-851.5",
+                 "script": "scripts/Makefile.dtbs:131",
+                 "src_file": "arch/arm64/boot/dts/qcom/qcs8300.dtsi",
+                 "target": "arch/arm64/boot/dts/qcom/qcs8300-ride.dtb"
+             }
+         ]
+     }),
+
+    # DTC check errors also occur while compiling overlays.
+    ('kbuild_023.log',
+     'kbuild',
+     {
+         "errors": [
+             {
+                 "check": "duplicate_label",
+                 "error_summary": "/fragment@200/__overlay__/pca@70/i2c@1/cef168@d: Duplicate label 'vcm_node'",
+                 "error_type": "kbuild.dtc.check_error",
+                 "location": "26.20-31.3",
+                 "script": "scripts/Makefile.dtbs:142",
+                 "src_file": "arch/arm64/boot/dts/overlays/imx477_378.dtsi",
+                 "target": "arch/arm64/boot/dts/overlays/camera-mux-2port.dtbo"
+             }
+         ]
+     }),
+
+    # Some DTC failures only emit a fatal diagnostic without a source
+    # location, for example when an input file cannot be opened.
+    ('kbuild_024.log',
+     'kbuild',
+     {
+         "errors": [
+             {
+                 "error_summary": "Couldn't open \"arch/arm64/boot/dts/qcom/missing.dtsi\": No such file or directory",
+                 "error_type": "kbuild.dtc.fatal_error",
+                 "script": "scripts/Makefile.dtbs:142",
+                 "src_file": "arch/arm64/boot/dts/qcom/missing.dtsi",
+                 "target": "arch/arm64/boot/dts/qcom/example.dtb"
+             }
+         ]
      })
 ])
 def test_kbuild(log_file, parser_id, expected):
@@ -484,3 +575,32 @@ def test_kbuild(log_file, parser_id, expected):
     expected_as_str = json.dumps(expected, indent=4, sort_keys=True, ensure_ascii=False)
     parsed_data_as_str = format_data_output(parsed_data)
     assert expected_as_str == parsed_data_as_str
+
+
+def test_kbuild_dtc_report():
+    log_file = os.path.join(LOG_DIR, 'kbuild_020.log')
+    parsed_data = load_parser_and_parse_log(
+        log_file, 'kbuild', tests.setup.PARSER_DEFS_FILE
+    )
+
+    assert parsed_data['errors'][0]._report == (
+        "Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:"
+        "169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'\n"
+        "FATAL ERROR: Syntax error parsing input tree\n"
+    )
+
+
+def test_kbuild_dtc_check_report():
+    log_file = os.path.join(LOG_DIR, 'kbuild_022.log')
+    parsed_data = load_parser_and_parse_log(
+        log_file, 'kbuild', tests.setup.PARSER_DEFS_FILE
+    )
+
+    assert parsed_data['errors'][0]._report == (
+        "arch/arm64/boot/dts/qcom/qcs8300.dtsi:714.22-851.5: "
+        "ERROR (phandle_references): /soc@0/pci@1c00000: Reference to "
+        "non-existent node or label \"pcie_smmu\"\n"
+        "  also defined at arch/arm64/boot/dts/qcom/qcs8300-ride.dts:"
+        "288.8-296.3\n"
+        "ERROR: Input tree has errors, aborting (use -f to force output)\n"
+    )