From 08cdc9d29e301c04e34d07d4f44fc66093957656 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 2 Jul 2026 20:05:12 -0600 Subject: [PATCH 1/7] feat(locations): LocationData.code factory + get_locations() hash symmetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a LocationData.code() factory so parsers can emit static-analysis code coordinates as first-class locations: identity is file_path (+ line), while volatile context (snippet, end_line, sink/source metadata) is carried separately and omitted when unset, so downstream location subtypes can route it onto the finding reference instead of the location identity. Also fixes an asymmetry in Finding.get_locations(): the saved path filters a finding's location references to URL type before feeding the "endpoints" hash ingredient, but the unsaved path hashed every unsaved location type (dependencies included). A finding whose hash was computed before save could therefore never match its own hash recomputed after save. The unsaved path now applies the same URL-type filter. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Fable 5 --- dojo/finding/models.py | 11 +++- dojo/tools/locations.py | 30 +++++++++++ unittests/test_location_data.py | 89 +++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 unittests/test_location_data.py diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 8a55d3df9e6..86ab58c2d72 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -860,9 +860,18 @@ def _get_unsaved_locations(finding) -> str: from dojo.importers.location_manager import ( # noqa: PLC0415 -- lazy import, avoids circular dependency LocationManager, ) + from dojo.url.models import URL # noqa: PLC0415 -- lazy import, avoids circular dependency unsaved_locations = LocationManager.clean_unsaved_locations(finding.unsaved_locations) + # Only URL locations feed the "endpoints" hash ingredient — the + # saved path below filters to URL references, and hashing other + # location types here would make a finding's hash change between + # pre-save and post-save computation. + locations = sorted({ + location.get_location_value() + for location in unsaved_locations + if location.get_location_type() == URL.get_location_type() + }) # deduplicate (usually done upon saving finding) and sort locations - locations = sorted({location.get_location_value() for location in unsaved_locations}) return "".join(locations) # we can get here when the parser defines static_finding=True but leaves dynamic_finding defaulted # In this case, before saving the finding, both static_finding and dynamic_finding are True diff --git a/dojo/tools/locations.py b/dojo/tools/locations.py index 8b74a2817d9..0351c34dc37 100644 --- a/dojo/tools/locations.py +++ b/dojo/tools/locations.py @@ -41,6 +41,36 @@ def url( }, ) + @classmethod + def code( + cls, + *, + file_path: str = "", + line: int | None = None, + end_line: int | None = None, + snippet: str = "", + source_object: str = "", + sink_object: str = "", + source_file_path: str = "", + source_line: int | None = None, + ) -> LocationData: + """ + A static-analysis code coordinate. Identity is file_path (+ line); + the remaining keys are volatile context expected to ride the finding + reference rather than the location identity, so unset ones are omitted. + """ + data: dict[str, Any] = {"file_path": file_path, "line": line} + context = { + "end_line": end_line, + "snippet": snippet, + "source_object": source_object, + "sink_object": sink_object, + "source_file_path": source_file_path, + "source_line": source_line, + } + data.update({key: value for key, value in context.items() if value not in {"", None}}) + return cls(type="code", data=data) + @classmethod def dependency( cls, diff --git a/unittests/test_location_data.py b/unittests/test_location_data.py new file mode 100644 index 00000000000..85ad900b252 --- /dev/null +++ b/unittests/test_location_data.py @@ -0,0 +1,89 @@ +from django.utils import timezone + +from dojo.models import Engagement, Finding, Product, Product_Type, Test, Test_Type +from dojo.tools.locations import LocationData +from unittests.dojo_test_case import DojoTestCase, skip_unless_v3 + + +class TestLocationDataCodeFactory(DojoTestCase): + + def test_code_factory_identity_keys(self): + data = LocationData.code(file_path="src/db/queries.py", line=42) + self.assertEqual("code", data.type) + self.assertEqual("src/db/queries.py", data.data["file_path"]) + self.assertEqual(42, data.data["line"]) + + def test_code_factory_omits_unset_context(self): + data = LocationData.code(file_path="src/db/queries.py", line=42) + for key in ("end_line", "snippet", "source_object", "sink_object", "source_file_path", "source_line"): + self.assertNotIn(key, data.data) + + def test_code_factory_keeps_populated_context(self): + data = LocationData.code( + file_path="src/db/queries.py", + line=42, + end_line=44, + snippet='q = f"SELECT * FROM users"', + source_object="user_id", + sink_object="execute", + ) + self.assertEqual(44, data.data["end_line"]) + self.assertEqual('q = f"SELECT * FROM users"', data.data["snippet"]) + self.assertEqual("user_id", data.data["source_object"]) + self.assertEqual("execute", data.data["sink_object"]) + self.assertNotIn("source_file_path", data.data) + self.assertNotIn("source_line", data.data) + + def test_code_factory_line_none_is_allowed(self): + data = LocationData.code(file_path="Dockerfile") + self.assertEqual("Dockerfile", data.data["file_path"]) + self.assertIsNone(data.data["line"]) + + +@skip_unless_v3 +class TestGetLocationsHashSymmetry(DojoTestCase): + + @classmethod + def setUpTestData(cls): + product_type = Product_Type.objects.create(name="hash symmetry type") + product = Product.objects.create(name="hash symmetry product", description="hash symmetry", prod_type=product_type) + engagement = Engagement.objects.create( + product=product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + cls.test = Test.objects.create( + engagement=engagement, + test_type=Test_Type.objects.create(name="hash symmetry scanner"), + target_start=timezone.now(), + target_end=timezone.now(), + ) + + def _finding(self): + return Finding(test=self.test, title="Hash symmetry", severity="Low") + + def test_unsaved_non_url_locations_do_not_enter_the_endpoints_ingredient(self): + # The saved path filters references to URL locations; the unsaved path + # must filter the same way or a finding's hash changes across save + finding = self._finding() + finding.unsaved_locations = [ + LocationData.dependency(purl="pkg:npm/lodash@4.17.21", purl_type="npm", name="lodash", version="4.17.21"), + ] + self.assertEqual("", finding.get_locations()) + + def test_unsaved_url_locations_still_hash(self): + finding = self._finding() + finding.unsaved_locations = [ + LocationData.url(url="https://example.com/login", host="example.com", protocol="https", path="login"), + ] + self.assertIn("example.com", finding.get_locations()) + + def test_mixed_unsaved_locations_hash_only_urls(self): + finding = self._finding() + finding.unsaved_locations = [ + LocationData.url(url="https://example.com/login", host="example.com", protocol="https", path="login"), + LocationData.dependency(purl="pkg:npm/lodash@4.17.21", purl_type="npm", name="lodash", version="4.17.21"), + ] + value = finding.get_locations() + self.assertIn("example.com", value) + self.assertNotIn("lodash", value) From 5ac58a696315846820e1f06bfddaf80e20f8969d Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:00:51 -0600 Subject: [PATCH 2/7] docs(locations): document source code locations and location drift matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new pages plus cross-links from the existing docs: - asset_modelling/locations/PRO__source_code_locations.md — the Code location subtype: file+line identity, scan-managed lifecycle, the All Source Code / asset-scoped views, the movement-history trail, and the toggle-independent reference status sync. - triage_findings/finding_deduplication/PRO__location_drift_matching.md — the full feature guide: the churn problem, enabling the per-tool toggle, the two-stage matching model (stable identity + evidence passes per location type), severity re-score tolerance, scan-owned field refresh with human-edit protection, scan-pure hash adoption, location history, upgrade/enablement guidance for existing data, hash-field guidance (all-volatile fallback, location-in-title), the dependency dedupe stable-key interaction, the historical churn consolidation command, and the safeguards/limits. - PRO__deduplication_tuning.md — Content Fingerprint hash field (with the backfill-before-select ordering) and the Track Findings as Locations Change toggle, plus best-practice entries. - PRO__locations_overview.md — Source Code Locations join the subtype list. - import_data/import_intro/reimport.md — cross-link from the "line number shift" troubleshooting note to the drift matching page. Co-Authored-By: Claude Fable 5 --- .../locations/PRO__locations_overview.md | 5 +- .../locations/PRO__source_code_locations.md | 44 ++++++ .../import_data/import_intro/reimport.md | 2 + .../PRO__deduplication_tuning.md | 20 +++ .../PRO__location_drift_matching.md | 134 ++++++++++++++++++ 5 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 docs/content/asset_modelling/locations/PRO__source_code_locations.md create mode 100644 docs/content/triage_findings/finding_deduplication/PRO__location_drift_matching.md diff --git a/docs/content/asset_modelling/locations/PRO__locations_overview.md b/docs/content/asset_modelling/locations/PRO__locations_overview.md index 95c60dbe69a..4fa5f99cf44 100644 --- a/docs/content/asset_modelling/locations/PRO__locations_overview.md +++ b/docs/content/asset_modelling/locations/PRO__locations_overview.md @@ -17,12 +17,13 @@ The original Endpoints model was built around URLs and IP addresses — it carri 2. **Performance ceiling.** Per-Finding Endpoint_Status rows and the URL-shaped schema did not scale well at large customer volumes. 3. **Components were second-class.** Software libraries lived only as denormalised fields on a Finding, so a library could not exist independently of a vulnerability — making true SBOM management impossible. -Locations fix all three by introducing a **base `Location` object** with a typed payload, plus dedicated **subtypes** for each asset shape. The MVP ships two subtypes: +Locations fix all three by introducing a **base `Location` object** with a typed payload, plus dedicated **subtypes** for each asset shape: - **URL Locations** — functional equivalent of the old Endpoints, with the same protocol/host/port/path/query/fragment fields. - **Dependency Locations** — software libraries identified by [Package URL (pURL)](https://github.com/package-url/purl-spec), used to model SBOM contents. +- **[Source Code Locations](/asset_modelling/locations/pro__source_code_locations/)** — where a static-analysis finding lives in source, identified by file path and line number. Scan-managed, and the substrate for [tracking findings as their code moves](/triage_findings/finding_deduplication/pro__location_drift_matching/). -Future Location types under consideration include cloud provider resource IDs (AWS ARN, Azure Resource ID, GCP Full Resource Name), container images (registry/repository:tag and SHA256 fingerprints), and code repositories. +Future Location types under consideration include cloud provider resource IDs (AWS ARN, Azure Resource ID, GCP Full Resource Name) and container images (registry/repository:tag and SHA256 fingerprints). ## Key Concepts diff --git a/docs/content/asset_modelling/locations/PRO__source_code_locations.md b/docs/content/asset_modelling/locations/PRO__source_code_locations.md new file mode 100644 index 00000000000..dd963a958b3 --- /dev/null +++ b/docs/content/asset_modelling/locations/PRO__source_code_locations.md @@ -0,0 +1,44 @@ +--- +title: "Source Code Locations" +description: "Code locations model where a static-analysis finding lives in source, and record its movement history as code evolves" +weight: 6 +audience: pro +--- + +**Source Code Locations** extend the Locations model to static analysis: alongside URLs (DAST) and Dependencies (SCA), a **Code** location describes where a SAST finding lives in source — identified by its **file path and line number**. + +> Source Code Locations require the Locations feature (Beta). To enable Locations on your instance, contact [support@defectdojo.com](mailto:support@defectdojo.com). + +## What They Model + +Every static finding that reports a file path gets a Code location. The location's canonical value is `path/to/file.py:42` (or just the file path when the tool reports no line). Like all Locations, code locations are shared objects: two findings at the same file and line reference the same location, and the location carries per-finding and per-asset reference statuses. + +Code locations are **scan-managed**: they are created and updated by imports and reimports, not by hand. There is no "New Source Code Location" action — the scanner is the source of truth for where code findings live. + +## Where to Find Them + +- **All Source Code** in the sidebar lists every code location in the instance, with the same filtering and tagging as URLs and Dependencies. +- **View Source Code** in an Asset's Locations menu scopes the list to one asset. +- A finding's page shows its current code location and, when the finding has moved, its **location history**. + +## Movement History + +Source code moves constantly: commits shift line numbers, refactors rename files. When [Location Drift Matching](/triage_findings/finding_deduplication/pro__location_drift_matching/) is enabled for a tool, a finding that moves keeps its identity, and its code location references record the trail: + +- The finding's reference to the **old** location is mitigated and stamped with *where the finding moved* and *why the match was made* (nearest line, dataflow, file rename ...). +- A reference to the **new** location is created and stays active. + +The result is a browsable supersession chain — "this finding lived at `auth.py:42`, then `auth.py:57`, then `session.py:31`" — rendered as a timeline on the finding page. The same history mechanism covers URL moves and dependency version bumps, so all three location types share one timeline UI. + +History is recorded from the moment Locations is enabled on the instance. Findings that moved before that keep their current location; past hops were applied but not recorded. For instances with years of pre-feature history, the [churn consolidation command](/triage_findings/finding_deduplication/pro__location_drift_matching/#consolidating-historical-churn) can reconstruct trails while merging historical close-and-recreate chains. + +## Status Correctness + +Code location reference statuses are kept truthful by reimport on **every** matching algorithm, whether or not drift matching is enabled: + +- A matched finding's current code reference is synced on each reimport, so a finding that moved does not leave its old reference active forever. +- The same toggle-independent sync applies to dependency references: when an SCA finding's package version bumps, the old version's reference is mitigated rather than remaining active alongside the new one. + +## Relationship to Finding Fields + +The finding's own `file_path` / `line` fields remain the authoritative scalars (they are what filters, hashes, and the API expose); the Code location is the shared, reference-counted view of that same coordinate. Reimport refreshes the scalars from the latest scan and the location machinery derives locations from them — the two can not drift apart. diff --git a/docs/content/import_data/import_intro/reimport.md b/docs/content/import_data/import_intro/reimport.md index 04615686c28..0df9b5b4aeb 100644 --- a/docs/content/import_data/import_intro/reimport.md +++ b/docs/content/import_data/import_intro/reimport.md @@ -106,6 +106,8 @@ Reimport decides whether an incoming item matches an existing Finding using **[R If you are seeing Reimport close old Findings and create new Findings when only a minor attribute changes (for example, a line number shift), tune **Reimport Deduplication** for that tool to use stable identifiers that ignore those attributes (such as Unique ID From Tool). +**DefectDojo Pro** can solve this directly for tools without reliable unique IDs: enabling **[Location Drift Matching](/triage_findings/finding_deduplication/pro__location_drift_matching/)** makes Reimport recognize a Finding whose location moved — a line shift, file rename, URL move, or dependency version bump — as the *same* Finding, updating it in place and preserving its location history. + ## Reimport via API - special note Note that the /reimport API endpoint can both **extend an existing Test** (apply the method in this article) **or create a new Test** with new data \- an initial call to `/import`, or setting up a Test in advance is not required. diff --git a/docs/content/triage_findings/finding_deduplication/PRO__deduplication_tuning.md b/docs/content/triage_findings/finding_deduplication/PRO__deduplication_tuning.md index e2eb9ec45bd..195cde91094 100644 --- a/docs/content/triage_findings/finding_deduplication/PRO__deduplication_tuning.md +++ b/docs/content/triage_findings/finding_deduplication/PRO__deduplication_tuning.md @@ -39,6 +39,16 @@ DefectDojo Pro offers the following deduplication methods for same-tool deduplic #### Hash Code Uses a combination of selected fields to generate a unique hash. When selected, a third dropdown will appear showing the fields being used to calculate the hash. +##### Content Fingerprint + +**Content Fingerprint** is a selectable hash field (available in all three configuration areas) that provides a *location-invariant* identity for static-analysis findings. It is derived from the vulnerable code snippet a tool includes in the finding — normalized so that indentation, line-number annotations, and formatting differences do not change it. Two findings about the same vulnerable code hash identically even when the code moved to a different line or file. + +Content Fingerprint is computed for tools that include a code snippet in the finding description — including **Bandit**, **Gosec**, **Brakeman**, **Checkmarx One**, and any tool whose description carries a fenced code block or SARIF snippet. + +> **Before selecting Content Fingerprint as a hash field**, populate fingerprints for existing findings by running `./manage.py backfill_fingerprints`. Findings imported after the feature is present get fingerprints automatically, but pre-existing findings have none — selecting the field without backfilling makes existing and incoming findings hash differently, splitting every match until the backfill runs. + +Content Fingerprint pairs well with **CWE** for tools that embed file paths or line numbers inside their titles, where other identity fields change every time the code moves. See [Location Drift Matching](/triage_findings/finding_deduplication/pro__location_drift_matching/#choosing-hash-fields-for-tracked-tools). + #### Unique ID From Tool Leverages the security tool's own internal identifier for findings, ensuring perfect deduplication when the scanner provides reliable unique IDs. @@ -88,6 +98,14 @@ The following algorithm options are available for Reimport Deduplication: Reimport can completely discard Findings before they are recorded, so Reimport Deduplication settings should be adjusted with caution. +### Track Findings as Locations Change + +When a tool's Reimport Deduplication algorithm is **Hash Code**, an additional toggle appears: **Track findings as locations change**. With it enabled, a finding whose location moved between reimports — a line shift or file rename, a URL move, or a dependency version bump — is treated as the *same* finding, even if the tool re-scored its severity. One finding is maintained in place and its location history is preserved, instead of the old finding closing and an identical new one being created. + +The toggle is off by default and applies only to the Hash Code reimport algorithm (tools with a reliable Unique ID From Tool already track movement through their stable IDs). Enabling it automatically re-hashes the tool's existing findings in the background so historical data participates immediately. + +See [Location Drift Matching](/triage_findings/finding_deduplication/pro__location_drift_matching/) for how the matching works, what is preserved, and guidance for enabling it on large instances. + ## Running Deduplication Retroactively on Existing Data A common situation when first turning on Deduplication Tuning is having a large backlog of Findings that were imported *before* the dedup configuration changed. In DefectDojo Pro, you do not need to run a separate command to dedupe this historical data — **changing the Deduplication Settings for a tool automatically triggers a background re-hash of all existing Findings associated with that test type**. @@ -112,6 +130,8 @@ For optimal results with Deduplication Tuning: - **Use Hash Code for cross-tool deduplication**: When enabling cross-tool deduplication, select fields that reliably identify the same finding across different tools (such as vulnerability name, location, and severity). **IMPORTANT** Each tool enabled for cross-tool deduplication **MUST** have the same fields selected. - **Keep cross-tool sources in the same Asset**: Cross-Tool Deduplication is Asset-scoped. Findings split across separate Assets will not dedupe even with matching hash fields. See [Cross-Tool Deduplication is Scoped to a Single Asset](#cross-tool-deduplication-is-scoped-to-a-single-asset) above. - **Avoid overly broad deduplication**: Cross-tool deduplication with too few hash fields may result in false duplicates +- **Backfill before selecting Content Fingerprint**: run `./manage.py backfill_fingerprints` first, then select the field — the triggered re-hash then has fingerprints to work with. See [Content Fingerprint](#content-fingerprint) above. +- **Enable location tracking between scan runs**: the toggle's automatic re-hash covers the tool's whole backlog; on large instances let it finish before the next scheduled reimport. See [Location Drift Matching](/triage_findings/finding_deduplication/pro__location_drift_matching/#enabling-on-existing-data-upgrades). By tuning deduplication settings to your specific tools, you can significantly reduce duplicate noise. diff --git a/docs/content/triage_findings/finding_deduplication/PRO__location_drift_matching.md b/docs/content/triage_findings/finding_deduplication/PRO__location_drift_matching.md new file mode 100644 index 00000000000..42adf1c0f90 --- /dev/null +++ b/docs/content/triage_findings/finding_deduplication/PRO__location_drift_matching.md @@ -0,0 +1,134 @@ +--- +title: "Location Drift Matching (Pro)" +description: "Track findings as their locations change across reimports — line shifts, file renames, URL moves, and dependency version bumps no longer close and recreate findings" +weight: 6 +audience: pro +--- + +**Location Drift Matching** lets reimport recognize a finding whose *location* moved as the **same finding**. Without it, reimport matches findings by an exact identity hash that includes location fields — so every location movement closes the old finding and creates an identical new one: + +- A commit shifts code and the finding's **line number** changes. +- A refactor **renames or moves the file**. +- A web application's **URL, port, or host** changes between DAST scans. +- A dependency **version bump** changes the vulnerable package version an SCA tool reports. + +Each of these previously produced a closed finding plus a "new" finding — losing the status, notes, SLA clock, risk acceptance, and JIRA linkage on the original, and generating false "new critical finding" noise. With Location Drift Matching enabled, one finding is maintained in place: its location is updated from the latest scan and its history is preserved. + +> Location Drift Matching is a DefectDojo Pro feature. It is **off by default** and enabled per security tool. + +## Enabling Location Tracking + +Location tracking is configured per tool under: +**Settings > Pro Settings > Deduplication Settings > Reimport Deduplication** + +1. Select the **Security Tool**. +2. Set the **Deduplication Algorithm** to **Hash Code**. Location tracking applies to the Hash Code algorithm only — tools with a reliable **Unique ID From Tool** already track movement through their stable IDs and do not need it. +3. Enable **Track findings as locations change**. + +Saving the setting automatically triggers a background re-hash of the tool's existing findings (see [Enabling on Existing Data](#enabling-on-existing-data-upgrades) below), so findings imported before the toggle participate immediately. + +## How Matching Works + +With tracking enabled, reimport matching happens in two stages: + +1. **Stable identity.** The reimport hash is computed *without* the volatile location fields (line, file path, description, component name/version, endpoints) — so a finding's identity captures *what* the finding is, not *where* it currently lives. Findings that did not move still match exactly, first, and are never disturbed. +2. **Evidence pairing.** Within each group of findings that share a stable identity, a location matcher pairs incoming findings with existing ones using location evidence, in deterministic passes from strongest to weakest. A finding is routed to exactly one matcher based on the location data it carries. + +### Code findings (SAST) + +| Pass | Pairs when | Notes | +|------|-----------|-------| +| Exact | Same file and line | Always wins; a moved neighbor can never "steal" an unmoved finding's match | +| Dataflow | Same source/sink objects (`sast_source_object` / `sast_sink_object`) | For tools that report dataflow; immune to line renumbering | +| Nearest line | Same file, closest line number | Greedy, closest-first; same-file only | +| File rename | Different file | Only when exactly **one** incoming and **one** existing finding remain — ambiguity fails closed | + +### URL findings (DAST) + +| Pass | Pairs when | +|------|-----------| +| Exact | Identical endpoint set | +| Endpoint set drift | Overlapping endpoint sets (endpoints added/removed) | +| Port move | Same host and path, different port | +| Path drift | Same host, similar path (mutual-best segment similarity) | +| Host move | Different host — only as an unambiguous 1×1 pairing, with a wildcard-DNS guard | + +### Dependency findings (SCA) + +| Pass | Pairs when | +|------|-----------| +| Exact | Same package, version, and manifest | +| Version bump | Same package, different version | +| Manifest move | Same package, different lockfile/manifest path | + +When the same vulnerable package appears in **several manifests**, each manifest's finding is tracked independently — a version bump in one lockfile never swallows the finding from another. + +### Severity re-scores + +Security tools re-score severities as their rule engines evolve. With tracking enabled, a tool-reported severity change does **not** split a finding's identity: the finding matches, and its severity is updated from the scan — unless a person has re-triaged the severity by hand, in which case the human's value always wins (see below). + +## What Is Preserved, What Refreshes + +A drift-matched finding keeps everything that matters about its lifecycle: status, notes, risk acceptance, SLA dates, JIRA linkage, and its finding ID. + +Its **location fields** (file path, line, dataflow fields, endpoints, component version) refresh from the incoming scan. + +Its **descriptive fields** (title, description, severity, component version) refresh from the scan *only when the scan still owns them*: DefectDojo records a digest of each field as last written by import/reimport. If the current value still matches that digest, the tool wrote it and the scan may update it; if a person edited the field since, the human's value is preserved permanently. Findings created before this feature have no digests and are treated as human-owned — reimport will never overwrite their descriptive fields. The single exception is **component version**, which is scan telemetry that people essentially never hand-edit: it refreshes even without a digest, so migrated SCA findings still receive version updates. + +### Identity always tracks the tool's report + +When a matched finding is refreshed, its stored identity hashes are **adopted from the incoming scan's values** — never recomputed from the finding's current fields. This distinction matters: the finding's fields after a refresh are a *merge* of scan values and human edits, and a hash computed from that merge would contain values no scan will ever report again, silently breaking every future reimport for that finding. Adoption guarantees that a person renaming a finding, re-triaging its severity, or editing its description can never break its ability to match the next scan. + +## Location History + +Under **Locations** (Beta), every drift match records where the finding used to live: the superseded source-code location, URL, or dependency version is kept as a reference on the finding, stamped with where it moved and why. The finding's location timeline — "this finding lived at `auth.py:42`, then `auth.py:57`, then `session.py:31`" — is visible on the finding page. See [Source Code Locations](/asset_modelling/locations/pro__source_code_locations/). + +Location Drift Matching itself works **with or without** the Locations feature: matching pairs on the finding's own fields and endpoints, so findings survive movement either way. Locations adds the recorded, visible history on top. History starts recording from the moment Locations is enabled — earlier moves were applied but not recorded. + +## Enabling on Existing Data (Upgrades) + +The feature is designed to be self-migrating: + +- **Nothing changes until you opt in.** With the toggle off, reimport hashes compute exactly as before. +- **Saving the toggle re-hashes existing findings.** The background job recomputes the tool's stored reimport hashes with the new (location-free) identity, and creates any missing Pro finding records for data migrated from open-source. Once it completes, old and new findings speak the same identity language — a finding imported months ago is tracked exactly like one imported yesterday. +- **Enable between scan runs on large instances.** The re-hash is a background job over the tool's whole finding population. A reimport that lands while it is mid-flight can see a mix of old and new hashes and churn the unprocessed slice once. Flip the toggle at a quiet time, and let the job finish before the next scheduled reimport. +- **Hand-edited titles.** The opt-in re-hash computes from current database values. Every commonly-edited field is excluded from the tracked identity — severity edits are actually *healed* by the re-hash — but if a person renamed a finding's **title** (and title is a hash field for that tool), that one finding will churn once on its next reimport before stabilizing. + +## Choosing Hash Fields for Tracked Tools + +Location tracking removes the volatile location fields from the reimport hash automatically — you do not need to remove `line` or `file_path` from a tool's hash configuration yourself. Two configurations deserve attention: + +- **All-volatile configurations.** If a tool's hash fields are *entirely* location fields (for example just `file_path` + `line`), stripping them leaves nothing, and the hash falls back to the legacy title+CWE identity. Matching still works — the evidence passes carry the discrimination — but identity is much coarser. Prefer configurations that keep at least one stable content field. +- **Location embedded in stable fields.** Field exclusions cannot help when location data hides *inside* a field that must stay in the hash. A tool that titles findings "SQL Injection in queries.py:42" changes its title on every line move — the identity splits and tracking cannot see the pair. For such tools, choose hash fields that avoid the leaking field; **CWE + Content Fingerprint** is the strong combination (see [Content Fingerprint](/triage_findings/finding_deduplication/pro__deduplication_tuning/#content-fingerprint)). + +## Interaction with Deduplication + +Location tracking is a **reimport** feature: Same Tool and Cross Tool Deduplication are unchanged — their hashes compute exactly as before, and the exclusions never apply to them. Two deliberate integrations: + +- **Version bumps no longer block dependency deduplication.** The deduplication location gate normally requires two SCA findings to reference the *identical* package version. For tracking-enabled tools, a shared package identity (ecosystem + package name, with the namespace compared whenever both sides carry one) is enough — consistent with reimport treating a version bump as the same finding. This applies to Same Tool deduplication under Locations only. +- **Clean identity inputs.** Because matched findings adopt scan-reported hashes, the values deduplication consumes always reflect what the tool last reported — human edits can no longer contaminate them. + +## Consolidating Historical Churn + +Instances that ran for years without tracking accumulate close-and-recreate chains: the same finding closed and reopened as a new record every time it moved. A management command finds those chains (linked hop-by-hop by the same matchers, with a lifetime-overlap guard so findings that genuinely coexisted never merge) and consolidates each chain onto its most recent finding, marking the older copies as duplicates of the survivor: + +```bash +# Dry run — reports what would be consolidated, changes nothing +./manage.py consolidate_location_churn --product + +# Apply, with a confirmation prompt +./manage.py consolidate_location_churn --product --apply +``` + +The command is dry-run by default, never runs automatically, and can be scoped with `--test` or `--product`. Under Locations, the survivor's location history is reconstructed from the chain. + +## Safeguards and Limits + +- **Exact matches always win.** An unmoved finding pairs exactly before any fuzzy pass runs; movers can never steal its match. +- **Ambiguity fails closed.** File renames and host moves pair only when exactly one candidate remains on each side. Two findings that both disappeared while two new ones appeared stay unmatched rather than guess. +- **Very large groups degrade gracefully.** If a single identity bucket exceeds the pairing cap (40,000 comparisons), matching degrades to exact-only for that bucket instead of consuming unbounded time. +- **Accepted trade-off:** the 1×1 rename/host-move passes can create false continuity when one finding disappears and an unrelated finding with the same stable identity appears in the same reimport. This is the deliberate price of tracking renames; the stable identity (same tool, title, CWE, severity ...) bounds how wrong the pairing can be. + +## Location Refresh Without the Toggle + +Independent of location tracking, reimport keeps every matched finding's location current on **all** algorithms: a finding matched by Unique ID From Tool (or any other algorithm) refreshes its `line`, `file_path`, dataflow fields, and `component_version` from the incoming report, and reported endpoints are attached while vanished ones are mitigated. Values a scan omits never overwrite existing data, and a human-pinned component version is preserved. This closes the long-standing gap where uid-matched SAST findings displayed the line number from their first import forever. It can be disabled instance-wide with `DD_REIMPORT_REFRESH_LOCATION_FIELDS=False`. From 58938cd200d482079393a498449deb5418d2b8b7 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:43:10 -0600 Subject: [PATCH 3/7] feat(locations): parsers emit LocationData.code for source-code findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows the method PR #14395 established for dependencies: rather than downstream synthesis from finding scalars alone, static parsers now emit their code coordinates as LocationData.code() into unsaved_locations, gated on V3_FEATURE_LOCATIONS, alongside the unchanged scalar fields. Context the parser already extracts rides along (snippet, end_line, source/sink objects, taint source file/line) so location subtypes can route it onto the finding reference instead of the location identity. Coverage: 45 parser directories converted (~60 emission sites) — SAST, secret scanners, and IaC scanners. SCA parsers are deliberately skipped (their file_path is a dependency manifest; they already emit LocationData.dependency), as are dynamic findings and non-source paths (container-image filesystem hits, JIRA/Confluence URLs, cloud service names). Mixed parsers (trivy, checkmarx, aws_inspector2, wizcli, xygeni, mobsf, deepfence, rusty_hog, snyk_issue_api) emit only for their code-shaped findings. Multi-location: SARIF explodes results into one finding per location, each with its own emission; xygeni secrets emits one location per leaked line. Fabricated line numbers (trufflehog's line=0 dedup placeholder) are never propagated — real coordinates only. Secret scanners never pass matched secret content as snippets. unittests/test_code_location_emission.py exercises nine representative parsers against real fixtures: emission coherence with the finding scalars, snippet context, SARIF per-location behavior, and a gate-off class proving the sweep is inert without V3 (full unittests.tools suite: 1531 tests green with V3 off). Known follow-ups (out of scope): mend and meterian have no dependency emission from #14395 either; github_vulnerability's code-scanning branch parses no location fields at all today. Co-Authored-By: Claude Fable 5 --- dojo/tools/api_sonarqube/importer.py | 9 ++ dojo/tools/aws_inspector2/parser.py | 4 + dojo/tools/bandit/parser.py | 10 ++ dojo/tools/bearer_cli/parser.py | 13 ++ dojo/tools/brakeman/parser.py | 10 ++ dojo/tools/checkmarx/parser.py | 25 ++++ dojo/tools/checkmarx_cxflow_sast/parser.py | 21 +++ dojo/tools/checkmarx_one/parser.py | 31 ++++- dojo/tools/checkov/parser.py | 10 +- dojo/tools/codechecker/parser.py | 7 + dojo/tools/coverity_api/parser.py | 7 + dojo/tools/coverity_scan/parser.py | 8 ++ dojo/tools/cred_scan/parser.py | 7 + dojo/tools/deepfence_threatmapper/secret.py | 17 ++- dojo/tools/detect_secrets/parser.py | 6 + dojo/tools/eslint/parser.py | 8 ++ dojo/tools/fortify/fpr_parser.py | 13 ++ dojo/tools/fortify/xml_parser.py | 35 +++-- dojo/tools/generic/json_parser.py | 13 ++ dojo/tools/ggshield/parser.py | 11 ++ dojo/tools/github_sast/parser.py | 8 ++ .../github_secrets_detection_report/parser.py | 11 ++ dojo/tools/gitlab_sast/parser.py | 16 +++ .../gitlab_secret_detection_report/parser.py | 12 ++ dojo/tools/gitleaks/parser.py | 12 ++ dojo/tools/gosec/parser.py | 12 ++ dojo/tools/hadolint/parser.py | 8 ++ dojo/tools/hcl_asoc_sast/parser.py | 6 + dojo/tools/horusec/parser.py | 6 + dojo/tools/huskyci/parser.py | 15 ++- dojo/tools/kics/parser.py | 7 + dojo/tools/kiuwan/parser.py | 10 ++ dojo/tools/mobsf/api_report_json.py | 6 + dojo/tools/mobsf/report.py | 7 + dojo/tools/noseyparker/parser.py | 11 ++ dojo/tools/php_security_audit_v2/parser.py | 8 ++ dojo/tools/pmd/parser.py | 10 ++ dojo/tools/progpilot/parser.py | 14 ++ dojo/tools/pwn_sast/parser.py | 8 ++ dojo/tools/rubocop/parser.py | 7 + dojo/tools/rusty_hog/parser.py | 13 ++ dojo/tools/sarif/parser.py | 15 +++ dojo/tools/semgrep/parser.py | 8 ++ dojo/tools/semgrep_pro/parser.py | 8 ++ dojo/tools/snyk_issue_api/parser.py | 8 ++ dojo/tools/solar_appscreener/parser.py | 8 ++ .../tools/sonarqube/sonarqube_restapi_json.py | 14 ++ dojo/tools/sonarqube/soprasteria_helper.py | 15 +++ dojo/tools/spotbugs/parser.py | 11 ++ dojo/tools/talisman/parser.py | 8 ++ dojo/tools/terrascan/parser.py | 7 + dojo/tools/tfsec/parser.py | 7 + dojo/tools/trivy/parser.py | 8 ++ dojo/tools/trivy_operator/secrets_handler.py | 7 + dojo/tools/trufflehog/parser.py | 13 ++ dojo/tools/trufflehog3/parser.py | 12 ++ dojo/tools/veracode/json_parser.py | 10 ++ dojo/tools/veracode/xml_parser.py | 9 ++ dojo/tools/whispers/parser.py | 48 ++++--- dojo/tools/wizcli_common_parsers/parsers.py | 14 ++ dojo/tools/xanitizer/parser.py | 10 ++ dojo/tools/xygeni/sast.py | 15 +++ dojo/tools/xygeni/secrets.py | 15 ++- unittests/test_code_location_emission.py | 123 ++++++++++++++++++ 64 files changed, 825 insertions(+), 40 deletions(-) create mode 100644 unittests/test_code_location_emission.py diff --git a/dojo/tools/api_sonarqube/importer.py b/dojo/tools/api_sonarqube/importer.py index d0ae635cfa0..3560e4d4305 100644 --- a/dojo/tools/api_sonarqube/importer.py +++ b/dojo/tools/api_sonarqube/importer.py @@ -11,6 +11,7 @@ from dojo.models import Finding, Sonarqube_Issue from dojo.notifications.helper import create_notification +from dojo.tools.locations import LocationData from .api_client import SonarQubeAPI @@ -240,6 +241,10 @@ def import_issues(self, test): sonarqube_issue=sonarqube_issue, unique_id_from_tool=issue.get("key"), ) + if settings.V3_FEATURE_LOCATIONS and component_key: + find.unsaved_locations.append( + LocationData.code(file_path=component_key, line=line), + ) items.append(find) except Exception as e: @@ -361,6 +366,10 @@ def import_hotspots(self, test): sonarqube_issue=sonarqube_issue, unique_id_from_tool=f"hotspot:{hotspot.get('key')}", ) + if settings.V3_FEATURE_LOCATIONS and component_key: + find.unsaved_locations.append( + LocationData.code(file_path=component_key, line=line), + ) items.append(find) except Exception as e: diff --git a/dojo/tools/aws_inspector2/parser.py b/dojo/tools/aws_inspector2/parser.py index 2174d06e977..a4752276f46 100644 --- a/dojo/tools/aws_inspector2/parser.py +++ b/dojo/tools/aws_inspector2/parser.py @@ -190,6 +190,10 @@ def get_code_vulnerability(self, finding: Finding, raw_finding: dict) -> Finding finding.sast_source_file_path = f"{file_path}{file_name}" finding.line = start_line finding.sast_source_line = start_line + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=start_line, end_line=end_line), + ) if start_line is None: start_line = "N/A" if end_line is None: diff --git a/dojo/tools/bandit/parser.py b/dojo/tools/bandit/parser.py index 8f4861bd218..9cccb14e73f 100644 --- a/dojo/tools/bandit/parser.py +++ b/dojo/tools/bandit/parser.py @@ -1,8 +1,10 @@ import json import dateutil.parser +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData class BanditParser: @@ -105,6 +107,14 @@ def get_findings(self, filename, test): finding.scanner_confidence = confidence if "more_info" in item: finding.references = item["more_info"] + if settings.V3_FEATURE_LOCATIONS and item["filename"]: + finding.unsaved_locations.append( + LocationData.code( + file_path=item["filename"], + line=item["line_number"], + snippet=item.get("code") or "", + ), + ) results.append(finding) diff --git a/dojo/tools/bearer_cli/parser.py b/dojo/tools/bearer_cli/parser.py index 4fafeb26861..aaf48dc1142 100644 --- a/dojo/tools/bearer_cli/parser.py +++ b/dojo/tools/bearer_cli/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class BearerCLIParser: @@ -49,6 +52,16 @@ def get_findings(self, file, test): # the fingerprint is not constant over time, but because it's not used for dedupe it's safe and useful to set it unique_id_from_tool=bearerfinding["fingerprint"], ) + if settings.V3_FEATURE_LOCATIONS and bearerfinding["filename"]: + finding.unsaved_locations.append( + LocationData.code( + file_path=bearerfinding["filename"], + line=bearerfinding["line_number"], + snippet=bearerfinding.get("snippet", bearerfinding.get("code_extract")) or "", + source_file_path=bearerfinding["filename"], + source_line=bearerfinding["source"]["start"], + ), + ) items.append(finding) diff --git a/dojo/tools/brakeman/parser.py b/dojo/tools/brakeman/parser.py index 09d0c2df8d8..6a015d3560b 100644 --- a/dojo/tools/brakeman/parser.py +++ b/dojo/tools/brakeman/parser.py @@ -3,8 +3,10 @@ import json from dateutil import parser +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData class BrakemanParser: @@ -112,6 +114,14 @@ def get_findings(self, filename, test): date=find_date, static_finding=True, ) + if settings.V3_FEATURE_LOCATIONS and item["file"]: + find.unsaved_locations.append( + LocationData.code( + file_path=item["file"], + line=item["line"], + snippet=item["code"] or "", + ), + ) dupes[dupe_key] = find diff --git a/dojo/tools/checkmarx/parser.py b/dojo/tools/checkmarx/parser.py index 3cdf42f5560..cf626de78a2 100644 --- a/dojo/tools/checkmarx/parser.py +++ b/dojo/tools/checkmarx/parser.py @@ -4,8 +4,10 @@ from dateutil import parser from defusedxml import ElementTree +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData from dojo.utils import add_language logger = logging.getLogger(__name__) @@ -233,6 +235,10 @@ def _process_result_file_name_aggregated( static_finding=True, nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and sinkFilename: + find.unsaved_locations.append( + LocationData.code(file_path=sinkFilename), + ) dupes[aggregateKeys] = find # a list containing the vuln_id_from_tool values. They are # formatted once we have analysed all the findings @@ -360,6 +366,17 @@ def _process_result_detailed( sast_source_file_path=sourceFilename, vuln_id_from_tool=queryId, ) + if settings.V3_FEATURE_LOCATIONS and sinkFilename: + find.unsaved_locations.append( + LocationData.code( + file_path=sinkFilename, + line=sinkLineNumber, + source_object=sourceObject, + sink_object=sinkObject, + source_file_path=sourceFilename, + source_line=sourceLineNumber, + ), + ) dupes[aggregateKeys] = find # Return filename, lineNumber and object (function/parameter...) for a @@ -478,6 +495,10 @@ def _get_findings_json(self, file, test): last_node = vulnerability["nodes"][-1] finding.file_path = last_node.get("fileName") finding.line = last_node.get("line") + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=finding.line), + ) finding.unsaved_tags = [result_type] findings.append(finding) if result_type == "sca" and results.get(result_type) is not None: @@ -554,6 +575,10 @@ def _get_findings_json(self, file, test): finding.unique_id_from_tool = str( vulnerability.get("similarityId"), ) + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=finding.line), + ) finding.unsaved_tags = [result_type, name] findings.append(finding) return findings diff --git a/dojo/tools/checkmarx_cxflow_sast/parser.py b/dojo/tools/checkmarx_cxflow_sast/parser.py index f35dfca36a9..209fdb7fed8 100644 --- a/dojo/tools/checkmarx_cxflow_sast/parser.py +++ b/dojo/tools/checkmarx_cxflow_sast/parser.py @@ -3,12 +3,21 @@ from dataclasses import dataclass import dateutil.parser +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData logger = logging.getLogger(__name__) +def _to_int(value): + try: + return int(value) + except (TypeError, ValueError): + return None + + class _PathNode: def __init__(self, file: str, line: str, column: str, node_object: str, length: str, snippet: str): self.file = file @@ -131,6 +140,18 @@ def _get_findings_json(self, file, test): verified=self.is_verify(detail.state), active=self.is_active(detail.state), ) + if settings.V3_FEATURE_LOCATIONS and filename: + finding.unsaved_locations.append( + LocationData.code( + file_path=filename, + line=_to_int(detail.sink.line) if detail.sink is not None else None, + snippet=(detail.sink.snippet if detail.sink is not None else "") or "", + source_object=(detail.source.node_object if detail.source is not None else "") or "", + sink_object=(detail.sink.node_object if detail.sink is not None else "") or "", + source_file_path=(detail.source.file if detail.source is not None else "") or "", + source_line=_to_int(detail.source.line) if detail.source is not None else None, + ), + ) findings.append(finding) diff --git a/dojo/tools/checkmarx_one/parser.py b/dojo/tools/checkmarx_one/parser.py index ac25cc07acf..366e1a05760 100644 --- a/dojo/tools/checkmarx_one/parser.py +++ b/dojo/tools/checkmarx_one/parser.py @@ -6,6 +6,7 @@ from django.conf import settings from dojo.models import Finding, Test +from dojo.tools.locations import LocationData class CheckmarxOneParser: @@ -100,6 +101,10 @@ def parse_iac_vulnerabilities( ) # Add at tag indicating what kind of finding this is finding.unsaved_tags = ["iac"] + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path), + ) # Add the finding to the running list findings.append(finding) return findings @@ -188,6 +193,10 @@ def get_node_snippet(nodes: list) -> str: finding.description += f"\n---\n{node_snippet}" # Add at tag indicating what kind of finding this is finding.unsaved_tags = ["sast"] + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=finding.line), + ) # Add the finding to the running list findings.append(finding) return findings @@ -223,6 +232,14 @@ def parse_vulnerabilities( dynamic_finding=False, **self.determine_state(result), ) + if settings.V3_FEATURE_LOCATIONS and locations_uri: + finding.unsaved_locations.append( + LocationData.code( + file_path=locations_uri, + line=locations_startLine, + end_line=locations_endLine, + ), + ) findings.append(finding) return findings @@ -263,7 +280,7 @@ def get_results_sast( if description is None: description = vulnerability.get("severity").title() + " " + vulnerability.get("data").get("queryName").replace("_", " ") - return Finding( + finding = Finding( description=description, title=description, file_path=file_path, @@ -273,6 +290,11 @@ def get_results_sast( unique_id_from_tool=unique_id_from_tool, **self.determine_state(vulnerability), ) + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path), + ) + return finding def get_results_kics( self, @@ -285,7 +307,7 @@ def get_results_kics( if description is None: description = vulnerability.get("severity").title() + " " + vulnerability.get("data").get("queryName").replace("_", " ") - return Finding( + finding = Finding( title=description, description=description, severity=vulnerability.get("severity").title(), @@ -295,6 +317,11 @@ def get_results_kics( unique_id_from_tool=unique_id_from_tool, **self.determine_state(vulnerability), ) + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path), + ) + return finding def get_results_sca( self, diff --git a/dojo/tools/checkov/parser.py b/dojo/tools/checkov/parser.py index edbc02b5af1..ed124c0ff89 100644 --- a/dojo/tools/checkov/parser.py +++ b/dojo/tools/checkov/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class CheckovParser: @@ -146,7 +149,7 @@ def get_item(vuln, test, check_type): severity = vuln["severity"].capitalize() references = vuln.get("guideline", "") - return Finding( + finding = Finding( title=title, test=test, description=description, @@ -159,3 +162,8 @@ def get_item(vuln, test, check_type): static_finding=True, dynamic_finding=False, ) + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=source_line), + ) + return finding diff --git a/dojo/tools/codechecker/parser.py b/dojo/tools/codechecker/parser.py index f1bf1586650..6b2d6b78dc1 100644 --- a/dojo/tools/codechecker/parser.py +++ b/dojo/tools/codechecker/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class CodeCheckerParser: @@ -114,6 +117,10 @@ def get_item(vuln): static_finding=True, dynamic_finding=False, ) + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=line), + ) finding.unsaved_tags = [vuln["analyzer_name"]] return finding diff --git a/dojo/tools/coverity_api/parser.py b/dojo/tools/coverity_api/parser.py index 3b3a72a7862..13b558a695f 100644 --- a/dojo/tools/coverity_api/parser.py +++ b/dojo/tools/coverity_api/parser.py @@ -1,7 +1,10 @@ import json from datetime import datetime +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class CoverityApiParser: @@ -63,6 +66,10 @@ def get_findings(self, file, test): if "displayFile" in issue: finding.file_path = issue["displayFile"] + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=None), + ) if "occurrenceCount" in issue: finding.nb_occurences = int(issue["occurrenceCount"]) diff --git a/dojo/tools/coverity_scan/parser.py b/dojo/tools/coverity_scan/parser.py index 18d701fd338..d8635d204f5 100644 --- a/dojo/tools/coverity_scan/parser.py +++ b/dojo/tools/coverity_scan/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class CoverityScanParser: @@ -65,6 +68,11 @@ def get_findings(self, file, test): vuln_id_from_tool=vuln_id, ) + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=finding.line), + ) + findings.append(finding) return findings diff --git a/dojo/tools/cred_scan/parser.py b/dojo/tools/cred_scan/parser.py index cd99d57f670..4c4dde4f1d5 100644 --- a/dojo/tools/cred_scan/parser.py +++ b/dojo/tools/cred_scan/parser.py @@ -2,8 +2,10 @@ import io from dateutil import parser +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData class CredScanParser: @@ -57,6 +59,11 @@ def get_findings(self, filename, test): file_path=row["Source"], line=row["Line"], ) + if settings.V3_FEATURE_LOCATIONS and row["Source"]: + line = int(row["Line"]) if str(row["Line"]).isdigit() else None + finding.unsaved_locations.append( + LocationData.code(file_path=row["Source"], line=line), + ) # Update the finding date if it specified if "TimeofDiscovery" in row: finding.date = parser.parse( diff --git a/dojo/tools/deepfence_threatmapper/secret.py b/dojo/tools/deepfence_threatmapper/secret.py index 1915e4be694..c5e3e6ffc26 100644 --- a/dojo/tools/deepfence_threatmapper/secret.py +++ b/dojo/tools/deepfence_threatmapper/secret.py @@ -1,4 +1,7 @@ +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class DeepfenceThreatmapperSecret: @@ -29,7 +32,7 @@ def _parse_old_format(self, row, headers, test): description += f"**Content:** {Content}\n" description += f"**Signature:** {Signature}\n" if Name and Severity: - return Finding( + finding = Finding( title=str(Name), description=description, file_path=Filename, @@ -38,6 +41,11 @@ def _parse_old_format(self, row, headers, test): dynamic_finding=True, test=test, ) + if settings.V3_FEATURE_LOCATIONS and Filename: + finding.unsaved_locations.append( + LocationData.code(file_path=Filename, line=None), + ) + return finding return None def _parse_new_format(self, row, headers, test): @@ -61,7 +69,7 @@ def _parse_new_format(self, row, headers, test): description += f"**Masked:** {Masked}\n" title = f"{Rule} in {Filename}" if Rule else "Secret Finding" if Severity: - return Finding( + finding = Finding( title=title, description=description, file_path=Filename, @@ -70,6 +78,11 @@ def _parse_new_format(self, row, headers, test): dynamic_finding=True, test=test, ) + if settings.V3_FEATURE_LOCATIONS and Filename: + finding.unsaved_locations.append( + LocationData.code(file_path=Filename, line=None), + ) + return finding return None def severity(self, severity_input): diff --git a/dojo/tools/detect_secrets/parser.py b/dojo/tools/detect_secrets/parser.py index 53a00393d23..d51e56d4e45 100644 --- a/dojo/tools/detect_secrets/parser.py +++ b/dojo/tools/detect_secrets/parser.py @@ -2,8 +2,10 @@ import json import dateutil.parser +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData class DetectSecretsParser: @@ -61,5 +63,9 @@ def get_findings(self, filename, test): false_p="is_secret" in item and item["is_secret"] is False, ) + if settings.V3_FEATURE_LOCATIONS and file: + finding.unsaved_locations.append( + LocationData.code(file_path=file, line=line), + ) dupes[dupe_key] = finding return list(dupes.values()) diff --git a/dojo/tools/eslint/parser.py b/dojo/tools/eslint/parser.py index 613545e214b..e35ee478f01 100644 --- a/dojo/tools/eslint/parser.py +++ b/dojo/tools/eslint/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class ESLintParser: @@ -60,5 +63,10 @@ def get_findings(self, filename, test): impact="N/A", ) + if settings.V3_FEATURE_LOCATIONS and item["filePath"]: + find.unsaved_locations.append( + LocationData.code(file_path=item["filePath"], line=message["line"]), + ) + items.append(find) return items diff --git a/dojo/tools/fortify/fpr_parser.py b/dojo/tools/fortify/fpr_parser.py index b13689e565e..47b37c0e7d8 100644 --- a/dojo/tools/fortify/fpr_parser.py +++ b/dojo/tools/fortify/fpr_parser.py @@ -3,9 +3,11 @@ from xml.etree.ElementTree import Element from defusedxml import ElementTree +from django.conf import settings from dojo.models import Finding, Test from dojo.tools.fortify.fortify_data import DescriptionData, RuleData, SnippetData, VulnerabilityData +from dojo.tools.locations import LocationData from dojo.tools.utils import safe_read_all_zip logger = logging.getLogger(__name__) @@ -139,6 +141,17 @@ def convert_vulnerabilities_to_findings(self, root: Element, audit_log: Element, finding.line = int(self.compute_line(vuln_data, snippet)) finding.unique_id_from_tool = vuln_data.instance_id + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + end_line = vuln_data.source_location_line_end + finding.unsaved_locations.append( + LocationData.code( + file_path=finding.file_path, + line=finding.line, + end_line=int(end_line) if end_line and str(end_line).isdigit() else None, + snippet=snippet.text if snippet and snippet.text else "", + ), + ) + findings.append(finding) return findings diff --git a/dojo/tools/fortify/xml_parser.py b/dojo/tools/fortify/xml_parser.py index 57c4bb29fbe..82e189de8a8 100644 --- a/dojo/tools/fortify/xml_parser.py +++ b/dojo/tools/fortify/xml_parser.py @@ -141,19 +141,30 @@ def xml_structure_before_24_2(self, root, test): issue["Category"], issue["FileName"], issue["LineStart"], ) if title not in dupes: - items.append( - Finding( - title=title, - severity=issue["Friority"], - file_path=issue["FilePath"], - line=int(issue["LineStart"]), - static_finding=True, - test=test, - description=self.format_description(issue, cat_meta), - mitigation=self.format_mitigation(issue, cat_meta), - unique_id_from_tool=issue_key, - ), + finding = Finding( + title=title, + severity=issue["Friority"], + file_path=issue["FilePath"], + line=int(issue["LineStart"]), + static_finding=True, + test=test, + description=self.format_description(issue, cat_meta), + mitigation=self.format_mitigation(issue, cat_meta), + unique_id_from_tool=issue_key, ) + if settings.V3_FEATURE_LOCATIONS and issue["FilePath"]: + source = issue.get("Source") or {} + source_line = source.get("LineStart") + finding.unsaved_locations.append( + LocationData.code( + file_path=issue["FilePath"], + line=int(issue["LineStart"]), + snippet=issue["Snippet"] if issue["Snippet"] != "n/a" else "", + source_file_path=source.get("FilePath") or "", + source_line=int(source_line) if source_line and str(source_line).isdigit() else None, + ), + ) + items.append(finding) dupes.add(title) return items diff --git a/dojo/tools/generic/json_parser.py b/dojo/tools/generic/json_parser.py index c177db5fc58..cbe528e0b2a 100644 --- a/dojo/tools/generic/json_parser.py +++ b/dojo/tools/generic/json_parser.py @@ -156,6 +156,19 @@ def _get_test_json(self, data): file_path=file_path, ), ) + if file_path: + line = item.get("line") + source_line = item.get("sast_source_line") + finding.unsaved_locations.append( + LocationData.code( + file_path=file_path, + line=int(line) if line is not None and str(line).isdigit() else None, + source_object=item.get("sast_source_object") or "", + sink_object=item.get("sast_sink_object") or "", + source_file_path=item.get("sast_source_file_path") or "", + source_line=int(source_line) if source_line is not None and str(source_line).isdigit() else None, + ), + ) if unsaved_files: for unsaved_file in unsaved_files: data = base64.b64decode(unsaved_file.get("data")) diff --git a/dojo/tools/ggshield/parser.py b/dojo/tools/ggshield/parser.py index a64ad6f6b14..6d569a83bfc 100644 --- a/dojo/tools/ggshield/parser.py +++ b/dojo/tools/ggshield/parser.py @@ -2,8 +2,10 @@ import json from dateutil import parser +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData class GgshieldParser: @@ -101,6 +103,15 @@ def get_items(self, item, findings, dupes, test): date=findings["commit_date"], ) + if settings.V3_FEATURE_LOCATIONS and findings["file_path"]: + finding.unsaved_locations.append( + LocationData.code( + file_path=findings["file_path"], + line=line_start, + end_line=line_end, + ), + ) + key = hashlib.md5( ( title diff --git a/dojo/tools/github_sast/parser.py b/dojo/tools/github_sast/parser.py index 7d8275fc27f..59780027243 100644 --- a/dojo/tools/github_sast/parser.py +++ b/dojo/tools/github_sast/parser.py @@ -1,7 +1,10 @@ import json from urllib.parse import urlparse +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class GithubSASTParser: @@ -78,6 +81,11 @@ def get_findings(self, filename, test): finding.file_path = loc.get("path") finding.line = loc.get("start_line") + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=finding.line), + ) + if html_url: finding.url = html_url diff --git a/dojo/tools/github_secrets_detection_report/parser.py b/dojo/tools/github_secrets_detection_report/parser.py index 4b2f0eedddc..2b1bf864e45 100644 --- a/dojo/tools/github_secrets_detection_report/parser.py +++ b/dojo/tools/github_secrets_detection_report/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class GithubSecretsDetectionReportParser: @@ -136,6 +139,14 @@ def get_findings(self, file, test): if first_location: finding.file_path = first_location.get("path") finding.line = first_location.get("start_line") + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code( + file_path=finding.file_path, + line=finding.line, + end_line=first_location.get("end_line"), + ), + ) # Set external URL if html_url: diff --git a/dojo/tools/gitlab_sast/parser.py b/dojo/tools/gitlab_sast/parser.py index 456e5b98c6c..61b2676674a 100644 --- a/dojo/tools/gitlab_sast/parser.py +++ b/dojo/tools/gitlab_sast/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData from dojo.tools.parser_test import ParserTest @@ -162,4 +165,17 @@ def get_item(self, vuln, scanner): if vulnerability_id: finding.unsaved_vulnerability_ids = [vulnerability_id] + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code( + file_path=file_path, + line=location.get("start_line"), + end_line=location.get("end_line"), + source_object=sast_object or "", + sink_object=sast_object or "", + source_file_path=sast_source_file_path or "", + source_line=sast_source_line, + ), + ) + return finding diff --git a/dojo/tools/gitlab_secret_detection_report/parser.py b/dojo/tools/gitlab_secret_detection_report/parser.py index 5398ae87371..097c2483e62 100644 --- a/dojo/tools/gitlab_secret_detection_report/parser.py +++ b/dojo/tools/gitlab_secret_detection_report/parser.py @@ -1,7 +1,10 @@ import json from datetime import datetime +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class GitlabSecretDetectionReportParser: @@ -61,6 +64,15 @@ def get_findings(self, file, test): "\n" + vulnerability["raw_source_code_extract"] ) + if settings.V3_FEATURE_LOCATIONS and location.get("file"): + finding.unsaved_locations.append( + LocationData.code( + file_path=location["file"], + line=int(location["start_line"]) if "start_line" in location else None, + snippet=vulnerability.get("raw_source_code_extract", ""), + ), + ) + findings.append(finding) return findings diff --git a/dojo/tools/gitleaks/parser.py b/dojo/tools/gitleaks/parser.py index f148f2fe029..866841480ca 100644 --- a/dojo/tools/gitleaks/parser.py +++ b/dojo/tools/gitleaks/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class GitleaksParser: @@ -139,6 +142,11 @@ def get_finding_legacy(self, issue, test, dupes): # manage tags finding.unsaved_tags = issue.get("tags", "").split(", ") + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=line), + ) + dupe_key = hashlib.sha256( (issue["offender"] + file_path + str(line)).encode("utf-8"), ).hexdigest() @@ -216,4 +224,8 @@ def get_finding_current(self, issue, test, dupes): ) if tags: finding.unsaved_tags = tags + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=line), + ) dupes[dupe_key] = finding diff --git a/dojo/tools/gosec/parser.py b/dojo/tools/gosec/parser.py index 3e6a737f065..e3711f7aac2 100644 --- a/dojo/tools/gosec/parser.py +++ b/dojo/tools/gosec/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class GosecParser: @@ -93,6 +96,15 @@ def get_findings(self, filename, test): static_finding=True, ) + if settings.V3_FEATURE_LOCATIONS and filename: + find.unsaved_locations.append( + LocationData.code( + file_path=filename, + line=line, + snippet=item["code"], + ), + ) + dupes[dupe_key] = find return list(dupes.values()) diff --git a/dojo/tools/hadolint/parser.py b/dojo/tools/hadolint/parser.py index d781e83b8a0..7dfc047ed1c 100644 --- a/dojo/tools/hadolint/parser.py +++ b/dojo/tools/hadolint/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class HadolintParser: @@ -63,4 +66,9 @@ def get_item(vulnerability, test): finding.description = finding.description.strip() + if settings.V3_FEATURE_LOCATIONS and vulnerability["file"]: + finding.unsaved_locations.append( + LocationData.code(file_path=vulnerability["file"], line=vulnerability["line"]), + ) + return finding diff --git a/dojo/tools/hcl_asoc_sast/parser.py b/dojo/tools/hcl_asoc_sast/parser.py index 948986fbc93..29a07d730fb 100644 --- a/dojo/tools/hcl_asoc_sast/parser.py +++ b/dojo/tools/hcl_asoc_sast/parser.py @@ -1,8 +1,10 @@ from xml.dom import NamespaceErr from defusedxml import ElementTree +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData class HCLASoCSASTParser: @@ -146,6 +148,10 @@ def get_findings(self, file, test): dynamic_finding=False, static_finding=True, ) + if settings.V3_FEATURE_LOCATIONS and location: + prepared_finding.unsaved_locations.append( + LocationData.code(file_path=location, line=line), + ) findings.append(prepared_finding) return findings return findings diff --git a/dojo/tools/horusec/parser.py b/dojo/tools/horusec/parser.py index f1f08865d3c..4838089ebf7 100644 --- a/dojo/tools/horusec/parser.py +++ b/dojo/tools/horusec/parser.py @@ -2,8 +2,10 @@ from datetime import datetime from dateutil.parser import parse +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData from dojo.tools.parser_test import ParserTest @@ -84,4 +86,8 @@ def _get_finding(self, data, date): and data["vulnerabilities"]["line"].isdigit() ): finding.line = int(data["vulnerabilities"]["line"]) + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=finding.line), + ) return finding diff --git a/dojo/tools/huskyci/parser.py b/dojo/tools/huskyci/parser.py index 6c0045b15be..67b2a056e5a 100644 --- a/dojo/tools/huskyci/parser.py +++ b/dojo/tools/huskyci/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class HuskyCIParser: @@ -71,7 +74,7 @@ def get_item(item_node, test): if "securitytool" in item_node: description += "\nSecurity Tool: " + item_node.get("securitytool") - return Finding( + finding = Finding( title=item_node.get("title"), test=test, severity=item_node.get("severity"), @@ -88,3 +91,13 @@ def get_item(item_node, test): dynamic_finding=False, impact="No impact provided", ) + + file_path = item_node.get("file") + if settings.V3_FEATURE_LOCATIONS and file_path: + line = item_node.get("line") + line = int(line) if line is not None and str(line).isdigit() else None + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=line), + ) + + return finding diff --git a/dojo/tools/kics/parser.py b/dojo/tools/kics/parser.py index 878999ee70a..156133b89f0 100644 --- a/dojo/tools/kics/parser.py +++ b/dojo/tools/kics/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class KICSParser: @@ -81,5 +84,9 @@ def get_findings(self, filename, test): nb_occurences=1, references=query_url, ) + if settings.V3_FEATURE_LOCATIONS and file_name: + finding.unsaved_locations.append( + LocationData.code(file_path=file_name, line=line_number), + ) dupes[dupe_key] = finding return list(dupes.values()) diff --git a/dojo/tools/kiuwan/parser.py b/dojo/tools/kiuwan/parser.py index c90c5211ac3..b9d2abc59b8 100644 --- a/dojo/tools/kiuwan/parser.py +++ b/dojo/tools/kiuwan/parser.py @@ -2,7 +2,10 @@ import hashlib import io +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData __author__ = "dr3dd589" @@ -108,6 +111,13 @@ def get_findings(self, filename, test): if cwe.isdigit(): finding.cwe = int(cwe) + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + line = findingdict["line_number"] + line = int(line) if line is not None and str(line).isdigit() else None + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=line, snippet=row["Line text"]), + ) + if finding is not None: if finding.title is None: finding.title = "" diff --git a/dojo/tools/mobsf/api_report_json.py b/dojo/tools/mobsf/api_report_json.py index 6f5bd1c6c75..854bdb5a0dd 100644 --- a/dojo/tools/mobsf/api_report_json.py +++ b/dojo/tools/mobsf/api_report_json.py @@ -1,8 +1,10 @@ from datetime import datetime +from django.conf import settings from html2text import html2text from dojo.models import Finding +from dojo.tools.locations import LocationData class MobSFapireport: @@ -334,6 +336,10 @@ def get_findings(self, data, test): ) if mobsf_finding["file_path"]: finding.file_path = mobsf_finding["file_path"] + if settings.V3_FEATURE_LOCATIONS: + finding.unsaved_locations.append( + LocationData.code(file_path=mobsf_finding["file_path"]), + ) dupe_key = sev + title + description + mobsf_finding["file_path"] else: dupe_key = sev + title + description diff --git a/dojo/tools/mobsf/report.py b/dojo/tools/mobsf/report.py index 3f076e2f8a5..3d79537fee2 100644 --- a/dojo/tools/mobsf/report.py +++ b/dojo/tools/mobsf/report.py @@ -1,7 +1,10 @@ import hashlib import re +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class MobSFjsonreport: @@ -63,6 +66,10 @@ def get_findings(self, data, test): finding.file_path = file_path finding.line = line finding.description = f"{description}\n**Snippet:** `{snippet}`" + if settings.V3_FEATURE_LOCATIONS: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=line, snippet=snippet), + ) dupe_key = hashlib.sha256( (key + str(cwe) + masvs + owasp_mobile + file_path).encode("utf-8"), diff --git a/dojo/tools/noseyparker/parser.py b/dojo/tools/noseyparker/parser.py index 70b28269047..c73a7d0accc 100644 --- a/dojo/tools/noseyparker/parser.py +++ b/dojo/tools/noseyparker/parser.py @@ -2,7 +2,10 @@ import json from datetime import datetime +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class NoseyParkerParser: @@ -101,6 +104,10 @@ def version_0_16_0(self, line, test): dynamic_finding=False, ) + if settings.V3_FEATURE_LOCATIONS and filepath: + finding.unsaved_locations.append( + LocationData.code(file_path=filepath, line=line_num), + ) self.dupes[key] = finding def version_0_22_0(self, line, test): @@ -160,4 +167,8 @@ def version_0_22_0(self, line, test): nb_occurences=1, dynamic_finding=False, ) + if settings.V3_FEATURE_LOCATIONS and filepath: + finding.unsaved_locations.append( + LocationData.code(file_path=filepath, line=line_num), + ) self.dupes[key] = finding diff --git a/dojo/tools/php_security_audit_v2/parser.py b/dojo/tools/php_security_audit_v2/parser.py index 674f35f44c8..37bac82b6ad 100644 --- a/dojo/tools/php_security_audit_v2/parser.py +++ b/dojo/tools/php_security_audit_v2/parser.py @@ -1,7 +1,10 @@ import json import math +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class PhpSecurityAuditV2Parser: @@ -62,6 +65,11 @@ def get_findings(self, filename, test): dynamic_finding=False, ) + if settings.V3_FEATURE_LOCATIONS and filepath: + find.unsaved_locations.append( + LocationData.code(file_path=filepath, line=issue["line"]), + ) + dupes[dupe_key] = find findingdetail = "" diff --git a/dojo/tools/pmd/parser.py b/dojo/tools/pmd/parser.py index 1047a92a950..c171f298f91 100644 --- a/dojo/tools/pmd/parser.py +++ b/dojo/tools/pmd/parser.py @@ -2,7 +2,10 @@ import hashlib import io +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class PmdParser: @@ -54,6 +57,13 @@ def get_findings(self, filename, test): finding.impact = "No impact provided" finding.mitigation = "No mitigation provided" + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + line = row["Line"] + line = int(line) if line is not None and str(line).isdigit() else None + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=line), + ) + key = hashlib.sha256( f"{finding.title}|{finding.description}|{finding.file_path}|{finding.line}".encode(), ).hexdigest() diff --git a/dojo/tools/progpilot/parser.py b/dojo/tools/progpilot/parser.py index 6badb4c0442..21319645af4 100644 --- a/dojo/tools/progpilot/parser.py +++ b/dojo/tools/progpilot/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class ProgpilotParser: @@ -76,5 +79,16 @@ def get_findings(self, filename, test): find.file_path = vuln_file if vuln_cwe is not None: find.cwe = int(vuln_cwe.split("CWE_")[1]) + if settings.V3_FEATURE_LOCATIONS and find.file_path: + find.unsaved_locations.append( + LocationData.code( + file_path=find.file_path, + line=find.line, + source_object=source_name or "", + sink_object=sink_name or "", + source_file_path=source_file or "", + source_line=source_line if isinstance(source_line, int) else None, + ), + ) findings.append(find) return findings diff --git a/dojo/tools/pwn_sast/parser.py b/dojo/tools/pwn_sast/parser.py index 8be69874bbe..6d06e5fc894 100644 --- a/dojo/tools/pwn_sast/parser.py +++ b/dojo/tools/pwn_sast/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class PWNSASTParser: @@ -119,6 +122,11 @@ def get_findings(self, filename, test): finding.fix_available = True else: finding.fix_available = False + if settings.V3_FEATURE_LOCATIONS and offending_file: + line_number = int(line_no) if line_no is not None and str(line_no).isdigit() else None + finding.unsaved_locations.append( + LocationData.code(file_path=offending_file, line=line_number, snippet=contents or ""), + ) findings[unique_finding_key] = finding return list(findings.values()) diff --git a/dojo/tools/rubocop/parser.py b/dojo/tools/rubocop/parser.py index 8cc4ffea3c0..4335a53728e 100644 --- a/dojo/tools/rubocop/parser.py +++ b/dojo/tools/rubocop/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class RubocopParser: @@ -62,5 +65,9 @@ def get_findings(self, scan_file, test): static_finding=True, dynamic_finding=False, ) + if settings.V3_FEATURE_LOCATIONS and path: + finding.unsaved_locations.append( + LocationData.code(file_path=path, line=line), + ) findings.append(finding) return findings diff --git a/dojo/tools/rusty_hog/parser.py b/dojo/tools/rusty_hog/parser.py index 64c3553de84..6dd3f548c2f 100644 --- a/dojo/tools/rusty_hog/parser.py +++ b/dojo/tools/rusty_hog/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData from dojo.tools.parser_test import ParserTest @@ -214,8 +217,18 @@ def __getitem(self, vulnerabilities, scanner): if scanner == "Choctaw Hog": finding.line = int(vulnerability.get("new_line_num")) finding.mitigation = "Please ensure no secret material nor confidential information is kept in clear within git repositories." + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=finding.line), + ) elif scanner == "Duroc Hog": finding.mitigation = "Please ensure no secret material nor confidential information is kept in clear within directories, files, and archives." + if settings.V3_FEATURE_LOCATIONS and file_path: + linenum = vulnerability.get("linenum") + linenum = int(linenum) if linenum is not None and str(linenum).isdigit() else None + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=linenum), + ) elif scanner == "Gottingen Hog": finding.mitigation = "Please ensure no secret material nor confidential information is kept in clear within JIRA Tickets." elif scanner == "Essex Hog": diff --git a/dojo/tools/sarif/parser.py b/dojo/tools/sarif/parser.py index 0ec6800d568..9f411cb85c8 100644 --- a/dojo/tools/sarif/parser.py +++ b/dojo/tools/sarif/parser.py @@ -4,9 +4,11 @@ import textwrap import dateutil.parser +from django.conf import settings from django.utils.translation import gettext as _ from dojo.models import Finding +from dojo.tools.locations import LocationData from dojo.tools.parser_test import ParserTest logger = logging.getLogger(__name__) @@ -216,6 +218,19 @@ def get_items_from_result(self, result, rules, artifacts, run_date): references=get_references(rule), ) + if settings.V3_FEATURE_LOCATIONS and file_path: + end_line = None + if location and "physicalLocation" in location: + end_line = location["physicalLocation"].get("region", {}).get("endLine") + finding.unsaved_locations.append( + LocationData.code( + file_path=file_path, + line=line, + end_line=end_line, + snippet=get_snippet(location) or "", + ), + ) + if "ruleId" in result: finding.vuln_id_from_tool = result["ruleId"] # for now we only support when the id of the rule is a CVE diff --git a/dojo/tools/semgrep/parser.py b/dojo/tools/semgrep/parser.py index 2dabcac1954..0369d2788a1 100644 --- a/dojo/tools/semgrep/parser.py +++ b/dojo/tools/semgrep/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class SemgrepParser: @@ -88,6 +91,11 @@ def get_findings(self, filename, test): nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and item["path"]: + finding.unsaved_locations.append( + LocationData.code(file_path=item["path"], line=item["start"]["line"]), + ) + # fingerprint detection unique_id_from_tool = item.get("extra", {}).get("fingerprint") # treat "requires login" as if the fingerprint is absent diff --git a/dojo/tools/semgrep_pro/parser.py b/dojo/tools/semgrep_pro/parser.py index c8da673d967..5e757cc101c 100644 --- a/dojo/tools/semgrep_pro/parser.py +++ b/dojo/tools/semgrep_pro/parser.py @@ -2,7 +2,10 @@ import json from datetime import datetime +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class SemgrepProParser: @@ -46,6 +49,11 @@ def get_findings(self, filename, test): verified=verified, ) + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=line or None), + ) + # Add CWE if available if "rule" in item and "cwe_names" in item["rule"]: try: diff --git a/dojo/tools/snyk_issue_api/parser.py b/dojo/tools/snyk_issue_api/parser.py index 2f4daa36cf7..5776b1eaa99 100644 --- a/dojo/tools/snyk_issue_api/parser.py +++ b/dojo/tools/snyk_issue_api/parser.py @@ -1,7 +1,10 @@ import json from datetime import datetime +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class SnykIssueApiParser: @@ -262,6 +265,11 @@ def get_finding(self, issue, test): risk_accepted=False, ) + if settings.V3_FEATURE_LOCATIONS and issue_type == "code" and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path, line=line), + ) + # sca only if attributes.get("key"): finding.vuln_id_from_tool = attributes.get("key") diff --git a/dojo/tools/solar_appscreener/parser.py b/dojo/tools/solar_appscreener/parser.py index e0a39b1e4bd..cac608bba1c 100644 --- a/dojo/tools/solar_appscreener/parser.py +++ b/dojo/tools/solar_appscreener/parser.py @@ -1,7 +1,10 @@ import csv import io +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class SolarAppscreenerParser: @@ -55,6 +58,11 @@ def get_findings(self, filename, test): finding.sast_source_line = finding.line + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=finding.file_path, line=finding.line or None), + ) + if finding is not None: if finding.title is None: finding.title = "" diff --git a/dojo/tools/sonarqube/sonarqube_restapi_json.py b/dojo/tools/sonarqube/sonarqube_restapi_json.py index a01dd44037d..3adf5320446 100644 --- a/dojo/tools/sonarqube/sonarqube_restapi_json.py +++ b/dojo/tools/sonarqube/sonarqube_restapi_json.py @@ -1,9 +1,11 @@ import re import dateutil.parser +from django.conf import settings from django.utils import timezone from dojo.models import Finding +from dojo.tools.locations import LocationData class SonarQubeRESTAPIJSON: @@ -132,6 +134,10 @@ def get_json_items(self, json_content, test, mode): line=line, date=date, ) + if settings.V3_FEATURE_LOCATIONS and component: + item.unsaved_locations.append( + LocationData.code(file_path=component, line=line), + ) item.unsaved_tags = ["vulnerability"] vulnids = [] if "Reference: CVE" in message: @@ -203,6 +209,10 @@ def get_json_items(self, json_content, test, mode): line=line, date=date, ) + if settings.V3_FEATURE_LOCATIONS and component: + item.unsaved_locations.append( + LocationData.code(file_path=component, line=line), + ) item.unsaved_tags = ["code_smell"] items.append(item) if json_content.get("hotspots"): @@ -252,6 +262,10 @@ def get_json_items(self, json_content, test, mode): line=line, date=date, ) + if settings.V3_FEATURE_LOCATIONS and component: + item.unsaved_locations.append( + LocationData.code(file_path=component, line=line), + ) item.unsaved_tags = ["hotspot"] items.append(item) return items diff --git a/dojo/tools/sonarqube/soprasteria_helper.py b/dojo/tools/sonarqube/soprasteria_helper.py index 5de3029464c..173f123bf06 100644 --- a/dojo/tools/sonarqube/soprasteria_helper.py +++ b/dojo/tools/sonarqube/soprasteria_helper.py @@ -1,10 +1,12 @@ import logging import re +from django.conf import settings from django.utils.html import strip_tags from lxml import etree from dojo.models import Finding +from dojo.tools.locations import LocationData logger = logging.getLogger(__name__) @@ -89,6 +91,12 @@ def process_result_file_name_aggregated( dynamic_finding=False, nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and vuln_file_path: + find.unsaved_locations.append( + # No line number because we have aggregated different + # vulnerabilities that may have different line numbers + LocationData.code(file_path=vuln_file_path), + ) dupes[aggregateKeys] = find else: # We have already created a finding for this aggregate: updates the @@ -137,4 +145,11 @@ def process_result_detailed( dynamic_finding=False, unique_id_from_tool=vuln_key, ) + if settings.V3_FEATURE_LOCATIONS and vuln_file_path: + find.unsaved_locations.append( + LocationData.code( + file_path=vuln_file_path, + line=int(vuln_line) if str(vuln_line).isdigit() else None, + ), + ) dupes[aggregateKeys] = find diff --git a/dojo/tools/spotbugs/parser.py b/dojo/tools/spotbugs/parser.py index 83764ec0455..c5bb3e2b9c9 100644 --- a/dojo/tools/spotbugs/parser.py +++ b/dojo/tools/spotbugs/parser.py @@ -2,8 +2,10 @@ import html2text from defusedxml import ElementTree +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData class SpotbugsParser: @@ -116,6 +118,15 @@ def get_findings(self, filename, test): finding.line = int(source_extract.get("start")) finding.sast_source_line = int(source_extract.get("start")) + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code( + file_path=finding.file_path, + line=finding.line, + source_object=source_extract.get("classname") or "", + ), + ) + if bug.get("type") in mitigation_patterns: finding.mitigation = mitigation_patterns[bug.get("type")] finding.references = reference_patterns[bug.get("type")] diff --git a/dojo/tools/talisman/parser.py b/dojo/tools/talisman/parser.py index 00f0a3fad4e..1760c901501 100644 --- a/dojo/tools/talisman/parser.py +++ b/dojo/tools/talisman/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class TalismanParser: @@ -60,6 +63,11 @@ def get_findings(self, filename, test): severity=severity, ) + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path), + ) + key = hashlib.md5( ( title diff --git a/dojo/tools/terrascan/parser.py b/dojo/tools/terrascan/parser.py index e9f7e762002..4fb81789737 100644 --- a/dojo/tools/terrascan/parser.py +++ b/dojo/tools/terrascan/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class TerrascanParser: @@ -69,5 +72,9 @@ def get_findings(self, filename, test): vuln_id_from_tool=rule_id, nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and file: + finding.unsaved_locations.append( + LocationData.code(file_path=file, line=line), + ) dupes[dupe_key] = finding return list(dupes.values()) diff --git a/dojo/tools/tfsec/parser.py b/dojo/tools/tfsec/parser.py index b86861b6edb..90891d2a7b5 100644 --- a/dojo/tools/tfsec/parser.py +++ b/dojo/tools/tfsec/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class TFSecParser: @@ -80,5 +83,9 @@ def get_findings(self, filename, test): vuln_id_from_tool=rule_id, nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and file: + finding.unsaved_locations.append( + LocationData.code(file_path=file, line=start_line, end_line=end_line), + ) dupes[dupe_key] = finding return list(dupes.values()) diff --git a/dojo/tools/trivy/parser.py b/dojo/tools/trivy/parser.py index 25746cd4089..8e772758182 100644 --- a/dojo/tools/trivy/parser.py +++ b/dojo/tools/trivy/parser.py @@ -406,6 +406,10 @@ def get_result_items(self, test, results, service_name=None, artifact_name=""): finding.unsaved_vulnerability_ids = [] finding.unsaved_vulnerability_ids.append(misc_avdid) finding.unsaved_tags = [tag for tag in (target_type, target_class) if tag] + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code(file_path=file_path), + ) items.append(finding) secrets = target_data.get("Secrets", []) @@ -437,6 +441,10 @@ def get_result_items(self, test, results, service_name=None, artifact_name=""): service=service_name, ) finding.unsaved_tags = [tag for tag in (target_class,) if tag] + if settings.V3_FEATURE_LOCATIONS and target_target: + finding.unsaved_locations.append( + LocationData.code(file_path=target_target, line=secret_start_line), + ) items.append(finding) licenses = target_data.get("Licenses", []) diff --git a/dojo/tools/trivy_operator/secrets_handler.py b/dojo/tools/trivy_operator/secrets_handler.py index e0a1b1996f5..97f4ef51080 100644 --- a/dojo/tools/trivy_operator/secrets_handler.py +++ b/dojo/tools/trivy_operator/secrets_handler.py @@ -1,4 +1,7 @@ +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData TRIVY_SEVERITIES = { "CRITICAL": "Critical", @@ -55,6 +58,10 @@ def handle_secrets(self, labels, secrets, test): service=service, fix_available=True, ) + if settings.V3_FEATURE_LOCATIONS and secret_target: + finding.unsaved_locations.append( + LocationData.code(file_path=secret_target), + ) if resource_namespace: finding.unsaved_tags = [resource_namespace] findings.append(finding) diff --git a/dojo/tools/trufflehog/parser.py b/dojo/tools/trufflehog/parser.py index 0948c50cadc..de41da06a7a 100644 --- a/dojo/tools/trufflehog/parser.py +++ b/dojo/tools/trufflehog/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class TruffleHogParser: @@ -95,6 +98,12 @@ def get_findings_v2(self, data, test): nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and file: + finding.unsaved_locations.append( + # line is None: the scalar 0 above is a fake value for deduplication only + LocationData.code(file_path=file), + ) + dupes[dupe_key] = finding return list(dupes.values()) @@ -195,6 +204,10 @@ def get_findings_v3(self, data, test): verified=verified, nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and file: + finding.unsaved_locations.append( + LocationData.code(file_path=file, line=line_number or None), + ) dupes[dupe_key] = finding return list(dupes.values()) diff --git a/dojo/tools/trufflehog3/parser.py b/dojo/tools/trufflehog3/parser.py index 9602fa853b9..e5be0b75c58 100644 --- a/dojo/tools/trufflehog3/parser.py +++ b/dojo/tools/trufflehog3/parser.py @@ -1,7 +1,10 @@ import hashlib import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class TruffleHog3Parser: @@ -92,6 +95,11 @@ def get_finding_legacy(self, json_data, test, dupes): static_finding=True, nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and file: + finding.unsaved_locations.append( + # line is None: the scalar 0 above is a fake value for deduplication only + LocationData.code(file_path=file), + ) dupes[dupe_key] = finding def get_finding_current(self, json_data, test, dupes): @@ -164,4 +172,8 @@ def get_finding_current(self, json_data, test, dupes): static_finding=True, nb_occurences=1, ) + if settings.V3_FEATURE_LOCATIONS and file: + finding.unsaved_locations.append( + LocationData.code(file_path=file, line=line or None), + ) dupes[dupe_key] = finding diff --git a/dojo/tools/veracode/json_parser.py b/dojo/tools/veracode/json_parser.py index e33445d1399..f2585beecc9 100644 --- a/dojo/tools/veracode/json_parser.py +++ b/dojo/tools/veracode/json_parser.py @@ -175,6 +175,16 @@ def add_static_details(self, finding, finding_details, backup_title=None) -> Fin if module := finding_details.get("module"): finding.description += f"**Module**: {module}\n" + if settings.V3_FEATURE_LOCATIONS and (file_path := finding_details.get("file_path")): + function_object = finding_details.get("procedure") + finding.unsaved_locations.append( + LocationData.code( + file_path=file_path, + line=finding_details.get("file_line_number") or None, + source_object=function_object if isinstance(function_object, str) else "", + ), + ) + return finding def add_dynamic_details(self, finding, finding_details, backup_title=None) -> Finding: diff --git a/dojo/tools/veracode/xml_parser.py b/dojo/tools/veracode/xml_parser.py index bf7508c543f..96802e520b5 100644 --- a/dojo/tools/veracode/xml_parser.py +++ b/dojo/tools/veracode/xml_parser.py @@ -247,6 +247,15 @@ def __xml_static_flaw_to_finding( if isinstance(sast_source_obj, str): finding.sast_source_object = sast_source_obj or None + if settings.V3_FEATURE_LOCATIONS and finding.file_path: + finding.unsaved_locations.append( + LocationData.code( + file_path=finding.file_path, + line=finding.line, + source_object=sast_source_obj if isinstance(sast_source_obj, str) else "", + ), + ) + finding.unsaved_tags = ["sast"] return finding diff --git a/dojo/tools/whispers/parser.py b/dojo/tools/whispers/parser.py index 8d21ba708a4..03e09d1e2aa 100644 --- a/dojo/tools/whispers/parser.py +++ b/dojo/tools/whispers/parser.py @@ -1,6 +1,9 @@ import json +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData class WhispersParser: @@ -92,28 +95,31 @@ def get_findings(self, file, test): f'in {vuln.get("file")}:{vuln.get("line")}' ) description = f'{summary} `{self._mask(vuln.get("value"))}`' - findings.append( - Finding( - title=summary, - description=description, - mitigation=( - "Replace hardcoded secret with a placeholder (ie: ENV-VAR). " - "Invalidate the leaked secret and generate a new one. " - "Supply the new secret through a placeholder to avoid disclosing " - "sensitive information in code." - ), - references="https://cwe.mitre.org/data/definitions/798.html", - cwe=798, - severity=self.SEVERITY_MAP.get( - vuln.get("severity"), "Info", - ), - file_path=vuln.get("file"), - line=int(vuln.get("line")), - vuln_id_from_tool=vuln.get("message"), - static_finding=True, - dynamic_finding=False, - test=test, + finding = Finding( + title=summary, + description=description, + mitigation=( + "Replace hardcoded secret with a placeholder (ie: ENV-VAR). " + "Invalidate the leaked secret and generate a new one. " + "Supply the new secret through a placeholder to avoid disclosing " + "sensitive information in code." + ), + references="https://cwe.mitre.org/data/definitions/798.html", + cwe=798, + severity=self.SEVERITY_MAP.get( + vuln.get("severity"), "Info", ), + file_path=vuln.get("file"), + line=int(vuln.get("line")), + vuln_id_from_tool=vuln.get("message"), + static_finding=True, + dynamic_finding=False, + test=test, ) + if settings.V3_FEATURE_LOCATIONS and vuln.get("file"): + finding.unsaved_locations.append( + LocationData.code(file_path=vuln.get("file"), line=int(vuln.get("line"))), + ) + findings.append(finding) return findings diff --git a/dojo/tools/wizcli_common_parsers/parsers.py b/dojo/tools/wizcli_common_parsers/parsers.py index 87961fa2f25..3eb6949d2eb 100644 --- a/dojo/tools/wizcli_common_parsers/parsers.py +++ b/dojo/tools/wizcli_common_parsers/parsers.py @@ -113,6 +113,13 @@ def parse_secrets(secrets, test): mitigation=None, test=test, ) + if settings.V3_FEATURE_LOCATIONS and file_name: + finding.unsaved_locations.append( + LocationData.code( + file_path=file_name, + line=line_number if isinstance(line_number, int) else None, + ), + ) findings.append(finding) return findings @@ -160,6 +167,13 @@ def parse_rule_matches(rule_matches, test): mitigation=None, test=test, ) + if settings.V3_FEATURE_LOCATIONS and file_name: + finding.unsaved_locations.append( + LocationData.code( + file_path=file_name, + line=line_number if isinstance(line_number, int) else None, + ), + ) findings.append(finding) return findings diff --git a/dojo/tools/xanitizer/parser.py b/dojo/tools/xanitizer/parser.py index 31adb7f42b5..83ddeea0f60 100644 --- a/dojo/tools/xanitizer/parser.py +++ b/dojo/tools/xanitizer/parser.py @@ -3,8 +3,10 @@ import re from defusedxml import ElementTree +from django.conf import settings from dojo.models import Finding +from dojo.tools.locations import LocationData class XanitizerParser: @@ -74,6 +76,14 @@ def get_findings_internal(self, root, test): if vulnerability_id: dojofinding.unsaved_vulnerability_ids = [vulnerability_id] + if settings.V3_FEATURE_LOCATIONS and dojofinding.file_path: + dojofinding.unsaved_locations.append( + LocationData.code( + file_path=dojofinding.file_path, + line=int(line) if line else None, + ), + ) + items.append(dojofinding) return items diff --git a/dojo/tools/xygeni/sast.py b/dojo/tools/xygeni/sast.py index 2b2f9bd9497..5ad132f7f5a 100644 --- a/dojo/tools/xygeni/sast.py +++ b/dojo/tools/xygeni/sast.py @@ -1,6 +1,9 @@ """Parse Xygeni SAST reports into DefectDojo Findings.""" +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData from dojo.tools.xygeni._common import map_severity, parse_cwe @@ -45,6 +48,18 @@ def _build_finding(vuln, test): ) _apply_code_flow_fields(finding, vuln.get("codeFlows") or []) + + if settings.V3_FEATURE_LOCATIONS and file_path: + finding.unsaved_locations.append( + LocationData.code( + file_path=file_path, + line=line, + snippet=code or "", + source_object=finding.sast_source_object or "", + sink_object=finding.sast_sink_object or "", + ), + ) + return finding diff --git a/dojo/tools/xygeni/secrets.py b/dojo/tools/xygeni/secrets.py index 9432a48486b..f1fce5680b4 100644 --- a/dojo/tools/xygeni/secrets.py +++ b/dojo/tools/xygeni/secrets.py @@ -8,7 +8,10 @@ from pathlib import PurePosixPath +from django.conf import settings + from dojo.models import Finding +from dojo.tools.locations import LocationData from dojo.tools.xygeni._common import map_severity, parse_cwe DEFAULT_CWE = 798 # CWE-798: Use of Hard-coded Credentials @@ -58,7 +61,7 @@ def _build_finding(occurrences, test): cwe = parse_cwe(tags=secret.get("tags")) or DEFAULT_CWE - return Finding( + finding = Finding( test=test, title=f"{secret_type} secret detected in {filename}", description="\n\n".join(description_parts) if description_parts else "", @@ -78,3 +81,13 @@ def _build_finding(occurrences, test): unique_id_from_tool=secret.get("uniqueHash"), vuln_id_from_tool=secret.get("detector"), ) + + if settings.V3_FEATURE_LOCATIONS and filepath: + # One location per line the secret is leaked on; the aggregated Finding + # above keeps a single scalar line while the locations carry them all. + for begin_line in lines or [location.get("beginLine")]: + finding.unsaved_locations.append( + LocationData.code(file_path=filepath, line=begin_line), + ) + + return finding diff --git a/unittests/test_code_location_emission.py b/unittests/test_code_location_emission.py new file mode 100644 index 00000000000..54bbe61f57d --- /dev/null +++ b/unittests/test_code_location_emission.py @@ -0,0 +1,123 @@ +""" +Parsers emit ``LocationData.code`` for source-code findings under V3. + +Mirrors how dependency emission (PR #14395) is exercised: parse a real report +fixture, then assert the findings carry code ``LocationData`` in +``unsaved_locations`` that matches the finding's own coordinates. A gate-off +class proves the emission is completely inert without V3_FEATURE_LOCATIONS. +""" + +from django.test import override_settings + +from dojo.models import Test +from dojo.tools.bandit.parser import BanditParser +from dojo.tools.brakeman.parser import BrakemanParser +from dojo.tools.gitleaks.parser import GitleaksParser +from dojo.tools.gosec.parser import GosecParser +from dojo.tools.huskyci.parser import HuskyCIParser +from dojo.tools.sarif.parser import SarifParser +from dojo.tools.semgrep.parser import SemgrepParser +from dojo.tools.tfsec.parser import TFSecParser +from dojo.tools.whispers.parser import WhispersParser +from unittests.dojo_test_case import DojoTestCase, get_unit_tests_scans_path + +# (parser class, scans dir, fixture) — fixtures known to produce static findings +PARSER_FIXTURES = [ + (BanditParser, "bandit", "many_vulns.json"), + (BrakemanParser, "brakeman", "many_findings.json"), + (GitleaksParser, "gitleaks", "gitleaks8_many.json"), + (GosecParser, "gosec", "many_vulns.json"), + (HuskyCIParser, "huskyci", "huskyci_report_many_finding_one_tool.json"), + (SarifParser, "sarif", "appendix_k.sarif"), + (SemgrepParser, "semgrep", "close_old_findings_report_line31.json"), + (TFSecParser, "tfsec", "many_findings_current.json"), + (WhispersParser, "whispers", "whispers_many_vul.json"), +] + + +def _parse(parser_class, scans_dir, fixture): + with (get_unit_tests_scans_path(scans_dir) / fixture).open(encoding="utf-8") as testfile: + return parser_class().get_findings(testfile, Test()) + + +def _code_locations(finding): + return [ + location + for location in getattr(finding, "unsaved_locations", []) + if getattr(location, "type", None) == "code" + ] + + +@override_settings(V3_FEATURE_LOCATIONS=True) +class TestCodeLocationEmission(DojoTestCase): + + """Every converted parser emits code locations coherent with its scalars.""" + + def test_parsers_emit_code_locations(self): + for parser_class, scans_dir, fixture in PARSER_FIXTURES: + with self.subTest(parser=parser_class.__name__): + findings = _parse(parser_class, scans_dir, fixture) + emitting = [f for f in findings if _code_locations(f)] + self.assertTrue( + emitting, + f"{parser_class.__name__} produced no code LocationData for {fixture}", + ) + for finding in emitting: + for location in _code_locations(finding): + self.assertTrue(location.data["file_path"], "code location must carry a file path") + line = location.data.get("line") + self.assertTrue( + line is None or isinstance(line, int), + f"line must be int or None, got {line!r}", + ) + + def test_emitted_identity_matches_finding_scalars(self): + """ + The finding's own file_path/line and the emitted location agree — + the scalars stay the single source the location machinery derives from. + """ + findings = _parse(SemgrepParser, "semgrep", "close_old_findings_report_line31.json") + self.assertEqual(1, len(findings)) + finding = findings[0] + locations = _code_locations(finding) + self.assertEqual(1, len(locations)) + self.assertEqual(finding.file_path, locations[0].data["file_path"]) + self.assertEqual(31, locations[0].data["line"]) + self.assertEqual(finding.line, locations[0].data["line"]) + + def test_snippet_context_rides_along_when_extracted(self): + """Parsers that already extract a code snippet pass it as context.""" + findings = _parse(BanditParser, "bandit", "many_vulns.json") + snippets = [ + location.data.get("snippet") + for finding in findings + for location in _code_locations(finding) + ] + self.assertTrue(any(snippets), "bandit extracts code blocks; at least one snippet expected") + + def test_sarif_emits_per_location_with_region_context(self): + """ + The SARIF parser explodes multi-location results into one finding per + location; each finding carries its own code location. + """ + findings = _parse(SarifParser, "sarif", "appendix_k.sarif") + emitting = [f for f in findings if f.file_path] + self.assertTrue(emitting) + for finding in emitting: + locations = _code_locations(finding) + self.assertEqual(1, len(locations)) + self.assertEqual(finding.file_path, locations[0].data["file_path"]) + + +@override_settings(V3_FEATURE_LOCATIONS=False) +class TestCodeLocationEmissionGateOff(DojoTestCase): + + """Without V3 the emission is inert: no code locations, scalars untouched.""" + + def test_no_code_locations_without_v3(self): + for parser_class, scans_dir, fixture in PARSER_FIXTURES: + with self.subTest(parser=parser_class.__name__): + findings = _parse(parser_class, scans_dir, fixture) + self.assertTrue(findings, f"fixture must still parse: {fixture}") + for finding in findings: + self.assertEqual([], _code_locations(finding)) From dc08c8023d7b41788d9c28ff346eec48cd140428 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:32:11 -0600 Subject: [PATCH 4/7] fix(locations): LocationData.code robust to unhashable context; progpilot scalars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI (the V3-on rest-framework matrix variant) caught two coupled defects the local V3-off run could not, since the emission only fires under V3: - LocationData.code filtered unset context with `value not in {"", None}` — a SET membership test that hashes `value`, so any unhashable value crashed with "cannot use 'list' as a set element". Switched to tuple membership (`in ("", None)`), which compares by equality and never hashes — robust for every call site, present and future. - progpilot reports taint-source fields as single-element arrays (source_name=["$sql"], source_line=[610]); the sweep passed those lists straight into the context kwargs. Now collapsed to scalars via a _first() helper, with str()/int guards. Audited every other converted parser's context kwargs: all pass .text/int/ string scalars (checkmarx pathnode elements, sarif get_snippet, etc.) — progpilot was the only list-valued source. Verified against the full unittests.tools suite run with V3_FEATURE_LOCATIONS=True. Co-Authored-By: Claude Fable 5 --- dojo/tools/locations.py | 4 +++- dojo/tools/progpilot/parser.py | 23 +++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/dojo/tools/locations.py b/dojo/tools/locations.py index 0351c34dc37..57db376cb3f 100644 --- a/dojo/tools/locations.py +++ b/dojo/tools/locations.py @@ -68,7 +68,9 @@ def code( "source_file_path": source_file_path, "source_line": source_line, } - data.update({key: value for key, value in context.items() if value not in {"", None}}) + # Tuple (not set) membership: uses == rather than hashing, so an unset + # check never crashes on an unhashable value a parser might pass. + data.update({key: value for key, value in context.items() if value not in ("", None)}) return cls(type="code", data=data) @classmethod diff --git a/dojo/tools/progpilot/parser.py b/dojo/tools/progpilot/parser.py index 21319645af4..91485770c12 100644 --- a/dojo/tools/progpilot/parser.py +++ b/dojo/tools/progpilot/parser.py @@ -6,6 +6,18 @@ from dojo.tools.locations import LocationData +def _first(value): + """ + Collapse progpilot taint-source fields to a scalar. + + progpilot reports them as single-element arrays (e.g. source_name=["$sql"], + source_line=[610]); the location context expects the scalar value. + """ + if isinstance(value, list): + return value[0] if value else None + return value + + class ProgpilotParser: def get_scan_types(self): return ["Progpilot Scan"] @@ -80,14 +92,17 @@ def get_findings(self, filename, test): if vuln_cwe is not None: find.cwe = int(vuln_cwe.split("CWE_")[1]) if settings.V3_FEATURE_LOCATIONS and find.file_path: + source_name_scalar = _first(source_name) + source_file_scalar = _first(source_file) + source_line_scalar = _first(source_line) find.unsaved_locations.append( LocationData.code( file_path=find.file_path, line=find.line, - source_object=source_name or "", - sink_object=sink_name or "", - source_file_path=source_file or "", - source_line=source_line if isinstance(source_line, int) else None, + source_object=str(source_name_scalar) if source_name_scalar else "", + sink_object=str(sink_name) if sink_name else "", + source_file_path=str(source_file_scalar) if source_file_scalar else "", + source_line=source_line_scalar if isinstance(source_line_scalar, int) else None, ), ) findings.append(find) From 4b082b4340e0d22a60089543dd2e97a0ccefa382 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:35:05 -0600 Subject: [PATCH 5/7] style(locations): lint-clean the unset-context filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the crash-prone set-membership filter with a truthiness dict comprehension: never hashes the value (crash-robust for any parser input), and satisfies the repo's ruff rules (no manual loop, no != "" comparison). Drops None/""/0/empty — all "no data" for the code location's context fields. Same fix, lint-clean form. Co-Authored-By: Claude Fable 5 --- dojo/tools/locations.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dojo/tools/locations.py b/dojo/tools/locations.py index 57db376cb3f..605b6dfba5c 100644 --- a/dojo/tools/locations.py +++ b/dojo/tools/locations.py @@ -68,9 +68,10 @@ def code( "source_file_path": source_file_path, "source_line": source_line, } - # Tuple (not set) membership: uses == rather than hashing, so an unset - # check never crashes on an unhashable value a parser might pass. - data.update({key: value for key, value in context.items() if value not in ("", None)}) + # Truthiness filter (not set membership): never hashes `value`, so the + # unset check can't crash on an unhashable value a parser might pass. + # Drops None/""/0/empty — all "no data" for these context fields. + data.update({key: value for key, value in context.items() if value}) return cls(type="code", data=data) @classmethod From 772a1406caeea1f0031cc2a49c263e2956a6c1e3 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:50:54 -0600 Subject: [PATCH 6/7] test(locations): pin the get_locations() unsaved-path URL filter with a raw AbstractLocation instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing symmetry tests feed LocationData, which the URL-only LocationManager already drops during cleaning — they pass even without the filter. Only a raw AbstractLocation instance (a plugin's Dependency/CodeLocation model object, passed through untyped by make_abstract_locations) reaches the hash comprehension, so this case is the one that actually fails without the fix: without the type filter its value leaks into the endpoints ingredient and drifts the pre-save hash. Co-Authored-By: Claude Fable 5 --- unittests/test_location_data.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/unittests/test_location_data.py b/unittests/test_location_data.py index 85ad900b252..9fa100b326a 100644 --- a/unittests/test_location_data.py +++ b/unittests/test_location_data.py @@ -6,7 +6,6 @@ class TestLocationDataCodeFactory(DojoTestCase): - def test_code_factory_identity_keys(self): data = LocationData.code(file_path="src/db/queries.py", line=42) self.assertEqual("code", data.type) @@ -42,7 +41,6 @@ def test_code_factory_line_none_is_allowed(self): @skip_unless_v3 class TestGetLocationsHashSymmetry(DojoTestCase): - @classmethod def setUpTestData(cls): product_type = Product_Type.objects.create(name="hash symmetry type") @@ -78,6 +76,32 @@ def test_unsaved_url_locations_still_hash(self): ] self.assertIn("example.com", finding.get_locations()) + def test_raw_abstract_location_instances_are_filtered_by_type(self): + """ + Pin the actual fix: LocationData non-URL entries are already dropped + by the URL-only LocationManager during cleaning, so only a raw + AbstractLocation instance (e.g. a plugin's Dependency/CodeLocation + model object, passed through untyped by make_abstract_locations) + reaches the hash comprehension — without the type filter it leaks + into the endpoints ingredient and drifts the pre-save hash. + """ + from unittest.mock import MagicMock + + from dojo.url.models import URL + + dependency_location = MagicMock(spec=URL) + dependency_location.get_location_type.return_value = "dependency" + dependency_location.get_location_value.return_value = "pkg:npm/lodash@4.17.21" + + finding = self._finding() + finding.unsaved_locations = [ + LocationData.url(url="https://example.com/login", host="example.com", protocol="https", path="login"), + dependency_location, + ] + value = finding.get_locations() + self.assertIn("example.com", value) + self.assertNotIn("lodash", value) + def test_mixed_unsaved_locations_hash_only_urls(self): finding = self._finding() finding.unsaved_locations = [ From 88b31079e47f1c008104d0fe76c0aaa09ed76677 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 16 Jul 2026 09:41:12 -0600 Subject: [PATCH 7/7] chore(lint): hoist test imports to module top for CI ruff (PLC0415) Co-Authored-By: Claude Fable 5 --- unittests/test_location_data.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/unittests/test_location_data.py b/unittests/test_location_data.py index 9fa100b326a..79dabf9d13b 100644 --- a/unittests/test_location_data.py +++ b/unittests/test_location_data.py @@ -1,7 +1,10 @@ +from unittest.mock import MagicMock + from django.utils import timezone from dojo.models import Engagement, Finding, Product, Product_Type, Test, Test_Type from dojo.tools.locations import LocationData +from dojo.url.models import URL from unittests.dojo_test_case import DojoTestCase, skip_unless_v3 @@ -85,10 +88,6 @@ def test_raw_abstract_location_instances_are_filtered_by_type(self): reaches the hash comprehension — without the type filter it leaks into the endpoints ingredient and drifts the pre-save hash. """ - from unittest.mock import MagicMock - - from dojo.url.models import URL - dependency_location = MagicMock(spec=URL) dependency_location.get_location_type.return_value = "dependency" dependency_location.get_location_value.return_value = "pkg:npm/lodash@4.17.21"