diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 51fcd366e32..c07a22a86a8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -68,7 +68,6 @@ jobs: "tests/tool_type_test.py", "tests/user_profile_test.py", "tests/user_test.py", - # "tests/import_scanner_test.py", # "tests/zap.py", ] os: [debian] diff --git a/.github/workflows/validate_docs_build.yml b/.github/workflows/validate_docs_build.yml index 2b288624eb7..616cbc5910e 100644 --- a/.github/workflows/validate_docs_build.yml +++ b/.github/workflows/validate_docs_build.yml @@ -44,6 +44,36 @@ jobs: - name: Check internal links uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2 with: - args: --offline --no-progress --root-dir docs/public './docs/public/**/*.html' + # --remap makes lychee resolve absolute docs.defectdojo.com URLs against + # the freshly built site, so absURL-rendered links (e.g. nav menu items) + # are verified as 404s instead of being skipped as remote URLs. + args: >- + --offline --no-progress + --root-dir ${{ github.workspace }}/docs/public + --remap "https://docs.defectdojo.com file://${{ github.workspace }}/docs/public" + './docs/public/**/*.html' fail: true + - name: Check in-app docs help links + # Find every file under dojo/ that hardcodes a docs.defectdojo.com URL + # (templates, settings, etc.) and check those links against the freshly + # built site. --remap turns the absolute docs URLs into local file lookups; + # --exclude '%7[BD]' drops URL-encoded Django template tags ({% ... %}) + # so only real external docs URLs are checked. lychee is on $PATH from + # the previous lychee-action step. + run: | + set -euo pipefail + mapfile -t files < <(grep -rl 'docs\.defectdojo\.com' dojo/ \ + --include='*.html' --include='*.py' --include='*.tpl') + if [ "${#files[@]}" -eq 0 ]; then + echo "No files reference docs.defectdojo.com — pattern is stale." >&2 + exit 1 + fi + printf 'Checking in-app docs links in:\n' + printf ' %s\n' "${files[@]}" + lychee --offline --no-progress \ + --root-dir "${GITHUB_WORKSPACE}/docs/public" \ + --remap "https://docs.defectdojo.com file://${GITHUB_WORKSPACE}/docs/public" \ + --exclude '%7[BD]' \ + "${files[@]}" + diff --git a/CLAUDE.md b/AGENTS.md similarity index 100% rename from CLAUDE.md rename to AGENTS.md diff --git a/Dockerfile.django-debian b/Dockerfile.django-debian index 8a494c72ae3..b08f15f7acf 100644 --- a/Dockerfile.django-debian +++ b/Dockerfile.django-debian @@ -67,7 +67,8 @@ RUN \ --no-cache-dir \ --no-index \ --find-links=/tmp/wheels \ - -r ./requirements.txt + -r ./requirements.txt && \ + apt-get -y purge --auto-remove git COPY \ docker/entrypoint-celery-beat.sh \ diff --git a/docker/entrypoint-integration-tests.sh b/docker/entrypoint-integration-tests.sh index 444c666e3e9..92790c35559 100755 --- a/docker/entrypoint-integration-tests.sh +++ b/docker/entrypoint-integration-tests.sh @@ -274,13 +274,6 @@ else # The below tests are commented out because they are still an unstable work in progress ## Once Ready they can be uncommented. - # echo "Import Scanner integration test" - # if python3 tests/import_scanner_test.py ; then - # echo "Success: Import Scanner integration tests passed" - # else - # echo "Error: Import Scanner integration test failed"; exit 1 - # fi - # echo "Zap integration test" # if python3 tests/zap.py ; then # echo "Success: zap integration tests passed" diff --git a/docs/assets/images/assets_ss1.png b/docs/assets/images/assets_ss1.png new file mode 100644 index 00000000000..b791e78c5a9 Binary files /dev/null and b/docs/assets/images/assets_ss1.png differ diff --git a/docs/assets/images/assets_ss2.png b/docs/assets/images/assets_ss2.png new file mode 100644 index 00000000000..34a2bda5550 Binary files /dev/null and b/docs/assets/images/assets_ss2.png differ diff --git a/docs/assets/images/engagements_ss99.png b/docs/assets/images/engagements_ss99.png new file mode 100644 index 00000000000..6deddded007 Binary files /dev/null and b/docs/assets/images/engagements_ss99.png differ diff --git a/docs/assets/images/product_ss1.png b/docs/assets/images/product_ss1.png new file mode 100644 index 00000000000..956eb53493b Binary files /dev/null and b/docs/assets/images/product_ss1.png differ diff --git a/docs/assets/images/product_ss2.png b/docs/assets/images/product_ss2.png new file mode 100644 index 00000000000..cd37c0578d7 Binary files /dev/null and b/docs/assets/images/product_ss2.png differ diff --git a/docs/assets/images/product_ss3.png b/docs/assets/images/product_ss3.png new file mode 100644 index 00000000000..f45fa3cdb8c Binary files /dev/null and b/docs/assets/images/product_ss3.png differ diff --git a/docs/assets/images/product_ss4.png b/docs/assets/images/product_ss4.png new file mode 100644 index 00000000000..40a7c227bcd Binary files /dev/null and b/docs/assets/images/product_ss4.png differ diff --git a/docs/assets/images/product_ss5.png b/docs/assets/images/product_ss5.png new file mode 100644 index 00000000000..4ea580146c0 Binary files /dev/null and b/docs/assets/images/product_ss5.png differ diff --git a/docs/assets/images/product_ss6.png b/docs/assets/images/product_ss6.png new file mode 100644 index 00000000000..3f372106104 Binary files /dev/null and b/docs/assets/images/product_ss6.png differ diff --git a/docs/assets/images/product_ss7.png b/docs/assets/images/product_ss7.png new file mode 100644 index 00000000000..d7da19c5cb5 Binary files /dev/null and b/docs/assets/images/product_ss7.png differ diff --git a/docs/assets/images/product_ss8.png b/docs/assets/images/product_ss8.png new file mode 100644 index 00000000000..4840d71fd63 Binary files /dev/null and b/docs/assets/images/product_ss8.png differ diff --git a/docs/config/_default/menus/menus.en.toml b/docs/config/_default/menus/menus.en.toml index 1b65ca0786f..f82d38e9abc 100644 --- a/docs/config/_default/menus/menus.en.toml +++ b/docs/config/_default/menus/menus.en.toml @@ -15,7 +15,7 @@ [[main]] name = "Model Your Assets" - url = "/asset_modelling/hierarchy/pro__assets_organizations/" + url = "/asset_modelling/pro_hierarchy/assets_organizations/" weight = 13 [[main]] diff --git a/docs/content/admin/sso/PRO__saml.md b/docs/content/admin/sso/PRO__saml.md index e4a7a798224..d8292a94a6e 100644 --- a/docs/content/admin/sso/PRO__saml.md +++ b/docs/content/admin/sso/PRO__saml.md @@ -41,6 +41,19 @@ DefectDojo can use the SAML assertion to automatically assign users to [User Gro The **Group Name Attribute** field specifies which attribute in the SAML assertion contains the user's group memberships. When a user logs in, DefectDojo reads this attribute and assigns the user to any matching groups. To limit which groups from the assertion are considered, use the **Group Limiter Regex Expression** field. +The value must match the attribute name your Identity Provider emits in the assertion exactly, including any namespace prefix. A short, friendly name like `groups` will only work if your IdP is configured to emit that literal attribute name — many IdPs use a fully qualified claim URI instead. + +### Group Name Attribute by Identity Provider + +| Identity Provider | Default attribute name to use | +|---|---| +| **Entra ID / Azure AD** | `http://schemas.microsoft.com/ws/2008/06/identity/claims/groups` | +| **Okta** | `groups` (the attribute name you configured on the SAML app's Group Attribute Statement) | +| **Keycloak** | `groups` (or whatever you set as the "SAML Attribute Name" on the Group List mapper) | +| **PingFederate / generic** | Whatever value you configured on the IdP side — check your IdP's assertion before assuming `groups` | + +If group mapping appears to do nothing — users log in successfully but no groups are created or assigned — the most common cause is a mismatch between this field and the attribute name your IdP is actually sending. Enable **Enable SAML Debugging** (see [Additional Options](#additional-options)) to see the raw attributes coming back from the IdP. + If no group with a matching name exists, DefectDojo will automatically create one. Note that a newly created group will not have any permissions configured — those can be set later by a Superuser. To activate group mapping, check the **Enable Group Mapping** checkbox at the bottom of the form. diff --git a/docs/content/admin/user_management/OS__creating_new_users.md b/docs/content/admin/user_management/OS__creating_new_users.md new file mode 100644 index 00000000000..2a237a1c05c --- /dev/null +++ b/docs/content/admin/user_management/OS__creating_new_users.md @@ -0,0 +1,41 @@ +--- +title: "Creating a new user" +description: "How to onboard a new user onto your DefectDojo instance" +audience: opensource +weight: 1 +--- + +This page describes the recommended onboarding workflow for adding new users to a DefectDojo instance. DefectDojo users can be used as both standard, human-operated accounts and as service accounts. + +The admin who creates the account is responsible for delivering the initial credentials (username and password) to the new user. + +## Recommended workflow + +1. **Create the user account** in DefectDojo (Superuser only): + * Navigate to **👤 Users → Users** to open the All Users table. + * Click the 🛠️ (crossed wrench and screwdriver) icon. + * Enter the new user's name and email address. + * Set a temporary password. + * Submit the form. + +2. **Assign permissions** as appropriate — Product/Product Type membership, Configuration Permissions, Global Role, or Superuser status. See [Set a User's permissions](../set_user_permissions/) for details. A new user with no assignments will not be able to see any Products or Findings. + +3. **Send the credentials to the new user out-of-band** (over email, your team's chat tool, or however you normally share secrets). Include: + * The DefectDojo instance URL. + * The username (typically their email address). + * The temporary password you just set. + * A note that they should change the password and enable MFA (if your instance uses MFA) on first login. + +4. **The new user logs in and rotates the credential.** They can either: + * Log in with the temporary password and then change it from their profile menu, or + * Use the **I forgot my password** link on the login page to set a password directly without using the temporary one. The temporary password is still required for the initial account record to exist, but the user does not need to remember it if they use the password-reset flow. + +5. **The new user configures MFA** from their profile menu. We strongly recommend requiring MFA for all users on instances that aren't behind SSO. + +## SSO Users + +If your instance is configured with [SSO](../configure_sso/), the workflow is different — users are typically created on first login from the Identity Provider, and you only need to grant them group membership or roles afterwards. + +## Recovering from a lost MFA token + +If a user loses access to their MFA device, see the [MFA recovery section](/get_started/pro/cloud/connectivity-troubleshooting/#ive-lost-access-to-my-mfa-codes) of the connectivity troubleshooting guide. There is currently no way to remove MFA from an account without an MFA code — the workaround is to create a new account for the user and re-grant the same permissions. diff --git a/docs/content/admin/user_management/PRO__creating_new_users.md b/docs/content/admin/user_management/PRO__creating_new_users.md new file mode 100644 index 00000000000..68c4d6d3c49 --- /dev/null +++ b/docs/content/admin/user_management/PRO__creating_new_users.md @@ -0,0 +1,40 @@ +--- +title: "Creating a new user" +description: "How to onboard a new user onto your DefectDojo instance" +audience: pro +weight: 1 +--- + +This page describes the recommended onboarding workflow for adding new users to a DefectDojo instance. DefectDojo users can be used as both standard, human-operated accounts and as service accounts. + +The admin who creates the account is responsible for delivering the initial credentials (username and password) to the new user. + +## Recommended workflow + +1. **Create the user account** in DefectDojo (Superuser only): + * Navigate to **👤 Users → ➕ New User**. + * Enter the new user's name and email address. + * Set a temporary password. + * Submit the form. + +2. **Assign permissions** as appropriate — Product/Product Type membership, Configuration Permissions, Global Role, or Superuser status. See [Set a User's permissions](../set_user_permissions/) for details. A new user with no assignments will not be able to see any Products or Findings. + +3. **Send the credentials to the new user out-of-band** (over email, your team's chat tool, or however you normally share secrets). Include: + * The DefectDojo instance URL. + * The username (typically their email address). + * The temporary password you just set. + * A note that they should change the password and enable MFA (if your instance uses MFA) on first login. + +4. **The new user logs in and rotates the credential.** They can either: + * Log in with the temporary password and then change it from their profile menu, or + * Use the **I forgot my password** link on the login page to set a password directly without using the temporary one. The temporary password is still required for the initial account record to exist, but the user does not need to remember it if they use the password-reset flow. + +5. **The new user configures MFA** from their profile menu. We strongly recommend requiring MFA for all users on instances that aren't behind SSO. + +## SSO Users + +If your instance is configured with [SSO](../configure_sso/), the workflow is different — users are typically created on first login from the Identity Provider, and you only need to grant them group membership or roles afterwards. + +## Recovering from a lost MFA token + +If a user loses access to their MFA device, see the [MFA recovery section](/get_started/pro/cloud/connectivity-troubleshooting/#ive-lost-access-to-my-mfa-codes) of the connectivity troubleshooting guide. There is currently no way to remove MFA from an account without an MFA code — the workaround is to create a new account for the user and re-grant the same permissions. diff --git a/docs/content/asset_modelling/components/PRO__components.md b/docs/content/asset_modelling/components/PRO__components.md new file mode 100644 index 00000000000..71499b79afb --- /dev/null +++ b/docs/content/asset_modelling/components/PRO__components.md @@ -0,0 +1,68 @@ +--- +title: "Components" +description: "Tracking third-party libraries and software components in DefectDojo Pro" +audience: pro +weight: 1 +--- + +In DefectDojo, Components represent third-party libraries, software components, and modules that potentially have vulnerabilities. + + +## Component Views + +DefectDojo Pro includes a dedicated table view for Components, which can be found in the sidebar. This view shows Active Findings, Duplicate Findings, and Total Findings for each Component. These figures include all Assets on the DefectDojo instance. + +An individual Asset's Components can be seen on the Asset view. + +## The Component Table + +The Component Table displays the following columns: + +* **Component** — the name of the component, populated from scan data. +* **Version** — the component version, populated from scan data. +* **Active Findings** — count of Active Findings associated with the component. +* **Duplicate Findings** — count of Duplicate Findings associated with the component. +* **Total Findings** — total count of all Findings associated with the component. + +Clicking on the Component Name or the values for Active Findings, Duplicate Findings, or Total Findings opens a filtered list of Findings for the respective field. + +A **None** Component is displayed on the table, which shows all Findings that are not associated with any Component. + +Imported Components remain on the table even if all of their associated Findings are Mitigated. When Findings are imported for a specific Component, the Component Table is updated to accurately reflect the new Finding totals. + + +### Example + +A Component imported from a Dependency-Check scan against an application with a vulnerable `lodash` dependency might appear on the table as: + +| Component | Version | Active Findings | Duplicate Findings | Total Findings | +| --- | --- | --- | --- | --- | +| npm:lodash | 4.17.15 | 3 | 1 | 5 | + +Clicking `npm:lodash` opens the list of every Finding that references this Component. Clicking `3` opens the same list filtered to Active Findings only. + +## Adding Components + +Components can be parsed from a scan import or by manually editing a Finding. Once a Component Name is associated with a Finding, a corresponding entry will be added to the Component Table automatically. If the Component is already associated with other Findings in DefectDojo, the totals for Active Findings, Duplicate Findings, and Total Findings are updated accordingly. + +### How Components are Parsed from Scan Data + +When a scan is imported, parsers populate the **Component Name** and **Component Version** fields on each Finding from the scan output. The Component Table is then built from those values. The level of detail and the naming convention depend on the tool that produced the scan: + +* **Software Composition Analysis (SCA) tools** typically report a package name and exact version. For example, OWASP Dependency-Check derives the Component from the [Package URL](https://github.com/package-url/purl-spec) in its identifier — a `pkg:npm/lodash@4.17.15` purl becomes `Component Name: npm:lodash`, `Component Version: 4.17.15`. +* **Container and OS package scanners** such as Trivy, Anchore Grype, and Anchore Engine report the affected OS or language package — for example, `Component Name: curl`, `Component Version: 7.68.0`. +* **Language-specific dependency scanners** such as npm Audit, pip-audit, bundler-audit, Retire.js, Govulncheck, and OSV-Scanner populate the offending package and version from their respective ecosystem manifests. + +Scanners focused on configuration, infrastructure, or source-code logic (such as SAST and IaC tools) generally do not populate the Component fields, and their Findings appear under the **None** Component. + +To add or change a Component manually, edit the Finding and set the **Component Name** and **Component Version** fields directly. The Component Table updates as soon as the Finding is saved. + +## Updating Components + +To update a Component Name or Version, all Findings associated with the Component must have their Component Name or Component Version field updated. + +## Removing Components + +To remove a Component from the Component Table, all Findings associated with the Component must be updated to remove their Component Name and Component Version fields. Components are also removed if all of their associated Findings are deleted. + +If all of a Component's Findings are Mitigated, the Component remains on the table but its Active Findings value is set to 0. diff --git a/docs/content/asset_modelling/components/_index.md b/docs/content/asset_modelling/components/_index.md new file mode 100644 index 00000000000..77cd30856ff --- /dev/null +++ b/docs/content/asset_modelling/components/_index.md @@ -0,0 +1,10 @@ +--- +title: "Components & Endpoints" +date: 2021-02-02T20:46:29+01:00 +draft: false +type: docs +weight: 4 +sidebar: + collapsed: false +exclude_search: true +--- diff --git a/docs/content/asset_modelling/engagements_tests/OS__calendar.md b/docs/content/asset_modelling/engagements_tests/OS__calendar.md index 74d7afbf31f..64286709dbb 100644 --- a/docs/content/asset_modelling/engagements_tests/OS__calendar.md +++ b/docs/content/asset_modelling/engagements_tests/OS__calendar.md @@ -2,7 +2,7 @@ title: "Calendar" description: "How to use the Calendar in DefectDojo Pro" audience: opensource -weight: 2 +weight: 9 --- DefectDojo’s Calendar provides a centralized timeline view of all Engagements and Tests with defined start and end dates, allowing Users to quickly understand testing activity across Products, identify scheduling overlaps, and navigate directly to related objects. diff --git a/docs/content/asset_modelling/engagements_tests/OS__engagements.md b/docs/content/asset_modelling/engagements_tests/OS__engagements.md index b6b28ddf0a6..781f654af44 100644 --- a/docs/content/asset_modelling/engagements_tests/OS__engagements.md +++ b/docs/content/asset_modelling/engagements_tests/OS__engagements.md @@ -2,7 +2,7 @@ title: "Engagements" description: "Understanding Engagements in DefectDojo OS" audience: opensource -weight: 2 +weight: 3 --- Product Types → Products → **ENGAGEMENTS** → Tests → Findings diff --git a/docs/content/asset_modelling/engagements_tests/OS__products.md b/docs/content/asset_modelling/engagements_tests/OS__products.md new file mode 100644 index 00000000000..0a2fa253a25 --- /dev/null +++ b/docs/content/asset_modelling/engagements_tests/OS__products.md @@ -0,0 +1,177 @@ +--- +title: "Products" +description: "Understanding Products in DefectDojo OS" +audience: opensource +weight: 2 +--- +Product Types → **PRODUCTS** → Engagements → Tests → Findings + +## Overview + +**Products** sit at the center of how security work is organized within DefectDojo’s object hierarchy. Products represent any project, program, software, or physical asset that your security team is testing, and host all of the security work and testing history related to the testing goal. Examples of Products can include: +- Software releases +- Third-party software +- Virtual machines or assets in production +- A single application +- A microservice +- An API +- A SaaS platform +- A mobile app +- An internal system +- A business service +- A customer-facing platform +- A cloud environment or infrastructure domain + +In general, a Product should represent the “thing” whose security posture you want to track over time. This includes the associated testing history, Findings, metrics, ownership, integrations, and remediation workflows related to that “thing.” + +### Product Examples + +Products can become even more granular depending on the needs of your organization. For example, you may consider creating separate DefectDojo Products in the following scenarios: + +- “ExampleProduct” has a Windows version, a Mac version, and a Cloud version +- “ExampleProduct 1.0” uses completely different software components from “ExampleProduct 2.0”, and both versions are actively supported by your company. +- The team assigned to work on “ExampleProduct version A” is different from the product team assigned to work on “ExampleProduct version B”, and needs to have different security permissions assigned as a result. + +While you may also elect to represent these variations as Engagements within a single Product, RBAC can only be set at the level of Products or Product Types, which may limit users’ access to the appropriate Engagement (as well as the Tests and Findings within those Engagements) if they’re organized as such. For more information on RBAC and permissions in DefectDojo, click [here](/admin/user_management/about_perms_and_roles/). + +## Product Data + +Products will always include the following components: + +- **Unique name** +- **Description** +- **Product Type** +- **SLA Configuration** + +Optional Product metadata includes: + +- **Tags** +- **Personnel information** (e.g., Product Manager, Team Manager, Technical Contact, etc.) +- **Regulations** (e.g., HIPAA, GLBA, OPPA, etc.) +- **Business criticality** +- **Platform** (e.g., API, Desktop, IoT, Mobile, Web, etc.) +- **Lifecycle** (e.g., Construction, Production, Retirement, etc.) +- **Origin** (e.g., Third-Party Library, Purchased, Open Source, etc.) +- **User records** (i.e., the estimated number of user records in the Product) +- **Revenue** + +This metadata improves filtering, reporting, and prioritization across your security program, but most importantly, Products also contain all of the Engagements, Tests, and Findings related to the testing efforts surrounding that Product. All Findings from Tests ultimately roll up to the Product level, enabling long-term tracking, trend analysis, and reporting. + +## Accessing Products + +Products are accessible via the sidebar. The submenu also provides the option to create a new Product. + +![image](images/product_ss3.png) + +### Permissions + +Products can have Role-Based Access Control (RBAC) rules applied, which limit team members’ ability to view and interact with them. + +Permissions cascade downward, meaning that access to a Product automatically grants access to all objects within that Product (e.g., Engagements, Tests, and Findings). + +For more information on user roles, see our [Introduction To Roles article](/admin/user_management/about_perms_and_roles/). + +## Product View + +Product views contain a variety of tables and charts to interpret a Product’s status at a glance. This includes: + +- **Metadata** + - Including Product Type, business criticality, revenue, and other details added from the Product settings. +- **Metrics** + - A list of open Findings within the Product, grouped by severity +- **Service Level Agreement by Severity** + - Applies the Product SLA configuration from settings to the Findings within the Product. +- **Technologies** + - E.g., next.js, vue.js, npm v.1.2.3, Django, nginx, Hugo +- **Regulations** +- **Benchmark Progress** +- **Members** +- **Groups** +- **Contacts** +- **Notifications** + - Toggles notifications on and off depending on specific events (e.g., an Engagement has been added or closed) + +## Product Lifecycle + +### Create Products + +There are multiple ways to create a new Product, including: + +- The **Add Product** button in the All Products list + +![image](images/product_ss2.png) + +- From the dropdown menu of the Products table within a Product Type’s view + - This will automatically create the Product within that Product Type. + +![image](images/product_ss1.png) + +- The **Add Product** button in the sidebar + +![image](images/product_ss5.png) + +### Edit Products + +A Product can be edited from its settings, which can be accessed in two ways: + +- The **Edit** button within ⋮ kebab menu to the left of the Product in the All Products view + +![image](images/product_ss6.png) + +- The **Edit** button within the **Settings** dropdown in the Product’s view + +![image](images/product_ss7.png) + +### Delete Products + +The option to delete a Product can be found at the bottom of the same menus described in the **Edit Products** section above. This action can’t be undone. Product can’t be closed and reopened later. + +Deleting a Product will also delete the following: +- Any Engagements and Tests contained within the Product +- All associated security history, including Findings and integrations +- Any linked Jira Epics +- All notes and file uploads associated with the Product’s Engagements and Tests + +## Product Boundaries + +### Deduplication + +Products are “walled-off” and do not interact with other Products. DefectDojo’s Smart Features, such as Deduplication, only apply within the context of a single Product. Findings across different Products will not be automatically deduplicated. + +### Metrics + +Most reporting and metrics aggregate data at the Product level, making Products the primary unit for measuring and tracking risk. + +As a result, many key metrics are calculated per Product, including: + +- Total number of Findings (by severity or status) +- Mean time to remediate (MTTR) +- SLA compliance and breach rates +- Risk trends over time + +This means that how Products are structured will directly impact the accuracy and usefulness of reports. For example, grouping multiple unrelated systems under a single Product may obscure risk visibility, while overly granular Product structures can fragment reporting, making it difficult to identify broader trends. + +Product-specific metrics can be accessed from the **Metrics** button in the top bar of the chosen Product’s view. + +![image](images/product_ss8.png) + +### CI/CD Pipeline + +CI/CD pipelines automate the import of scan results. Regardless of the integration method, all scan imports must be associated with a Product, making the Product the anchor point for pipeline-driven security data. + +When a pipeline submits scan results, it must either: + +- Specify an existing Product (and optionally an Engagement), or +- Be configured in a way that consistently maps results to the correct Product + +All imported Findings will inherit the Product’s context, including ownership, permissions, SLA configuration, and reporting scope. + +In practice, Products should be defined to reflect how systems are built and deployed within CI/CD to ensure that security results are consistently associated with the correct application or service. + +### Jira Relationships + +Products can be mapped directly to Jira Projects, which push the Product’s Findings into a Jira instance. + +Because Findings inherit risk, priority, and ownership from their parent Product, the Product effectively determines the remediation context that flows into Jira tickets and Integrator workflows. + +Importantly, Products are also the primary determining factor in a Finding’s SLA characteristics. Therefore, the SLA of a Findings depends on the SLA configuration of its parent Product. More information about SLA configurations can be found [here](/asset_modelling/os_hierarchy/os__sla_configuration/#main-content). diff --git a/docs/content/asset_modelling/engagements_tests/OS__tests.md b/docs/content/asset_modelling/engagements_tests/OS__tests.md index fe0f7e7078e..bda58c23bbf 100644 --- a/docs/content/asset_modelling/engagements_tests/OS__tests.md +++ b/docs/content/asset_modelling/engagements_tests/OS__tests.md @@ -2,7 +2,7 @@ title: "Tests" description: "Understanding Tests in DefectDojo OS" audience: opensource -weight: 2 +weight: 4 --- Organizations → Assets → Engagements → **TESTS** → Findings diff --git a/docs/content/asset_modelling/engagements_tests/PRO__assets.md b/docs/content/asset_modelling/engagements_tests/PRO__assets.md new file mode 100644 index 00000000000..131561b68ee --- /dev/null +++ b/docs/content/asset_modelling/engagements_tests/PRO__assets.md @@ -0,0 +1,185 @@ +--- +title: "Assets" +description: "Understanding Assets in DefectDojo Pro" +audience: pro +weight: 2 +--- +Organizations → **ASSETS** → Engagements → Tests → Findings + +## Overview + +**Assets** sit at the center of how security work is organized within DefectDojo’s object hierarchy. Assets represent any project, program, software, or physical asset that your security team is testing, and host all of the security work and testing history related to the testing goal. Examples of Assets can include: +- Software releases +- Third-party software +- Virtual machines or assets in production +- A single application +- A microservice +- An API +- A SaaS platform +- A mobile app +- An internal system +- A business service +- A customer-facing platform +- A cloud environment or infrastructure domain + +In general, an Asset should represent the “thing” whose security posture you want to track over time. This includes the associated testing history, Findings, metrics, ownership, integrations, and remediation workflows related to that “thing.” + +### Asset Examples + +Assets can become even more granular depending on the needs of your organization. For example, you may consider creating separate DefectDojo Assets in the following scenarios: + +- “ExampleAsset” has a Windows version, a Mac version, and a Cloud version +- “ExampleAsset 1.0” uses completely different software components from “ExampleAsset 2.0”, and both versions are actively supported by your company. +- The team assigned to work on “ExampleAsset version A” is different from the Asset team assigned to work on “ExampleAsset version B”, and needs to have different security permissions assigned as a result. + +While you may also elect to represent these variations as Engagements within a single Asset, RBAC can only be set at the level of Assets or Organizations, which may limit users’ access to the appropriate Engagement (as well as the Tests and Findings within those Engagements) if they’re organized as such. For more information on RBAC and permissions in DefectDojo, click [here](/admin/user_management/about_perms_and_roles/). + +## Asset Data + +Assets will always include the following components: + +- **Organization** +- **Unique name** +- **Description** +- **SLA Configuration** +- **Prioritization Engine** + +Optional Asset metadata includes: + +- **Tags** +- **Business criticality** +- **User records** (i.e., the estimated number of user records in the Asset) +- **Revenue** +- **Personnel information** (e.g., Asset Manager, Team Manager, Technical Contact, etc.) +- **Regulations** (e.g., HIPAA, GLBA, OPPA, etc.) +- **Platform** (e.g., API, Desktop, IoT, Mobile, Web, etc.) +- **Lifecycle** (e.g., Construction, Production, Retirement, etc.) +- **Origin** (e.g., Third-Party Library, Purchased, Open Source, etc.) + +This metadata improves filtering, reporting, and prioritization across your security program, but most importantly, Assets also contain all of the Engagements, Tests, and Findings related to the testing efforts surrounding that Asset. All Findings from Tests ultimately roll up to the Asset level, enabling long-term tracking, trend analysis, and reporting. + +## Accessing Assets + +Assets are accessible via the sidebar. The submenu provides access to the [Asset Hierarchy](/asset_modelling/engagements_tests/pro__assets/#asset-nesting) and All Assets, as well as the option to create a new Asset. + +![image](images/assets_ss1.png) + +### Permissions + +Assets can have Role-Based Access Control (RBAC) rules applied, which limit team members’ ability to view and interact with them. + +Permissions cascade downward, meaning that access to an Asset automatically grants access to all objects within that Asset (e.g., Engagements, Tests, and Findings). + +For more information on user roles, see our [Introduction To Roles](/admin/user_management/set_user_permissions/#introduction-to-permission-types) article. + +## Asset View + +Asset views contain a variety of tables and charts to interpret an Asset’s status at a glance. This includes: + +- **Open Finding Severity** + - A list of open Findings within the Asset, grouped by severity +- **Asset Overview** + - A breakdown of various features of the Asset, including Description, Components, Contacts, [User Groups](/admin/user_management/create_user_group/ +), Members, Technologies, and Regulations. + - Technologies: next.js, vue.js, npm v.1.2.3, Django, nginx, Hugo +- **Metadata** + - Including parent and child Assets, Organization, business criticality, revenue, and other details added from the Asset’s settings. +- **Service Level Agreement by Severity** + - Applies the Asset’s SLA configuration from settings to the Findings within the Asset. +- **Finding Severity Breakdown** + - A graph of the Findings within the Asset, organized by severity. +- **Finding Distribution** + - A breakdown of the Findings within the Asset, organized by status (e.g., Active, Mitigated, Static, and Dynamic) +- **All Engagements** + - A list of Engagements contained within the Asset. + +## Asset Lifecycle + +### Create Assets + +There are two ways to create Assets: + +- From the **New Asset** option in the side menu +- From the **New Asset** button at the top of the All Assets list + +## Edit Assets + +Assets can be edited by clicking **Edit Asset** from within the gear menu at the top right of the Asset’s view. The same menu can also be accessed by clicking the ⋮ kebab menu to the left of the Asset in the All Assets view. + +All ensuing fields that can be edited are also available when the Asset is being created. + +![image](images/assets_ss2.png) + +### Delete Assets + +Deleting an Asset can be performed by selecting **Delete Asset** from the Asset’s settings. This action can’t be undone. Assets can’t be closed and reopened later. + +Deleting an Asset will also delete the following: +- Any Engagements and Tests contained within the Asset +- All associated security history, including Findings and integrations +- Any linked Jira Epics +- All notes and file uploads associated with the Asset’s Engagements and Tests + +## Asset Boundaries + +### Deduplication + +Assets are “walled-off” and do not interact with other Assets. DefectDojo’s Smart Features, such as Deduplication, only apply within the context of a single Asset. Findings across different Assets will not be automatically deduplicated. + +### Reporting and Metrics + +Most reporting and metrics aggregate data at the Asset level, making Assets the primary unit for measuring and tracking risk. + +As a result, many key metrics are calculated per Asset, including: + +- Total number of Findings (by severity or status) +- Mean time to remediate (MTTR) +- SLA compliance and breach rates +- Risk trends over time + +This means that how Assets are structured will directly impact the accuracy and usefulness of reports. For example, grouping multiple unrelated systems under a single Asset may obscure risk visibility, while overly granular Asset structures can fragment reporting, making it difficult to identify broader trends. + +### Connectors + +In DefectDojo Pro, Connectors are mapped to different Assets in DefectDojo Pro, making them the primary integration point between DefectDojo and your broader security ecosystem. + +Once a Connector has been attached to an Asset, it will import scan results and create or update Engagements, Tests, and Findings within that Asset. + +For more information about Connectors, click [here](/import_data/pro/connectors/about_connectors/#main-content). + +### CI/CD Pipelines + +CI/CD pipelines automate the import of scan results. Regardless of the integration method, all scan imports must be associated with an Asset, making the Asset the anchor point for pipeline-driven security data. + +When a pipeline submits scan results, it must either: + +- Specify an existing Asset (and optionally an Engagement), or +- Be configured in a way that consistently maps results to the correct Asset + +All imported Findings will inherit the Asset’s context, including ownership, permissions, priority/risk configuration, and reporting scope. + +In practice, Assets should be defined to reflect how systems are built and deployed within CI/CD to ensure that security results are consistently associated with the correct application or service. + +### SLAs, Priority, and Risk + +In DefectDojo Pro, Findings inherit their SLA targets, Priority, and Risk from the Asset that contains them. Asset metadata (e.g., business criticality, revenue, etc.) are used to automatically calculate Priority and Risk values. + +This means that the same vulnerability may receive a different Priority or Risk score depending on whether it affects an internal development system or a production asset supporting critical business operations. + +### Jira / Integrators Relationships + +Assets can be mapped directly to [Jira](/issue_tracking/jira/pro__jira_guide/#main-content) or [Integrators](/issue_tracking/pro_integration/integrations_toolreference/#main-content) instances (e.g. GitHub, GitLab, ServiceNow, etc.), which push the Asset’s Findings outward into external ticketing/work-management systems. + +Because Findings inherit risk, priority, and ownership from their parent Asset, the Asset effectively determines the remediation context that flows into Jira tickets and Integrator workflows. + +Importantly, Assets are also the primary determining factor in a Finding’s SLA characteristics. Therefore, the SLA of a Findings depends on the SLA configuration of its parent Asset. More information about SLA configurations can be found [here](/asset_modelling/pro_hierarchy/priority_sla/#working-with-slas). + +## Asset Nesting + +DefectDojo supports parent-child relationship between two Assets within the same Organization. This can be configured during Asset creation or in the Asset’s settings. + +You can visualize the structure of Assets in DefectDojo and change relationships using the **Asset Hierarchy** option in the sidebar. + +After selecting the Assets to be visualized from the corresponding table, click **View Asset Hierarchy** to generate a flow chart of the relationship between the chosen Assets, if any. + +Further information on the effect of nesting Assets on deduplication, RBAC, and other details, as well as example use cases, can be found [here](/asset_modelling/pro_hierarchy/assets_organizations/#asset-nesting-examples). diff --git a/docs/content/asset_modelling/engagements_tests/PRO__calendar.md b/docs/content/asset_modelling/engagements_tests/PRO__calendar.md index 0509fb09ca4..d6bcafee948 100644 --- a/docs/content/asset_modelling/engagements_tests/PRO__calendar.md +++ b/docs/content/asset_modelling/engagements_tests/PRO__calendar.md @@ -2,7 +2,7 @@ title: "Calendar" description: "How to use the Calendar in DefectDojo Pro" audience: pro -weight: 2 +weight: 9 --- DefectDojo features a built-in Calendar so you can track all prior and active Engagements and Tests within your organization. Any time a User creates a new Engagement or Test and establishes the start and end dates, a corresponding entry will automatically be added to the Calendar. diff --git a/docs/content/asset_modelling/engagements_tests/PRO__engagements.md b/docs/content/asset_modelling/engagements_tests/PRO__engagements.md index 8fcba6738ce..06973a2e490 100644 --- a/docs/content/asset_modelling/engagements_tests/PRO__engagements.md +++ b/docs/content/asset_modelling/engagements_tests/PRO__engagements.md @@ -2,7 +2,7 @@ title: "Engagements" description: "Understanding Engagements in DefectDojo Pro" audience: pro -weight: 2 +weight: 3 --- Organizations → Assets → **ENGAGEMENTS** → Tests → Findings @@ -70,7 +70,7 @@ Engagements sit below Assets and above Tests in the object hierarchy. As such, a ### Create Engagements -Before creating an Engagement, you must first have created an Asset to contain it. +Before creating an Engagement, you must first have [created an Asset](/asset_modelling/engagements_tests/pro__assets/#create-assets) to contain it. There are several ways to create an Engagement: @@ -117,7 +117,11 @@ Changing an Engagement’s status to “Completed” will mean that most write o ### Edit Engagements -Engagements can be edited by clicking **Edit Engagement** from within the gear menu. All ensuing fields that can be edited are also available when the Engagement is being created. +Engagements can be edited by clicking **Edit Engagement** from within the gear menu. The same menu can also be accessed by clicking the ⋮ kebab menu to the left of the Asset in the All Assets view. + +All ensuing fields that can be edited are also available when the Engagement is being created. + +![image](images/engagements_ss99.png) ### Copy Engagements diff --git a/docs/content/asset_modelling/engagements_tests/PRO__tests.md b/docs/content/asset_modelling/engagements_tests/PRO__tests.md index dbe4e3f9629..9eb3c550729 100644 --- a/docs/content/asset_modelling/engagements_tests/PRO__tests.md +++ b/docs/content/asset_modelling/engagements_tests/PRO__tests.md @@ -2,7 +2,7 @@ title: "Tests" description: "Understanding Tests in DefectDojo Pro" audience: pro -weight: 2 +weight: 4 --- Organizations → Assets → Engagements → **TESTS** → Findings diff --git a/docs/content/get_started/about/about_defectdojo.md b/docs/content/get_started/about/about_defectdojo.md index 473ca746e15..adecfb0dcdc 100644 --- a/docs/content/get_started/about/about_defectdojo.md +++ b/docs/content/get_started/about/about_defectdojo.md @@ -68,8 +68,8 @@ For teams managing a smaller volume of Findings, DefectDojo Open-Source is a gre There are a few supported ways to install DefectDojo’s Open-Source edition ([available on Github](https://github.com/DefectDojo/django-DefectDojo)): [Docker Compose](https://github.com/DefectDojo/django-DefectDojo/blob/master/readme-docs/DOCKER.md) is the easiest method to install the core program and services required to run DefectDojo. -Our [Architecture](https://docs.defectdojo.com/open_source/installation/architecture/) guide gives you an overview of each service and component used by DefectDojo. -[Running In Production](https://docs.defectdojo.com/open_source/installation/running-in-production/) lists system requirements, performance tweaks and maintenance processes for running DefectDojo on a production server (with Docker Compose). +Our [Architecture](https://docs.defectdojo.com/get_started/open_source/architecture/) guide gives you an overview of each service and component used by DefectDojo. +[Running In Production](https://docs.defectdojo.com/get_started/open_source/running-in-production/) lists system requirements, performance tweaks and maintenance processes for running DefectDojo on a production server (with Docker Compose). Kubernetes is not fully supported at the Open-Source level, but this guide can be referenced and used as a starting point to integrate DefectDojo into Kubernetes architecture. diff --git a/docs/content/get_started/contributing/documentation.md b/docs/content/get_started/contributing/documentation.md index c3495e21fdd..64200b7e722 100644 --- a/docs/content/get_started/contributing/documentation.md +++ b/docs/content/get_started/contributing/documentation.md @@ -37,9 +37,22 @@ audience: opensource ## Unit tests for docs -DefectDojo's docs use Lychee to check for 404s and other link errors. To run this test locally, you can run this command from the root of the repo. This will delete anything in Hugo's `/public/` folder and then rebuild. +DefectDojo's docs use Lychee to check for 404s and other link errors. CI runs two checks: the rendered docs site, and any `docs.defectdojo.com` URLs hardcoded into the Django app (templates and settings). Both use a `--remap` so absolute `docs.defectdojo.com` URLs resolve against the freshly built site. To run both locally from the root of the repo: -`cd docs && rm -rf public/ && hugo --minify --gc --config config/production/hugo.toml && lychee --offline --no-progress --root-dir public './public/**/*.html'` +``` +cd docs && rm -rf public/ && hugo --minify --gc --config config/production/hugo.toml && cd .. + +lychee --offline --no-progress \ + --root-dir "$PWD/docs/public" \ + --remap "https://docs.defectdojo.com file://$PWD/docs/public" \ + './docs/public/**/*.html' + +lychee --offline --no-progress \ + --root-dir "$PWD/docs/public" \ + --remap "https://docs.defectdojo.com file://$PWD/docs/public" \ + --exclude '%7[BD]' \ + $(grep -rl 'docs\.defectdojo\.com' dojo/ --include='*.html' --include='*.py' --include='*.tpl') +``` ### Theme overrides diff --git a/docs/content/help/glossary.md b/docs/content/help/glossary.md index 45198c57f43..c9205c03a43 100644 --- a/docs/content/help/glossary.md +++ b/docs/content/help/glossary.md @@ -19,6 +19,8 @@ A scoped security activity representing a testing window, pipeline, or assessmen A single execution of a scanner or manual assessment within an Engagement. Tests store execution metadata and act as the ingestion point for Findings. ## Service (Both) An optional sub-object used to attribute Findings to a specific component or interface within an Asset. Services are most useful in OS DefectDojo, as their functionality is replicated and enhanced by Asset Hierarchy in the Pro version. +## Components (Both) +A third-party library, software module, or external dependency that is tracked in DefectDojo Pro. Imported Components are derived from scan data and associated with Findings. In the Pro UI, the Component Table aggregates Active, Duplicate, and Total Finding counts per Component and remains populated even when all associated Findings are Mitigated. ## Finding (Both) The most granular vulnerability object in DefectDojo's Product Hierarchy that represents a discrete security issue. ### Finding Status (Both) diff --git a/docs/content/import_data/import_intro/reimport.md b/docs/content/import_data/import_intro/reimport.md index ccfcbd112d0..04615686c28 100644 --- a/docs/content/import_data/import_intro/reimport.md +++ b/docs/content/import_data/import_intro/reimport.md @@ -59,6 +59,22 @@ If you’re using a triage\-less scanner, or you don’t otherwise want Closed F * Set **do\_not\_reactivate** to **True** if using the API * Check the **Do Not Reactivate** checkbox if using the UI +### Force Active and Force Verified behavior + +Setting `active=true` (UI: **Force Active**) or `verified=true` (UI: **Force Verified**) on a Reimport will set the corresponding status on every matched Finding, **including findings that would otherwise be Inactive because they were Mitigated**. This is the same reactivation behavior described above, just made explicit on every incoming Finding. + +Force Active and Force Verified do **not** override statuses that represent an explicit user or system decision about why a Finding should not be Active: + +| Status | Does Force Active reactivate it? | Why | +|---|---|---| +| Mitigated / Closed | Yes | Same as the default reactivation behavior | +| Risk Accepted | No | The Finding is Inactive because a user explicitly accepted the risk; reimport must not silently revoke that decision | +| Duplicate | No | The Finding is Inactive because deduplication marked it as a duplicate of another Finding; the original Finding (not the duplicate) is what should be active | +| False Positive | No | Same reasoning as Risk Accepted — an explicit triage decision | +| Out of Scope | No | Same reasoning as Risk Accepted — an explicit triage decision | + +If you want a Risk Accepted or Duplicate Finding to become Active again, you need to remove the Risk Acceptance or the Duplicate marker first. Force Active alone will not do it. + ## Opening the Reimport form The **Re\-Import Findings** form can be accessed on any Test page, under the **⚙️Gear** drop\-down menu. diff --git a/docs/content/metrics_reports/reports/using_the_report_builder.md b/docs/content/metrics_reports/reports/using_the_report_builder.md index b7f2c905f4e..b1e85ddbc9f 100644 --- a/docs/content/metrics_reports/reports/using_the_report_builder.md +++ b/docs/content/metrics_reports/reports/using_the_report_builder.md @@ -5,7 +5,7 @@ weight: 1 aliases: - /en/share_your_findings/pro_reports/using_the_report_builder --- -DefectDojo allows you to create Custom Reports for external audiences, which summarize the Findings or Endpoints that you wish to report on. Custom Reports can include branding and boilerplate text, and can also be used as **[Templates](https://docs.defectdojo.com/en/pro_reports/working-with-generated-reports/)** for future reports. +DefectDojo allows you to create Custom Reports for external audiences, which summarize the Findings or Endpoints that you wish to report on. Custom Reports can include branding and boilerplate text, and can also be used as **[Templates](https://docs.defectdojo.com/metrics_reports/reports/working_with_generated_reports/)** for future reports. ## Opening the Report Builder diff --git a/docs/content/releases/pro/changelog.md b/docs/content/releases/pro/changelog.md index a172e11f1be..4733e3dc737 100644 --- a/docs/content/releases/pro/changelog.md +++ b/docs/content/releases/pro/changelog.md @@ -19,6 +19,13 @@ For Open Source release notes, please see the [Releases page on GitHub](https:// ## May 2026: v2.58 +### May 11, 2026: v2.58.2 + +* **(Connectors)** Connectors now support subtypes, so a single connector type can be configured against multiple flavors of the same upstream tool. +* **(Pro UI)** Jira push success/error messages are now displayed correctly in the Pro UI, so it's clear whether the push actually reached Jira. +* **(MCP)** MCP acknowledgement settings are now consistent across superusers, so toggling acknowledgement no longer drifts between admin accounts. +* **(Classic UI)** The Celery Status page is now gated behind the `celery_status` feature flag, off by default. + ### May 6, 2026: v2.58.1 * **(Pro UI)** You can now activate or deactivate Test Types and Users directly from their list menus, so retiring or restoring entries no longer requires opening the edit form. diff --git a/docs/content/supported_tools/parsers/file/aws_prowler_v3plus.md b/docs/content/supported_tools/parsers/file/aws_prowler_v3plus.md index e8fe00b34a6..759907ba90e 100644 --- a/docs/content/supported_tools/parsers/file/aws_prowler_v3plus.md +++ b/docs/content/supported_tools/parsers/file/aws_prowler_v3plus.md @@ -5,7 +5,7 @@ toc_hide: true ### File Types DefectDojo parser accepts a native `json` file produced by prowler v3 with file extension `.json` or a `ocsf-json` file produced by prowler v4 with file extension `.ocsf.json`. -Please note: earlier versions of AWS Prowler create output data in a different format. See our other [prowler parser documentation](https://docs.defectdojo.com/supported_tools/file/aws_prowler/) if you are using an earlier version of AWS Prowler. +Please note: earlier versions of AWS Prowler create output data in a different format. See our other [prowler parser documentation](https://docs.defectdojo.com/supported_tools/parsers/file/aws_prowler/) if you are using an earlier version of AWS Prowler. JSON reports can be created from the [AWS Prowler v3 CLI](https://docs.prowler.com/projects/prowler-open-source/en/v3/tutorials/reporting/#json) using the following command: `prowler -M json` diff --git a/docs/content/supported_tools/parsers/file/burp_suite_dast.md b/docs/content/supported_tools/parsers/file/burp_suite_dast.md index 3113895fba5..4c671e686dd 100644 --- a/docs/content/supported_tools/parsers/file/burp_suite_dast.md +++ b/docs/content/supported_tools/parsers/file/burp_suite_dast.md @@ -7,7 +7,7 @@ toc_hide: true The Burp Suite DAST Scan parser processes HTML reports from Burp Suite DAST and imports the findings into DefectDojo. The parser extracts vulnerability details, severity ratings, descriptions, remediation steps, and other metadata from the HTML report. ## Supported File Types -The parser accepts a Standard Report as an HTML file. To parse an XML file instead, use the [Burp XML parser](https://docs.defectdojo.com/supported_tools/file/burp/). +The parser accepts a Standard Report as an HTML file. To parse an XML file instead, use the [Burp XML parser](https://docs.defectdojo.com/supported_tools/parsers/file/burp/). See the Burp documentation for information on how to export a Standard Report: [Burp Suite DAST Downloading reports](https://portswigger.net/burp/documentation/dast/user-guide/work-with-scan-results/generate-reports) diff --git a/docs/content/supported_tools/parsers/universal_parser.md b/docs/content/supported_tools/parsers/universal_parser.md index f8a3cfeb1ef..b2b97d135b6 100644 --- a/docs/content/supported_tools/parsers/universal_parser.md +++ b/docs/content/supported_tools/parsers/universal_parser.md @@ -161,3 +161,14 @@ You can edit the Test_Type associated with your Universal Parser to change: * Whether it is "active" or not. If not, it will not appear as an option in the "Scan Type" drop-down on the "Add Findings" page * Whether its findings should be marked "static" or "dynamic" * You can tweak the same-tool and cross-tool deduplication hash codes, as well as the reimport hash codes, for your Universal Parser under "Enterprise Settings". By default, only same-tool deduplication and reimport hash codes are populated, with the required values Title, Severity, and Description. + +### Current limitations: editing and deleting a Universal Parser + +The field-mapping configuration of an existing Universal Parser cannot be modified after it is created — only the associated Test_Type (above) is editable. There is also no in-UI option to delete a Universal Parser. + +The current workaround for both cases is: + +* **To change a parser's field mappings:** create a new Universal Parser with the desired mappings (using a representative sample file as in Step 1), and switch new imports to use the new parser. Existing Tests already imported under the old parser are unaffected. +* **To "retire" a parser you no longer want to use:** mark its associated Test_Type as inactive (uncheck "active" on the Test_Type). It will stop appearing in the "Scan Type" drop-down on the Add Findings page. The parser configuration itself will remain in the database. + +If you need a parser configuration permanently removed (for example, because it contains sensitive field names), contact [DefectDojo Support](mailto:support@defectdojo.com). diff --git a/docs/content/triage_findings/finding_deduplication/PRO_enabling_product_deduplication.md b/docs/content/triage_findings/finding_deduplication/PRO_enabling_product_deduplication.md index 12b5ce3dfdd..3e588bf57e0 100644 --- a/docs/content/triage_findings/finding_deduplication/PRO_enabling_product_deduplication.md +++ b/docs/content/triage_findings/finding_deduplication/PRO_enabling_product_deduplication.md @@ -1,55 +1,55 @@ --- title: "Enabling Deduplication" -description: "How to enable Deduplication at the Product level" +description: "How to enable Deduplication at the Product or Engagement level" weight: 2 audience: pro aliases: - /en/working_with_findings/finding_deduplication/enabling_product_deduplication --- -Deduplication can be implemented at either a Product level or at a more narrow Engagement level. +Deduplication can be applied at a Product\-wide level, or scoped more narrowly to a single Engagement. ## Deduplication for Products -1. Start by navigating to the System Settings page. This is nested under **Settings \> Pro Settings \> ⚙️ System Settings** on the sidebar. +1. Navigate to the System Settings page: **Settings \> Pro Settings \> ⚙️ System Settings** on the sidebar. ![image](images/enabling_product-level_deduplication.png) -2. **Deduplication and Finding Settings** are at the top of the **System Settings** menu. -​ +2. The **Deduplication and Finding Settings** card is at the top of the **System Settings** page. + ![image](images/enabling_product-level_deduplication_2.png) ### Enable Finding Deduplication -**Enable Finding Deduplication** will turn on the Deduplication Algorithm for all Findings. Deduplication will be triggered on all subsequent imports \- when this happens, DefectDojo will look at any Findings contained in the destination Product, and deduplicate as per your settings. - -### Delete Deduplicate Findings +**Enable Finding Deduplication** turns on the Deduplication Algorithm for all Findings. Once enabled, Deduplication runs on every subsequent import — DefectDojo compares imported Findings against existing Findings in the destination Product and marks duplicates according to your configuration. -**Delete Deduplicate Findings**, combined with the **Maximum Duplicates** field allows DefectDojo to limit the amount of Duplicate Findings stored. When this field is enabled, DefectDojo will only keep a certain number of Duplicate Findings. +### Delete Duplicate Findings -Applying **Delete Deduplicate Findings** will begin a deletion process immediately. DefectDojo will look at each Finding with Duplicates recorded, and will delete old duplicate Findings until the Maximum Duplicate number has been reached. +**Delete Duplicate Findings**, combined with the **Maximum Duplicates** field, limits how many duplicate Findings DefectDojo retains. When enabled, a background job periodically prunes excess duplicates so that each original Finding keeps no more than the configured **Maximum Duplicates** count. Oldest duplicates are removed first. ## Deduplication for Engagements -Rather than Deduplicating across an entire Product, you can set a deduplication scope to be within a single Engagement exclusively. +Rather than Deduplicating across an entire Product, you can scope Deduplication to a single Engagement. + +### Open the Engagement form -### Edit Engagement page +* **For a new Engagement:** open the **📥 Engagements** sub‑menu on the sidebar and click **\+ New Engagement**. -* To enable Deduplication within a New Engagement, start with the **\+ New Engagement** option from the sidebar, which you can find by opening the **📥Engagements** sub\-menu. -​ ![image](images/enabling_deduplication_within_an_engagement.png) -* To enable Deduplication within an existing Engagement: from the **All Engagements** page, select the **Edit Engagement** option from the **⋮** menu. -​ +* **For an existing Engagement (from the All Engagements page):** open the **⋮** menu for the Engagement and select **Edit Engagement**. + ![image](images/enabling_deduplication_within_an_engagement_2.png) -* You can also open this menu from a specific **Engagement Page** by clicking the ⚙️Gear icon in the top\-right hand corner. -​ +* **For an existing Engagement (from the Engagement page):** open the **⚙️ Gear** menu in the top‑right corner of the page and select **Edit Engagement**. + ![image](images/enabling_deduplication_within_an_engagement_3.png) -### Completing the Edit Engagement form +### Completing the Engagement form -1. Start by opening the **Optional Fields \+** menu at the bottom of the **Edit Engagement** form. -2. Click the ☐ **Deduplication Within This Engagement** box. +1. On the Engagement form, locate the ☐ **Isolate Deduplication from Other Engagements** checkbox. It appears above the **Optional Fields \+** panel. +2. Check the box to scope Deduplication to this Engagement. 3. Submit the form. -![image](images/enabling_deduplication_within_an_engagement_4.png) \ No newline at end of file +When this option is enabled, Findings in this Engagement will only be deduplicated against other Findings within the same Engagement. Findings in other Engagements on the same Product are ignored by the Deduplication Algorithm. + +![image](images/enabling_deduplication_within_an_engagement_4.png) diff --git a/dojo/api_v2/views.py b/dojo/api_v2/views.py index 9583a7a9e9c..3f8bb0cf169 100644 --- a/dojo/api_v2/views.py +++ b/dojo/api_v2/views.py @@ -1324,6 +1324,7 @@ def get_duplicate_cluster(self, request, pk): ) @action(detail=True, methods=["post"], url_path=r"duplicate/reset", permission_classes=(IsAuthenticated, permissions.UserHasFindingRelatedObjectPermission)) def reset_finding_duplicate_status(self, request, pk): + self.get_object() checked_duplicate_id = reset_finding_duplicate_status_internal( request.user, pk, ) @@ -1344,6 +1345,7 @@ def reset_finding_duplicate_status(self, request, pk): detail=True, methods=["post"], url_path=r"original/(?P\d+)", permission_classes=(IsAuthenticated, permissions.UserHasFindingRelatedObjectPermission), ) def set_finding_as_original(self, request, pk, new_fid): + self.get_object() success = set_finding_as_original_internal(request.user, pk, new_fid) if not success: return Response(status=status.HTTP_400_BAD_REQUEST) diff --git a/dojo/management/commands/migrate_endpoints_to_locations.py b/dojo/management/commands/migrate_endpoints_to_locations.py index 25c739abb0a..081f2077024 100644 --- a/dojo/management/commands/migrate_endpoints_to_locations.py +++ b/dojo/management/commands/migrate_endpoints_to_locations.py @@ -1,18 +1,55 @@ +import contextlib import datetime import logging +import time +from collections import defaultdict from django.core.management.base import BaseCommand +from django.db import connection +from django.db.models import Prefetch from django.utils import timezone -from dojo.location.models import Location -from dojo.location.status import FindingLocationStatus -from dojo.models import DojoMeta, Endpoint, Endpoint_Status +from dojo.location.models import Location, LocationFindingReference, LocationProductReference +from dojo.location.status import FindingLocationStatus, ProductLocationStatus +from dojo.models import DojoMeta, Endpoint, Endpoint_Status, Product from dojo.url.models import URL logger = logging.getLogger(__name__) -# Chunk size for DB cursor and progress report -CHUNK_SIZE = 1000 +# Chunk size for the DB iterator. Tunable via --batch-size. +DEFAULT_BATCH_SIZE = 1000 +# How often to emit per-chunk progress lines. Tunable via --progress-every. +DEFAULT_PROGRESS_EVERY = 50 + + +# `LocationFindingReference.created` is `auto_now_add=True` (inherited from +# BaseModel). The original migration sets `created` to the source +# Endpoint_Status.date in a post-save UPDATE so that auto_now_add is +# bypassed. With bulk_create we don't get a post-save UPDATE; Django's +# SQLInsertCompiler.pre_save_val still calls Field.pre_save(add=True), +# which auto_now_add overrides with `now()`, ignoring our explicit value. +# The cleanest single-process fix is to temporarily flip auto_now_add off +# around the bulk write. +@contextlib.contextmanager +def _suspend_auto_now_add(model, field_name: str): + field = model._meta.get_field(field_name) + saved = field.auto_now_add + field.auto_now_add = False + try: + yield + finally: + field.auto_now_add = saved + + +# Phases tracked by --benchmark. Order is preserved in the summary table. +PHASES = ( + "fetch_endpoint", # iterator yields the next endpoint + "url_create", # URL.get_or_create_from_values + Location side-effect + "tags", # endpoint tag copy onto the location + "meta", # DojoMeta copy onto the location + "finding_refs", # LocationFindingReference creation per Endpoint_Status + "product_refs", # LocationProductReference creation +) class Command(BaseCommand): @@ -27,9 +64,70 @@ class Command(BaseCommand): help = "Usage: manage.py migrate_endpoints_to_locations" + def add_arguments(self, parser): + parser.add_argument( + "--batch-size", + type=int, + default=DEFAULT_BATCH_SIZE, + help=f"Endpoint.objects.iterator() chunk size (default: {DEFAULT_BATCH_SIZE}).", + ) + parser.add_argument( + "--progress-every", + type=int, + default=DEFAULT_PROGRESS_EVERY, + help=f"Emit a progress line every N endpoints (default: {DEFAULT_PROGRESS_EVERY}).", + ) + parser.add_argument( + "--benchmark", + action="store_true", + help="Track per-phase wall-clock and print a summary table at the end.", + ) + parser.add_argument( + "--query-count", + action="store_true", + help="Force-debug the DB cursor and count queries per chunk. " + "Has measurable overhead; use only for profiling runs.", + ) + + # -- Per-phase timing helpers -------------------------------------------- + + def _bench_start(self) -> float: + return time.perf_counter() if self.benchmark else 0.0 + + def _bench_end(self, phase: str, t0: float) -> None: + if self.benchmark: + self.timings[phase] += time.perf_counter() - t0 + self.counts[phase] += 1 + + # -- Tag inheritance bookkeeping ----------------------------------------- + + def _track_product_location(self, product: Product, location: Location) -> None: + """ + Record a (product, location) pair for the post-migration tag inheritance pass. + + The migration creates locations that may be linked to multiple products + (via the endpoint's own product and via each finding's product). We + collect every contributing product per location so the post-pass can + call ``LocationManager(product)._bulk_inherit_tags(locations)`` once + per product group — covering the case where a location is shared + across products with differing ``enable_product_tag_inheritance`` + flags (the helper short-circuits via its own diff check on repeat + visits, so redundancy is safe). + """ + if product is None or product.id is None: + return + if location is None or location.id is None: + return + self.locations_by_product_id[product.id].add(location.id) + self.product_obj_by_id.setdefault(product.id, product) + self.location_obj_by_id.setdefault(location.id, location) + + # -- Migration logic -------------------------------------------------- + def _endpoint_to_url(self, endpoint: Endpoint) -> Location: # Create the raw URL object first # This should create the location object as well + t = self._bench_start() url = URL.get_or_create_from_values( protocol=endpoint.protocol, user_info=endpoint.userinfo, @@ -39,16 +137,30 @@ def _endpoint_to_url(self, endpoint: Endpoint) -> Location: query=endpoint.query, fragment=endpoint.fragment, ) - # Add the endpoint tags to the location tags - if endpoint.tags: - [url.location.tags.add(tag) for tag in set(endpoint.tags.values_list("name", flat=True))] - # Add any metadata from the endpoint to the location - for meta in endpoint.endpoint_meta.all(): - DojoMeta.objects.get_or_create( - name=meta.name, - value=meta.value, - location=url.location, - ) + self._bench_end("url_create", t) + + # Add the endpoint tags to the location tags. Read names from the + # prefetched `tags` manager and add them in a single splat call so we + # do one round-trip per endpoint instead of one per tag. + t = self._bench_start() + tag_names = {tag.name for tag in endpoint.tags.all()} + if tag_names: + url.location.tags.add(*tag_names) + self._bench_end("tags", t) + + # Add any metadata from the endpoint to the location. + # bulk_create with ignore_conflicts mirrors the previous get_or_create + # semantics — DojoMeta.unique_together = (location, name) so any + # conflict is by definition the same row we'd otherwise have fetched. + # One INSERT per endpoint instead of SELECT+INSERT per meta row. + t = self._bench_start() + meta_rows = [ + DojoMeta(name=m.name, value=m.value, location=url.location) + for m in endpoint.endpoint_meta.all() + ] + if meta_rows: + DojoMeta.objects.bulk_create(meta_rows, ignore_conflicts=True) + self._bench_end("meta", t) return url.location @@ -69,53 +181,339 @@ def _convert_endpoint_status_to_string_status(self, endpoint_status: Endpoint_St return FindingLocationStatus.Active def _associate_location_with_findings(self, endpoint: Endpoint, location: Location) -> None: - # Determine if we can associate from the finding, or if have to use the product (for cases of zero findings on an endpoint) - if endpoint.status_endpoint.exists(): - # Iterate over each endpoint status to get the status and the finding object - for endpoint_status in endpoint.status_endpoint.all(): - if finding := endpoint_status.finding: - # Determine the status of the location based on the status of the endpoint status - status = self._convert_endpoint_status_to_string_status(endpoint_status) - # Create the association (which will also associate with the product) - reference = location.associate_with_finding( - finding=finding, - status=status, - auditor=endpoint_status.mitigated_by, - audit_time=endpoint_status.mitigated_time or endpoint_status.last_modified, - ) - # Update the created date from the endpoint status date - reference.created = timezone.make_aware(datetime.datetime(endpoint_status.date.year, endpoint_status.date.month, endpoint_status.date.day)) - reference.save(update_fields=["created"]) - # If there are no findings, we can at least associate with the product if it exists - elif product := endpoint.product: - location.associate_with_product(product) + # Pull the prefetched list once. Avoids the redundant `.exists()` round- + # trip the prior code did and lets the loop iterate prefetched data. + statuses = list(endpoint.status_endpoint.all()) + + # No findings — associate with the endpoint's product if one exists. + if not statuses: + if endpoint.product_id: + t_p = self._bench_start() + LocationProductReference.objects.bulk_create( + [LocationProductReference( + location=location, + product=endpoint.product, + status=ProductLocationStatus.Mitigated, + relationship="", + relationship_data={}, + )], + ignore_conflicts=True, + ) + self._bench_end("product_refs", t_p) + return + + # Build LFR rows for every status, and build LPR rows deduplicated by + # product, deriving the product status as Active iff any of THIS + # endpoint's findings on that product are Active. This bypasses + # `Location.associate_with_finding` (which would trigger full_clean + # validation + the post_save inherit_tags signal per row) and is + # semantically equivalent to the prior behavior in the common case + # where each endpoint maps to a distinct location. As a side-effect + # it also fixes the existing `associate_with_product` first-write- + # wins bug (where a Mitigated status would stick even when later + # Active findings come in for the same product). + finding_refs: list[LocationFindingReference] = [] + product_status_by_id: dict[int, str] = {} + product_obj_by_id: dict[int, object] = {} + + for endpoint_status in statuses: + finding = endpoint_status.finding + if finding is None: + continue + product = finding.test.engagement.product + # Track this contributing product for the post-migration tag + # inheritance pass (covers the case where a finding's product + # differs from endpoint.product). + self._track_product_location(product, location) + status = self._convert_endpoint_status_to_string_status(endpoint_status) + # Endpoint_Status.date is a Date; the original code persisted + # the same midnight-aware datetime in a post-save UPDATE. We + # set it directly here — bulk_create skips auto_now_add so the + # explicit value is honored. + created_dt = timezone.make_aware(datetime.datetime( + endpoint_status.date.year, + endpoint_status.date.month, + endpoint_status.date.day, + )) + finding_refs.append(LocationFindingReference( + location=location, + finding=finding, + status=status, + auditor=endpoint_status.mitigated_by, + audit_time=endpoint_status.mitigated_time or endpoint_status.last_modified, + relationship="", + relationship_data={}, + created=created_dt, + )) + if product.id not in product_obj_by_id: + product_obj_by_id[product.id] = product + product_status_by_id[product.id] = ( + ProductLocationStatus.Active + if status == FindingLocationStatus.Active + else ProductLocationStatus.Mitigated + ) + elif (status == FindingLocationStatus.Active + and product_status_by_id[product.id] != ProductLocationStatus.Active): + product_status_by_id[product.id] = ProductLocationStatus.Active + + t_f = self._bench_start() + if finding_refs: + with _suspend_auto_now_add(LocationFindingReference, "created"): + LocationFindingReference.objects.bulk_create( + finding_refs, ignore_conflicts=True, batch_size=500, + ) + self._bench_end("finding_refs", t_f) + + t_p = self._bench_start() + if product_obj_by_id: + product_refs = [ + LocationProductReference( + location=location, + product=product_obj_by_id[pid], + status=product_status_by_id[pid], + relationship="", + relationship_data={}, + ) + for pid in product_obj_by_id + ] + LocationProductReference.objects.bulk_create( + product_refs, ignore_conflicts=True, batch_size=500, + ) + self._bench_end("product_refs", t_p) + + # -- Progress + summary reporting ---------------------------------------- + + @staticmethod + def _fmt_duration(seconds: float) -> str: + s = int(seconds) + h, rem = divmod(s, 3600) + m, s = divmod(rem, 60) + if h: + return f"{h}h {m}m" + if m: + return f"{m}m {s}s" + return f"{s}s" + + def _log_progress(self, i: int, total: int, run_t0: float, queries_this_chunk: int | None) -> None: + elapsed = time.time() - run_t0 + rate = i / elapsed if elapsed > 0 else 0.0 + remaining = (total - i) / rate if rate > 0 else 0.0 + pct = (i / total * 100.0) if total else 100.0 + line = (f"Migrated {i:,}/{total:,} endpoints ({pct:.1f}%) — " + f"{rate:.1f} endpoints/sec — ETA {self._fmt_duration(remaining)}") + if queries_this_chunk is not None: + # Per-endpoint query count for this chunk window only. + chunk_size = self.progress_every + line += f" — {queries_this_chunk / chunk_size:.1f} queries/endpoint" + self.stdout.write(self.style.SUCCESS(line)) + + if self.benchmark: + parts = [f"{p}={self.timings[p]:.1f}s" for p in PHASES] + self.stdout.write(" " + " ".join(parts)) + + def _print_benchmark_summary(self, total_endpoints: int, total_seconds: float) -> None: + if not self.benchmark: + return + total_phase = sum(self.timings.values()) or 1.0 + self.stdout.write(self.style.SUCCESS("=== Benchmark summary ===")) + self.stdout.write(f"{'phase':<16}{'total_s':>10}{'per_endpoint_ms':>18}{'share':>10}") + for phase in PHASES: + t = self.timings[phase] + per = (t * 1000.0 / total_endpoints) if total_endpoints else 0.0 + share = (t / total_phase * 100.0) + self.stdout.write(f"{phase:<16}{t:>10.2f}{per:>18.2f}{share:>9.1f}%") + self.stdout.write(f"{'(wall-clock)':<16}{total_seconds:>10.2f}" + f"{(total_seconds * 1000.0 / total_endpoints if total_endpoints else 0):>18.2f}" + f"{'100.0%':>10}") + + # -- Post-migration tag inheritance -------------------------------------- + + def _run_tag_inheritance(self) -> None: + """ + Drive `LocationManager._bulk_inherit_tags` once per contributing product. + + Each `LocationManager` call is wrapped in its own try/except so a + failure on one product group doesn't prevent the rest from running — + same philosophy as the per-endpoint loop. Tag inheritance is a + purely additive post-pass; the underlying location/reference rows + are already committed by the main loop, so partial failure here + leaves a consistent (if incomplete-inheritance) state that a + targeted re-run can finish. + """ + if not self.locations_by_product_id: + return + + # Lazy import: dojo.importers.* pulls in a lot of modules and we + # don't want it loaded at management-command discovery time. + from dojo.importers.location_manager import LocationManager # noqa: PLC0415 + + t0 = time.time() + n_products = len(self.locations_by_product_id) + n_pairs = sum(len(loc_ids) for loc_ids in self.locations_by_product_id.values()) + n_unique_locations = len(self.location_obj_by_id) + n_failures = 0 + for prod_id, loc_ids in self.locations_by_product_id.items(): + product = self.product_obj_by_id[prod_id] + locations = [self.location_obj_by_id[lid] for lid in loc_ids] + try: + LocationManager(product)._bulk_inherit_tags(locations) + except Exception: + logger.exception( + "Tag inheritance pass failed for product id=%s " + "(%d location(s)); continuing with remaining products", + prod_id, len(locations), + ) + n_failures += 1 + elapsed = time.time() - t0 + msg = ( + f"Tag inheritance pass: visited {n_pairs:,} (product, location) pair(s) " + f"across {n_products:,} product(s), {n_unique_locations:,} unique location(s), " + f"in {elapsed:.2f}s" + ) + if n_failures: + self.stdout.write(self.style.WARNING(f"{msg} — {n_failures} product group(s) failed")) + else: + self.stdout.write(self.style.SUCCESS(msg)) + + # -- handle --------------------------------------------------------------- def handle(self, *args, **options): + self.benchmark = bool(options.get("benchmark")) + self.query_count = bool(options.get("query_count")) + self.batch_size = int(options["batch_size"]) + self.progress_every = int(options["progress_every"]) + + # Per-phase wall-clock accumulators. + self.timings = dict.fromkeys(PHASES, 0.0) + self.counts = dict.fromkeys(PHASES, 0) + + # Bookkeeping for the post-migration tag inheritance pass. + # `locations_by_product_id` maps product.id -> set of location.ids + # contributed by that product (via endpoint.product OR finding.test. + # engagement.product). We hold the Product/Location objects in + # parallel maps so the post-pass can hand them directly to + # `LocationManager(product)._bulk_inherit_tags(locations)` without + # extra DB lookups. + self.locations_by_product_id: dict[int, set[int]] = defaultdict(set) + self.product_obj_by_id: dict[int, Product] = {} + self.location_obj_by_id: dict[int, Location] = {} + + # Collected per-endpoint failures so a single bad row doesn't abort + # a multi-hour migration. Each entry is (endpoint_id, exception_str). + self.failed_endpoints: list[tuple[int | None, str]] = [] + + if self.query_count: + connection.force_debug_cursor = True + queries_at_chunk_start = len(connection.queries) + else: + queries_at_chunk_start = 0 # unused + # Allow endpoints to work with V3/Locations enabled with Endpoint.allow_endpoint_init(): - # Progress counter - i = 0 - # Start off with the endpoint objects - it should contain everything we need - queryset = Endpoint.objects.all() + # Prefetch everything the per-endpoint loop will touch so the + # iterator doesn't trigger N+1 joins: + # - `product` is select_related so we don't lazy-load it for the + # no-findings branch + # - `tags` and `endpoint_meta` are prefetched managers + # - `status_endpoint` is prefetched together with the FK chain + # `finding -> test -> engagement -> product` and `mitigated_by` + # so `associate_with_finding` can read them without queries. + queryset = ( + Endpoint.objects.all() + .select_related("product") + .prefetch_related( + "tags", + "endpoint_meta", + Prefetch( + "status_endpoint", + queryset=Endpoint_Status.objects.select_related( + "finding__test__engagement__product", + "mitigated_by", + ), + ), + ) + ) # Grab the total count so we can communicate progress endpoint_count = queryset.count() + self.stdout.write(self.style.WARNING( + f"Starting migration of {endpoint_count:,} endpoints " + f"(batch={self.batch_size}, progress every {self.progress_every}, " + f"benchmark={'on' if self.benchmark else 'off'}, " + f"query-count={'on' if self.query_count else 'off'})", + )) + run_t0 = time.time() + i = 0 # Process each endpoint - for i, endpoint in enumerate(queryset.iterator(chunk_size=CHUNK_SIZE), 1): - # Progress report every chunk - if not i % CHUNK_SIZE: - self.stdout.write( - self.style.SUCCESS( - f"Migrated {i}/{endpoint_count} endpoints...", - ), - ) - # Get the URL object first - location = self._endpoint_to_url(endpoint) - # Associate the URL with the findings associated with the Findings - # the association to a finding will also apply to a product automatically - self._associate_location_with_findings(endpoint, location) - self.stdout.write( - self.style.SUCCESS( - f"Migrated {i} total endpoints.", - ), - ) + for i, endpoint in enumerate(queryset.iterator(chunk_size=self.batch_size), 1): + t_fetch = self._bench_start() + # iterator already produced `endpoint`; bill nothing meaningful + # to fetch_endpoint here — kept as a placeholder that B1's + # prefetch will start incrementing. + self._bench_end("fetch_endpoint", t_fetch) + + # Wrap the per-endpoint work so one failure doesn't abort a + # multi-hour migration. We log the full traceback and record + # the endpoint id, then continue. The bulk_create-based hot + # path makes partial-state on failure unlikely (each phase + # is its own bulk insert), and any rows that DID land remain + # valid and idempotent on re-run. + try: + # Get the URL object first + location = self._endpoint_to_url(endpoint) + # Track the endpoint's own product as a contributor for the + # post-migration tag inheritance pass (the no-findings + # branch of _associate_location_with_findings also depends + # on this product, and it won't be tracked otherwise). + if endpoint.product_id: + self._track_product_location(endpoint.product, location) + # Associate the URL with the findings associated with the Findings + # the association to a finding will also apply to a product automatically + self._associate_location_with_findings(endpoint, location) + except Exception as exc: + endpoint_id = getattr(endpoint, "id", None) + logger.exception("Failed to migrate endpoint id=%s; continuing", endpoint_id) + self.failed_endpoints.append((endpoint_id, str(exc))) + continue + + # Progress report every --progress-every endpoints + if i % self.progress_every == 0: + queries_in_chunk = None + if self.query_count: + queries_in_chunk = len(connection.queries) - queries_at_chunk_start + # Trim the query log so memory doesn't balloon on long runs; + # after clear() the next chunk's baseline is 0. + connection.queries_log.clear() + queries_at_chunk_start = 0 + self._log_progress(i, endpoint_count, run_t0, queries_in_chunk) + + elapsed = time.time() - run_t0 + successful = i - len(self.failed_endpoints) + self.stdout.write(self.style.SUCCESS( + f"Done. Migrated {successful:,}/{i:,} endpoints in {self._fmt_duration(elapsed)} " + f"({(i / elapsed if elapsed else 0):.2f} endpoints/sec).", + )) + if self.failed_endpoints: + preview_ids = [eid for eid, _ in self.failed_endpoints[:10]] + self.stdout.write(self.style.WARNING( + f"{len(self.failed_endpoints):,} endpoint(s) failed; see logger output above " + f"for tracebacks. First failing endpoint IDs: {preview_ids}", + )) + + # Run the post-migration tag inheritance pass. `bulk_create` skips + # the `inherit_tags_on_linked_instance` post_save signal, so for + # deployments with `enable_product_tag_inheritance` enabled (per + # product or system-wide) the migrated Locations would otherwise + # not pick up inherited product tags. We grouped (product, + # location) pairs during the main loop and now drive + # `LocationManager._bulk_inherit_tags` once per contributing + # product. The helper rediscovers each location's full product + # set via LocationProductReference/LocationFindingReference and + # diff-checks before writing, so revisits of shared locations + # across product groups are idempotent. + self._run_tag_inheritance() + + self._print_benchmark_summary(i, elapsed) + + if self.query_count: + connection.force_debug_cursor = False diff --git a/dojo/reports/queries.py b/dojo/reports/queries.py index 174427ca5c6..46896b8981c 100644 --- a/dojo/reports/queries.py +++ b/dojo/reports/queries.py @@ -24,7 +24,7 @@ def prefetch_related_findings_for_report(findings: QuerySet) -> QuerySet: ) -def prefetch_related_endpoints_for_report(endpoints: QuerySet) -> QuerySet: +def prefetch_related_endpoints_for_report(endpoints: QuerySet, product=None) -> QuerySet: if settings.V3_FEATURE_LOCATIONS: return annotate_location_counts_and_status( endpoints.prefetch_related( @@ -39,23 +39,24 @@ def prefetch_related_endpoints_for_report(endpoints: QuerySet) -> QuerySet: ), ) # TODO: Delete this after the move to Locations + findings_qs = Finding.objects.filter( + active=True, + out_of_scope=False, + mitigated__isnull=True, + false_p=False, + duplicate=False, + status_finding__false_positive=False, + status_finding__out_of_scope=False, + status_finding__risk_accepted=False, + ) + if product is not None: + findings_qs = findings_qs.filter(test__engagement__product=product) return endpoints.prefetch_related( "product", "tags", Prefetch( "findings", - queryset=prefetch_for_findings( - Finding.objects.filter( - active=True, - out_of_scope=False, - mitigated__isnull=True, - false_p=False, - duplicate=False, - status_finding__false_positive=False, - status_finding__out_of_scope=False, - status_finding__risk_accepted=False, - ).order_by("numerical_severity"), - ), + queryset=prefetch_for_findings(findings_qs.order_by("numerical_severity")), to_attr="active_annotated_findings", ), ) diff --git a/dojo/reports/views.py b/dojo/reports/views.py index 1cce1c9d884..6b372ad8df2 100644 --- a/dojo/reports/views.py +++ b/dojo/reports/views.py @@ -15,6 +15,8 @@ from openpyxl.styles import Font from dojo.authorization.authorization import user_has_permission_or_403 +from dojo.authorization.roles_permissions import Permissions +from dojo.endpoint.queries import get_authorized_endpoints from dojo.filters import ( EndpointFilter, EndpointFilterWithoutObjectLookups, @@ -27,6 +29,7 @@ from dojo.forms import ReportOptionsForm from dojo.labels import get_labels from dojo.location.models import Location +from dojo.location.queries import get_authorized_locations from dojo.location.status import FindingLocationStatus from dojo.models import Dojo_User, Endpoint, Engagement, Finding, Product, Product_Type, Test from dojo.reports.queries import prefetch_related_endpoints_for_report, prefetch_related_findings_for_report @@ -185,7 +188,7 @@ def get_context(self): def report_findings(request): - findings = Finding.objects.filter() + findings = get_authorized_findings(Permissions.Finding_View) filter_string_matching = get_system_setting("filter_string_matching", False) filter_class = ReportFindingFilterWithoutObjectLookups if filter_string_matching else ReportFindingFilter findings = filter_class(request.GET, queryset=findings) @@ -208,11 +211,12 @@ def report_findings(request): def report_endpoints(request): if settings.V3_FEATURE_LOCATIONS: - endpoints = Location.objects.filter(findings__status=FindingLocationStatus.Active).distinct() + endpoints = get_authorized_locations(Permissions.Location_View) + endpoints = endpoints.filter(findings__status=FindingLocationStatus.Active).distinct() endpoints = URLFilter(request.GET, queryset=endpoints) else: # TODO: Delete this after the move to Locations - endpoints = Endpoint.objects.filter( + endpoints = get_authorized_endpoints(Permissions.Location_View).filter( finding__active=True, finding__false_p=False, finding__duplicate=False, @@ -280,13 +284,14 @@ def product_endpoint_report(request, pid): endpoints = URLFilter(request.GET, queryset=endpoints) else: # TODO: Delete this after the move to Locations - endpoints = Endpoint.objects.filter(finding__active=True, + endpoints = Endpoint.objects.filter(product=product, + finding__active=True, finding__false_p=False, finding__duplicate=False, finding__out_of_scope=False) if get_system_setting("enforce_verified_status", True) or get_system_setting("enforce_verified_status_metrics", True): endpoints = endpoints.filter(finding__active=True) - endpoints = prefetch_related_endpoints_for_report(endpoints.distinct()) + endpoints = prefetch_related_endpoints_for_report(endpoints.distinct(), product=product) endpoints = EndpointReportFilter(request.GET, queryset=endpoints) paged_endpoints = get_page_items(request, endpoints.qs, 25) diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml index 40ef42827f9..1ac987564db 100644 --- a/helm/defectdojo/Chart.yaml +++ b/helm/defectdojo/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: "2.59.0-dev" description: A Helm chart for Kubernetes to install DefectDojo name: defectdojo -version: 1.9.28-dev +version: 1.9.29-dev icon: https://defectdojo.com/hubfs/DefectDojo_favicon.png maintainers: - name: madchap diff --git a/helm/defectdojo/README.md b/helm/defectdojo/README.md index 799542d7f4f..758cc2c65ca 100644 --- a/helm/defectdojo/README.md +++ b/helm/defectdojo/README.md @@ -511,7 +511,7 @@ The HELM schema will be generated for you. # General information about chart values -![Version: 1.9.28-dev](https://img.shields.io/badge/Version-1.9.28--dev-informational?style=flat-square) ![AppVersion: 2.59.0-dev](https://img.shields.io/badge/AppVersion-2.59.0--dev-informational?style=flat-square) +![Version: 1.9.29-dev](https://img.shields.io/badge/Version-1.9.29--dev-informational?style=flat-square) ![AppVersion: 2.59.0-dev](https://img.shields.io/badge/AppVersion-2.59.0--dev-informational?style=flat-square) A Helm chart for Kubernetes to install DefectDojo diff --git a/requirements.txt b/requirements.txt index 8dea1b4c159..47e3ec1f14d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ python-dateutil==2.9.0.post0 redis==7.4.0 requests==2.34.0 sqlalchemy==2.0.49 # Required by Celery broker transport -urllib3==2.6.3 +urllib3==2.7.0 uWSGI==2.0.31 vobject==0.9.9 whitenoise==5.2.0 diff --git a/tests/import_scanner_test.py b/tests/import_scanner_test.py deleted file mode 100644 index d429838dbc6..00000000000 --- a/tests/import_scanner_test.py +++ /dev/null @@ -1,244 +0,0 @@ -import logging -import os -import re -import shutil -import sys -import unittest -from pathlib import Path - -import git -from base_test_class import BaseTestCase -from product_test import ProductTest -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import Select - -dir_path = Path(os.path.realpath(__file__)).parent - -logger = logging.getLogger(__name__) - - -class ScannerTest(BaseTestCase): - def setUp(self): - super().setUp(self) - self.repo_path = dir_path / "scans" - if self.repo_path.is_dir(): - shutil.rmtree(self.repo_path) - self.repo_path.mkdir() - git.Repo.clone_from("https://github.com/DefectDojo/sample-scan-files", self.repo_path) - self.remove_items = ["__init__.py", "__init__.pyc", "factory.py", "factory.pyc", - "factory.py", "LICENSE", "README.md", ".gitignore", ".git", "__pycache__"] - tool_path = dir_path.parent / "dojo" / "tools" - tools = sorted(any(tool_path.iterdir())) - p = self.repo_path - tests = sorted(any(p.iterdir())) - self.tools = [i for i in tools if i not in self.remove_items] - self.tests = [i for i in tests if i not in self.remove_items] - - def test_check_test_file(self): - missing_tests = ["MISSING TEST FOLDER"] - for tool in self.tools: - if tool not in self.tests: - missing_tests += [tool] - - missing_tests += ["\nNO TEST FILES"] - - for test in self.tests: - p = self.repo_path / test - cases = sorted(any(p.iterdir())) - cases = [i for i in cases if i not in self.remove_items] - if len(cases) == 0 and tool not in missing_tests: - missing_tests += [test] - - if len(missing_tests) > 0: - logger.info("The following scanners are missing test cases or incorrectly named") - logger.info("Names must match those listed in /dojo/tools") - logger.info("Test cases can be added/modified here:") - logger.info("https://github.com/DefectDojo/sample-scan-files\n") - for test in missing_tests: - logger.info(test) - self.assertEqual(len(missing_tests), 0) - - def test_check_for_forms(self): - forms_path = dir_path.parent / "dojo" / "forms.py" - file = forms_path.open("r+", encoding="utf-8") - forms = file.readlines() - file.close() - - forms = [form.strip().lower() for form in forms] - forms = forms[forms.index('scan_type_choices = (("", "please select a scan type"),') + 1: - forms.index("sorted_scan_type_choices = sorted(scan_type_choices, key=lambda x: x[1])") - 1] - forms = [form.replace("(", "").replace(")", "").replace("-", " ").replace('"', "").replace(".", "") for form in forms] - forms = [form[:form.index(",")] for form in forms] - remove_patterns = [" scanner", " scan"] - for pattern in remove_patterns: - forms = [re.sub(pattern, "", fix) for fix in sorted(forms)] - - acronyms = [] - for words in forms: - acronyms += ["".join(word[0] for word in words.split())] - - missing_forms = [] - for tool in self.tools: - reg = re.compile(tool.replace("_", " ")) - matches = list(filter(reg.search, forms)) + list(filter(reg.search, acronyms)) - matches = [m.strip() for m in matches] - if len(matches) != 1: - if tool not in matches: - missing_forms += [tool] - - if len(missing_forms) > 0: - logger.info("The following scanners are missing forms") - logger.info("Names must match those listed in /dojo/tools") - logger.info("forms can be added here:") - logger.info("https://github.com/DefectDojo/django-DefectDojo/blob/master/dojo/forms.py\n") - for tool in missing_forms: - logger.info(tool) - self.assertEqual(len(missing_forms), 0) - - @unittest.skip("Deprecated since Dynamic Parser infrastructure") - def test_check_for_options(self): - template_path = dir_path.parent / "dojo" / "templates" / "dojo" / "import_scan_results.html" - file = template_path.open("r+", encoding="utf-8") - templates = file.readlines() - file.close() - - templates = [temp.strip().lower() for temp in templates] - templates = templates[templates.index("")] - remove_patterns = ["
  • ", "", "
    • ", " scanner", " scan"] - for pattern in remove_patterns: - templates = [re.sub(pattern, "", temp) for temp in templates] - - templates = [temp[:temp.index(" - ")] for temp in sorted(templates) if " - " in temp] - templates = [temp.replace("-", " ").replace(".", "").replace("(", "").replace(")", "") for temp in templates] - - acronyms = [] - for words in templates: - acronyms += ["".join(word[0] for word in words.split())] - - missing_templates = [] - for tool in self.tools: - temp_tool = tool.replace("_", " ") - reg = re.compile(temp_tool) - matches = list(filter(reg.search, templates)) + list(filter(reg.search, acronyms)) - matches = [m.strip() for m in matches] - if len(matches) == 0: - if temp_tool not in matches: - missing_templates += [tool] - - if len(missing_templates) > 0: - logger.info("The following scanners are missing templates") - logger.info("Names must match those listed in /dojo/tools") - logger.info("templates can be added here:") - logger.info("https://github.com/DefectDojo/django-DefectDojo/blob/master/dojo/templates/dojo/import_scan_results.html\n") - for tool in missing_templates: - logger.info(tool) - self.assertEqual(len(missing_templates), 0) - - def test_engagement_import_scan_result(self): - driver = self.driver - self.goto_product_overview(driver) - driver.find_element(By.CSS_SELECTOR, ".dropdown-toggle.pull-left").click() - driver.find_element(By.LINK_TEXT, "Add New Engagement").click() - driver.find_element(By.ID, "id_name").send_keys("Scan type mapping") - driver.find_element(By.NAME, "_Import Scan Results").click() - options_text = "".join(driver.find_element(By.NAME, "scan_type").text).split("\n") - options_text = [scan.strip() for scan in options_text] - - mod_options = options_text - mod_options = [scan.replace(" Scanner", "") for scan in mod_options] - mod_options = [scan.replace(" Scan", "") for scan in mod_options] - mod_options = [scan.lower().replace("-", " ").replace(".", "") for scan in mod_options] - - acronyms = [] - for scans in mod_options: - acronyms += ["".join(scan[0] for scan in scans.split())] - - potential_matches = mod_options + acronyms - scan_map = {} - - for test in self.tests: - temp_test = test.replace("_", " ").replace("-", " ") - reg = re.compile(".*" + temp_test + ".*") - found_matches = {} - for i in range(len(potential_matches)): - matches = list(filter(reg.search, [potential_matches[i]])) - if len(matches) > 0: - index = i - if i >= len(mod_options): - index = i - len(mod_options) - found_matches[index] = matches[0] - - if len(found_matches) == 1: - index = list(found_matches.keys())[0] - scan_map[test] = options_text[index] - elif len(found_matches) > 1: - index = list(found_matches.values()).index(temp_test) - scan_map[test] = options_text[list(found_matches.keys())[index]] - - failed_tests = [] - for test in self.tests: - p = self.repo_path / test - cases = sorted(any(p.iterdir())) - cases = [i for i in cases if i not in self.remove_items] - if len(cases) == 0: - failed_tests += [test.upper() + ": No test cases"] - for case in cases: - self.goto_product_overview(driver) - driver.find_element(By.CSS_SELECTOR, ".dropdown-toggle.pull-left").click() - driver.find_element(By.LINK_TEXT, "Add New Engagement").click() - driver.find_element(By.ID, "id_name").send_keys(test + " - " + case) - driver.find_element(By.NAME, "_Import Scan Results").click() - try: - driver.find_element(By.ID, "id_active").get_attribute("checked") - driver.find_element(By.ID, "id_verified").get_attribute("checked") - scan_type = scan_map[test] - Select(driver.find_element(By.ID, "id_scan_type")).select_by_visible_text(scan_type) - test_location = self.repo_path / test / case - driver.find_element(By.ID, "id_file").send_keys(str(test_location)) - driver.find_element(By.CSS_SELECTOR, "input.btn.btn-primary").click() - EngagementTXT = "".join(driver.find_element(By.TAG_NAME, "BODY").text).split("\n") - reg = re.compile(r"processed, a total of") - matches = list(filter(reg.search, EngagementTXT)) - if len(matches) != 1: - failed_tests += [test.upper() + " - " + case + ": Not imported"] - except Exception as e: - if e == "Message: timeout": - failed_tests += [test.upper() + " - " + case + ": Not imported due to timeout"] - else: - failed_tests += [test.upper() + ": Cannot auto select scan type"] - break - - if len(failed_tests) > 0: - logger.info("The following scan imports produced errors") - logger.info("Names of tests must match those listed in /dojo/tools") - logger.info("Tests can be added/modified here:") - logger.info("https://github.com/DefectDojo/sample-scan-files\n") - for test in failed_tests: - logger.info(test) - self.assertEqual(len(failed_tests), 0) - - def tearDown(self): - super().tearDown(self) - shutil.rmtree(self.repo_path) - - -def suite(): - suite = unittest.TestSuite() - suite.addTest(BaseTestCase("test_login")) - suite.addTest(BaseTestCase("disable_block_execution")) - suite.addTest(ScannerTest("test_check_test_file")) - suite.addTest(ScannerTest("test_check_for_doc")) - suite.addTest(ScannerTest("test_check_for_forms")) - suite.addTest(ScannerTest("test_check_for_options")) - suite.addTest(ProductTest("test_create_product")) - suite.addTest(ScannerTest("test_engagement_import_scan_result")) - suite.addTest(ProductTest("test_delete_product")) - return suite - - -if __name__ == "__main__": - runner = unittest.TextTestRunner(descriptions=True, failfast=True, verbosity=2) - ret = not runner.run(suite()).wasSuccessful() - BaseTestCase.tearDownDriver() - sys.exit(ret) diff --git a/unittests/test_location_finding_reference_authz.py b/unittests/test_location_finding_reference_authz.py new file mode 100644 index 00000000000..158ac0fb907 --- /dev/null +++ b/unittests/test_location_finding_reference_authz.py @@ -0,0 +1,151 @@ +from django.utils.timezone import now + +from dojo.authorization.roles_permissions import Permissions, Roles +from dojo.location.models import Location, LocationFindingReference, LocationProductReference +from dojo.location.queries import get_authorized_location_finding_reference +from dojo.location.status import FindingLocationStatus, ProductLocationStatus +from dojo.models import ( + Dojo_User, + Engagement, + Finding, + Product, + Product_Member, + Product_Type, + Role, + Test, + Test_Type, + User, +) +from unittests.dojo_test_case import DojoTestCase, versioned_fixtures + + +@versioned_fixtures +class TestLocationFindingReferenceAuthorization(DojoTestCase): + + """ + `get_authorized_location_finding_reference` was anchoring authorization to + Location.products (the set of products associated with the location). + When two products share a location, a Reader on Product A could read + LocationFindingReference rows for findings that belong to Product B. + + Authorization must be anchored to the finding's own product + (finding.test.engagement.product), so this test sets up a shared location + and verifies each Reader only sees their product's references. + """ + + fixtures = ["dojo_testdata.json"] + + @classmethod + def setUpTestData(cls): + prod_type, _ = Product_Type.objects.get_or_create(name="LocFRef PT") + test_type, _ = Test_Type.objects.get_or_create(name="LocFRef Scan") + reader_role = Role.objects.get(id=Roles.Reader) + + cls.product_a = Product.objects.create( + name="LocFRef Product A", + description="A", + prod_type=prod_type, + ) + cls.product_b = Product.objects.create( + name="LocFRef Product B", + description="B", + prod_type=prod_type, + ) + + cls.alice = User.objects.create_user( + username="locfref_alice", + password="not-a-real-secret", # noqa: S106 - test fixture user + ) + cls.bob = User.objects.create_user( + username="locfref_bob", + password="not-a-real-secret", # noqa: S106 - test fixture user + ) + Product_Member.objects.create(user=cls.alice, product=cls.product_a, role=reader_role) + Product_Member.objects.create(user=cls.bob, product=cls.product_b, role=reader_role) + # Legacy authorization collapses Reader/Writer/Maintainer/Owner into + # a single ``authorized_users`` membership; mirror the RBAC rows so + # the users are visible to ``get_authorized_*`` queries. + cls.product_a.authorized_users.add(Dojo_User.objects.get(pk=cls.alice.pk)) + cls.product_b.authorized_users.add(Dojo_User.objects.get(pk=cls.bob.pk)) + + cls.finding_a = cls._make_finding(cls.product_a, test_type, title="Finding A") + cls.finding_b = cls._make_finding(cls.product_b, test_type, title="Finding B") + + # Shared location across both products. + cls.shared_location = Location.objects.create( + location_type="URL", + location_value="https://shared.example.com/", + ) + LocationProductReference.objects.create( + location=cls.shared_location, + product=cls.product_a, + status=ProductLocationStatus.Active, + ) + LocationProductReference.objects.create( + location=cls.shared_location, + product=cls.product_b, + status=ProductLocationStatus.Active, + ) + cls.ref_a = LocationFindingReference.objects.create( + location=cls.shared_location, + finding=cls.finding_a, + status=FindingLocationStatus.Active, + ) + cls.ref_b = LocationFindingReference.objects.create( + location=cls.shared_location, + finding=cls.finding_b, + status=FindingLocationStatus.Active, + ) + + @classmethod + def _make_finding(cls, product, test_type, *, title): + engagement = Engagement.objects.create( + name=f"{product.name} Engagement", + product=product, + target_start=now(), + target_end=now(), + ) + test = Test.objects.create( + engagement=engagement, + test_type=test_type, + title=f"{product.name} Test", + target_start=now(), + target_end=now(), + ) + return Finding.objects.create( + test=test, + title=title, + description=title, + severity="High", + numerical_severity="S0", + active=True, + verified=True, + ) + + def test_alice_sees_only_product_a_finding_references(self): + results = list( + get_authorized_location_finding_reference( + Permissions.Location_View, user=self.alice, + ).filter(location=self.shared_location), + ) + result_ids = {ref.id for ref in results} + self.assertEqual(result_ids, {self.ref_a.id}) + + def test_bob_sees_only_product_b_finding_references(self): + results = list( + get_authorized_location_finding_reference( + Permissions.Location_View, user=self.bob, + ).filter(location=self.shared_location), + ) + result_ids = {ref.id for ref in results} + self.assertEqual(result_ids, {self.ref_b.id}) + + def test_superuser_sees_both_finding_references(self): + admin = User.objects.get(username="admin") + results = list( + get_authorized_location_finding_reference( + Permissions.Location_View, user=admin, + ).filter(location=self.shared_location), + ) + result_ids = {ref.id for ref in results} + self.assertEqual(result_ids, {self.ref_a.id, self.ref_b.id}) diff --git a/unittests/test_permissions_audit.py b/unittests/test_permissions_audit.py index cb174bbd749..fe7a31c24eb 100644 --- a/unittests/test_permissions_audit.py +++ b/unittests/test_permissions_audit.py @@ -1641,18 +1641,16 @@ def test_finding_metadata_reader_allowed(self): self.assertEqual(response.status_code, 200, response.content) # ── Finding: duplicate status actions ────────────────────────────── - # NOTE: reset_finding_duplicate_status and set_finding_as_original - # bypass self.get_object(), so DRF object-level permission checks - # (has_object_permission) never fire. The internal helpers do their - # own permission checks. These tests verify current behaviour. + # reset_finding_duplicate_status and set_finding_as_original call + # self.get_object() so DRF's object-level permission check runs via + # UserHasFindingRelatedObjectPermission (POST -> Finding_Edit). - def test_finding_reset_duplicate_reader(self): - """View bypasses get_object() — internal helper checks permissions.""" + def test_finding_reset_duplicate_reader_denied(self): + """Reader lacks Finding_Edit — POST must be denied before the helper runs.""" client = self._client_for_user(self.reader_user) url = reverse("finding-reset-finding-duplicate-status", args=(self.finding.id,)) response = client.post(url, format="json") - # Returns 400 (not a duplicate) — internal helper runs before perm check - self.assertEqual(response.status_code, 400, response.content) + self.assertEqual(response.status_code, 403, response.content) def test_finding_reset_duplicate_writer(self): client = self._client_for_user(self.writer_user) diff --git a/unittests/test_product_endpoint_report_scoping.py b/unittests/test_product_endpoint_report_scoping.py new file mode 100644 index 00000000000..ccfe17a01e9 --- /dev/null +++ b/unittests/test_product_endpoint_report_scoping.py @@ -0,0 +1,169 @@ +from django.test import Client +from django.utils.timezone import now + +from dojo.authorization.roles_permissions import Roles +from dojo.models import ( + Dojo_User, + Endpoint, + Endpoint_Status, + Engagement, + Finding, + Product, + Product_Member, + Product_Type, + Role, + Test, + Test_Type, + User, +) +from unittests.dojo_test_case import DojoTestCase, skip_unless_v2 + + +@skip_unless_v2 +class TestProductEndpointReportScoping(DojoTestCase): + + """ + The legacy `product_endpoint_report` view must only return endpoints and + findings belonging to the requested product. Previously the Endpoint + queryset was filtered by finding flags only and not scoped by product, + so an unrelated product's findings appeared in the report. + """ + + fixtures = ["dojo_testdata.json"] + + MARKER_A = "PRODUCT_A_UNIQUE_MARKER_b3c8aa1f" + MARKER_B = "PRODUCT_B_UNIQUE_MARKER_d9e2bc54" + + @classmethod + def setUpTestData(cls): + cls.user = User.objects.get(username="admin") + cls.prod_type, _ = Product_Type.objects.get_or_create(name="Scoping Test PT") + cls.test_type, _ = Test_Type.objects.get_or_create(name="Scoping Test Scan") + + cls.product_a = Product.objects.create( + name="Scoping Test Product A", + description=cls.MARKER_A, + prod_type=cls.prod_type, + ) + cls.product_b = Product.objects.create( + name="Scoping Test Product B", + description=cls.MARKER_B, + prod_type=cls.prod_type, + ) + + cls.finding_a = cls._create_finding_with_endpoint( + cls.product_a, "Finding for A", cls.MARKER_A, host="a.example.com", + ) + cls.finding_b = cls._create_finding_with_endpoint( + cls.product_b, "Finding for B", cls.MARKER_B, host="b.example.com", + ) + + cls.restricted_user = User.objects.create_user( + username="report_scoping_reader", + password="not-a-real-secret", # noqa: S106 - test fixture user + ) + reader_role = Role.objects.get(id=Roles.Reader) + Product_Member.objects.create( + user=cls.restricted_user, + product=cls.product_a, + role=reader_role, + ) + # Legacy authorization collapses Reader/Writer/Maintainer/Owner into + # a single ``authorized_users`` membership; mirror the RBAC row so + # the user is visible to ``get_authorized_*`` queries. + cls.product_a.authorized_users.add(Dojo_User.objects.get(pk=cls.restricted_user.pk)) + + @classmethod + def _create_finding_with_endpoint(cls, product, title, description, *, host): + engagement = Engagement.objects.create( + name=f"{product.name} Engagement", + product=product, + target_start=now(), + target_end=now(), + ) + test = Test.objects.create( + engagement=engagement, + test_type=cls.test_type, + title=f"{product.name} Test", + target_start=now(), + target_end=now(), + ) + finding = Finding.objects.create( + test=test, + title=title, + description=description, + severity="High", + numerical_severity="S0", + active=True, + verified=True, + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated=None, + reporter=cls.user, + ) + endpoint = Endpoint.objects.create( + host=host, + protocol="https", + product=product, + ) + Endpoint_Status.objects.create( + endpoint=endpoint, + finding=finding, + mitigated=False, + false_positive=False, + out_of_scope=False, + risk_accepted=False, + ) + finding.endpoints.add(endpoint) + return finding + + def setUp(self): + super().setUp() + self.client = Client() + self.client.force_login(self.user) + + def test_product_endpoint_report_only_includes_target_product_findings(self): + url = f"/product/{self.product_a.id}/endpoint/report?_generate=1&report_type=HTML" + response = self.client.get(url) + self.assertEqual(response.status_code, 200, response.content[:500]) + body = response.content.decode() + + self.assertIn(self.MARKER_A, body, "Expected Product A's finding description in report") + self.assertNotIn( + self.MARKER_B, + body, + "Product B's finding description must not appear in Product A's report", + ) + + def test_product_b_report_only_includes_product_b_findings(self): + url = f"/product/{self.product_b.id}/endpoint/report?_generate=1&report_type=HTML" + response = self.client.get(url) + self.assertEqual(response.status_code, 200, response.content[:500]) + body = response.content.decode() + + self.assertIn(self.MARKER_B, body) + self.assertNotIn(self.MARKER_A, body) + + def test_reports_findings_only_includes_user_authorized_findings(self): + # /reports/findings was previously unscoped; a Reader on Product A should + # see Product A's findings only. The template renders the title, not + # the description, so we assert against the unique titles. + restricted_client = Client() + restricted_client.force_login(self.restricted_user) + response = restricted_client.get("/reports/findings") + self.assertEqual(response.status_code, 200, response.content[:500]) + body = response.content.decode() + self.assertIn("Finding for A", body) + self.assertNotIn("Finding for B", body) + + def test_reports_endpoints_only_includes_user_authorized_endpoints(self): + # /reports/endpoints was previously unscoped; a Reader on Product A + # should only see Product A's endpoints. + restricted_client = Client() + restricted_client.force_login(self.restricted_user) + response = restricted_client.get("/reports/endpoints") + self.assertEqual(response.status_code, 200, response.content[:500]) + body = response.content.decode() + self.assertIn("a.example.com", body) + self.assertNotIn("b.example.com", body) diff --git a/unittests/test_rest_framework.py b/unittests/test_rest_framework.py index d1427f007f4..4455dba1374 100644 --- a/unittests/test_rest_framework.py +++ b/unittests/test_rest_framework.py @@ -1827,6 +1827,70 @@ def test_post_without_finding_returns_4xx(self): self.assertLess(response.status_code, 500) +@versioned_fixtures +class FindingActionAuthzTest(DojoAPITestCase): + + fixtures = ["dojo_testdata.json"] + + def _client_for(self, username): + user = User.objects.get(username=username) + token = Token.objects.get(user=user) + client = APIClient() + client.credentials(HTTP_AUTHORIZATION="Token " + token.key) + return client + + def test_admin_can_reset_finding_duplicate_status(self): + client = self._client_for("admin") + # Mark finding 2 as a duplicate of finding 3 first, then reset. + set_response = client.post("/api/v2/findings/2/original/3/") + self.assertEqual(set_response.status_code, status.HTTP_204_NO_CONTENT, set_response.content[:500]) + reset_response = client.post("/api/v2/findings/2/duplicate/reset/") + self.assertEqual(reset_response.status_code, status.HTTP_204_NO_CONTENT, reset_response.content[:500]) + refreshed = Finding.objects.get(pk=2) + self.assertFalse(refreshed.duplicate) + self.assertIsNone(refreshed.duplicate_finding) + + def test_admin_can_set_finding_as_original(self): + client = self._client_for("admin") + response = client.post("/api/v2/findings/2/original/3/") + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT, response.content[:500]) + refreshed = Finding.objects.get(pk=2) + self.assertTrue(refreshed.duplicate) + self.assertEqual(refreshed.duplicate_finding_id, 3) + + def test_unrelated_user_cannot_reset_finding_duplicate_status(self): + client = self._client_for("user2") + # Sanity: finding 7 is not visible to this user. + self.assertEqual(client.get("/api/v2/findings/7/").status_code, 404) + + before = Finding.objects.get(pk=7) + before_duplicate = before.duplicate + before_duplicate_finding_id = before.duplicate_finding_id + + response = client.post("/api/v2/findings/7/duplicate/reset/") + self.assertIn(response.status_code, (403, 404), response.content[:500]) + + after = Finding.objects.get(pk=7) + self.assertEqual(after.duplicate, before_duplicate) + self.assertEqual(after.duplicate_finding_id, before_duplicate_finding_id) + + def test_unrelated_user_cannot_set_finding_as_original(self): + client = self._client_for("user2") + # Sanity: finding 7 is not visible to this user. + self.assertEqual(client.get("/api/v2/findings/7/").status_code, 404) + + before = Finding.objects.get(pk=7) + before_duplicate = before.duplicate + before_duplicate_finding_id = before.duplicate_finding_id + + response = client.post("/api/v2/findings/7/original/2/") + self.assertIn(response.status_code, (403, 404), response.content[:500]) + + after = Finding.objects.get(pk=7) + self.assertEqual(after.duplicate, before_duplicate) + self.assertEqual(after.duplicate_finding_id, before_duplicate_finding_id) + + @versioned_fixtures class FilesTest(DojoAPITestCase): fixtures = ["dojo_testdata.json"]