diff --git a/.gitignore b/.gitignore index 0819916..5ef5b80 100644 --- a/.gitignore +++ b/.gitignore @@ -5,14 +5,16 @@ site/ addons/ collectors/ +# Notebooks/output notebooks - output graph logs dbt_packages -.vscode +# Ignore editor settings +.vscode +.idea # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/descriptions/faker/edges/RAND_ExampleReference.md b/descriptions/faker/edges/RAND_ExampleReference.md new file mode 100644 index 0000000..b9f6018 --- /dev/null +++ b/descriptions/faker/edges/RAND_ExampleReference.md @@ -0,0 +1,4 @@ +# Description + +This is an example description for the RAND_ExampleReference node. This will page will be embedded/combined with the +edge description when the pipeline generates automated documentation. \ No newline at end of file diff --git a/descriptions/faker/nodes/RAND_Computer.md b/descriptions/faker/nodes/RAND_Computer.md new file mode 100644 index 0000000..10d85b7 --- /dev/null +++ b/descriptions/faker/nodes/RAND_Computer.md @@ -0,0 +1,4 @@ +# Description + +This is an example description for the RAND_Computer node. This will page will be embedded/combined with the node +description when the pipeline generates automated documentation. \ No newline at end of file diff --git a/docs/.nav.yml b/docs/.nav.yml new file mode 100644 index 0000000..ce20ebd --- /dev/null +++ b/docs/.nav.yml @@ -0,0 +1,14 @@ +nav: + - index.md + - getting-started.md + - cli.md + - logging.md + - Development: + - "development/creating-collector.md" + - "development/collection.md" + - "development/modelling.md" + - "development/graph.md" + - API: + - "api/*" + - Sources: + - "sources/*" diff --git a/docs/development/collection.md b/docs/development/collection.md new file mode 100644 index 0000000..8ba3517 --- /dev/null +++ b/docs/development/collection.md @@ -0,0 +1,90 @@ +# Collecting resources +This page explains how OpenHound uses `dlt` sources and resources to collect and transform data. A DLT source groups resources and declares configuration items (like secrets), while a resource yield assets (eg. users/computers/roles etc) that are stored in JSONL/Parquet format as part of the pipeline. If you followed the [Creating a new collector](creating-collector.md) process, the cookiecutter template already generates an example source and resource for you that you can modify. + +## DLT source +The `@app.source` function is the starting point of a collector. It receives/parses configuration, builds shared clients or context and returns DLT resources or DLT transformers. The source decides which resources are part of the collection. By default, running the pipeline (ie. using the `collect` cli command) will collect all resources returned by this function, though a subset of resources can be selected as part of the CLI options. + +The source is defined in `source.py` (SuperIAM is the sample name for a custom service): + +```py +@app.source(name="SuperIAM", max_table_nesting=0) +def source(token=dlt.secrets.value, host=dlt.secrets.value): + ctx = SourceContext( + client=RESTClient( + base_url=host, + headers={"accept": "application/json"}, + auth=BearerTokenAuth(token=token), + paginator=SinglePagePaginator(), + ) + ) + + return (users(ctx),) +``` + +**Key points:** + +- `token` and `host` are read from `dlt` secrets so they are not hard-coded. These can either be read from your environment or stored inside a .dlt/secrets.toml and/or dlt/config.toml config file. +- A shared `SourceContext` containing a global request/Rest client is created and passed to all resources. +- The source returns a tuple of resources (in this case, only: `users`). + + +## DLT resource +The `@app.resource` function is a wrapper for a DLT resource (@dlt.resource) and yields assets (eg. users/computers/roles etc) to store on disk. In this case, we use DLT's included RESTClient, which automatically handles pagination, retries and rate-limiting. Compared to the original dlt resource, additional exception handling strategies can be added to continue the pipeline in case a single resource fails. + +The template includes a simple resource: + +```py +@app.resource(name="users", parallelized=True, columns=User) +def users(ctx: SourceContext): + response = ctx.client.get("/users").json() + # Option A: + # Yield individual users by iterating over the response + for user in response["users"]: + yield user + + # Option B: + # Or return the list as is if modifications are + # not needed + yield response["users"] +``` + +**Key Points:** + +- The resource uses RESTClient as part of the shared context `ctx` to fetch users via the `/users` endpoint. +- `yield` returns one "row" ie. user at a time. +- `name="users"` becomes the "table" name by default, in this case the directory name where the files will be stored on disk eg. SuperIAM/users/data.jsonl.gz +- `parallelized=True` allows the resource to run concurrently. The concurrency limits can be set via the .dlt/config.toml configuration file. + + +## Extending the source +This structure keeps your collectors clean and easy to extend. To collect another asset, create a new `@app.resource` function and add it to the returned resources as part of your source. + +### Example: adding a second resource + +```py +@app.resource(name="computers", columns=Computer) +def computers(ctx: SourceContext): + response = ctx.client.get("/computers").json() + yield response["computers"] + + +@app.resource(name="users", parallelized=True, columns=User) +def users(ctx: SourceContext): + response = ctx.client.get("/users").json() + yield response["users"] + +@app.source(name="SuperIAM", max_table_nesting=0) +def source(token=dlt.secrets.value, host=dlt.secrets.value): + ctx = SourceContext( + client=RESTClient( + base_url=host, + headers={"accept": "application/json"}, + auth=BearerTokenAuth(token=token), + paginator=SinglePagePaginator(), + ) + ) + + return (users(ctx), computers(ctx)) +``` + +That is all for at least the basics of collection. You may have noticed that each resource has a 'columns=' argument, pointing to a particular class. These are custom classess that validates data returned by the resource and provide a schema for the OpenGraph node/edge representation. Next up is defining these [resource models](modelling.md). diff --git a/docs/development/creating-collector.md b/docs/development/creating-collector.md new file mode 100644 index 0000000..25bfa05 --- /dev/null +++ b/docs/development/creating-collector.md @@ -0,0 +1,41 @@ +# Creating a new collector + +This page walks you through generating a new collector using Cookiecutter, creating a dedicated virtual environment and +running your new collector. The template creates a minimal project and intialises the new directory as a git repository. + +The collector is automatically registered as a new CLI command after installing the project dependencies. Running +`python main.py collect --help` confirms the project is configured correctly before you start adding logic. + +## 1. Prerequisites + +- Python 3.12+ +- [uv](https://docs.astral.sh/uv/) + +## 2. Install the OpenHound CLI + +```console +uv tool install openhound +``` + +## 3. Create a new collector + +Create a new collector using the "create collector" command, which will prompt you for the required details. + +```console +openhound create collector +``` + +## 4. Done + +You should now be able to see the name of your new collector registered using the following command: + +```console +cd +> python src/main.py collect --help +``` + +Running the newly created collector will generate 100 dummy assets. Run +`python src/main.py collect ./output` to test the asset generation process. Next up is writing the +actual [collection](collection.md) logic. + +PS. Executing the command for the first time may take a few seconds. diff --git a/docs/development/decorators.md b/docs/development/decorators.md new file mode 100644 index 0000000..e07749f --- /dev/null +++ b/docs/development/decorators.md @@ -0,0 +1,113 @@ +# Decorators + +OpenHound provides several decorators to simplify the development of collection pipeline. Each decorator dynamically +registers a CLI command and defines specific stages as part of the collection workflow. The collection decorators wrap +functions that interact with [DLT](https://dlthub.com/docs/intro) (Data Load Tool) to extract and transform data into +the OpenGraph format. + +## Initialization + +A collector is initialized with the `OpenHound` class, which provides decorators for each pipeline stage. The first +argument is the name of your collector with an additional help/description. The name provided will also register a CLI +command with the same name, ie. `OpenHound("aws")` will expose an `openhound collect aws` command. + +```python +from openhound import OpenHound + +app = OpenHound("aws", help="OpenHound collector for AWS") +``` + +## Decorators + +### @app.collect() + +Registers a CLI command that collects resources from the source system and stores them in the original (optionally +filtered) format on disk. This function should return your DLT [source](collection.md). + +```python +@app.collect() +def collect(ctx: CollectContext): + from openhound_aws.source import source as aws_source + return aws_source() +``` + +**Parameters:** + +- `ctx` (CollectContext): Pipeline context for the resource collection stage. + +### @app.preproc() + +Registers a CLI command that (optionally) preprocesses collected resources and builds lookup data for the OpenGraph +conversion stage. This function should return a dictionary mapping resource names to table names. + +Optionally, you can provide a `transformer` function that applies SQL-based transformations to the loaded data in +DuckDB. The transformer function receives a DuckDB connection and can create new tables derived from the loaded +resources. + +In the example below, the AWS `users` resource will be stored in the `users` table, `groups` in the `groups` table. The +transformer function is imported from a separate `transforms` module and applies custom SQL/Ibis transformations ( +optional). + +```python +from openhound_aws.transforms import transforms + + +@app.preproc(transformer=transforms) +def preproc(ctx: PreProcContext): + resources = { + "resources": "resources", + "users": "users", + "groups": "groups", + "roles": "roles", + "policies": "policies", + "policy_attachments": "policy_attachments", + } + return resources +``` + +**Parameters:** + +- `ctx` (PreProcContext): Pipeline context for the preprocessing stage. +- `transformer` (Callable, optional): Optional function that takes a DuckDB connection and applies transformations to + create custom tables. + +### @app.convert() + +Registers a CLI command that converts collected resources into OpenGraph nodes and edges. This function should return a +tuple containing the DLT source and a dictionary of extra context to be added to each asset. + +```python +@app.convert(lookup=AWSLookup) +def convert(ctx: ConvertContext) -> Tuple[DltSource, dict]: + from openhound_aws.source import source as aws_source + extras = {} + return aws_source(), extras +``` + +**Parameters:** + +- `lookup`: An optional lookup class for resolving resources during OpenGraph conversion. +- `ctx` (ConvertContext): Pipeline context for the OpenGraph conversion stage. + +## Pipeline Flow + +```mermaid +flowchart LR + Raw[(Local Storage)]:::storage + subgraph collect["@app.collect()"] + Collect[Source API] + end + + subgraph preproc["@app.preproc()"] + Preproc[Lookup Tables] + end + + subgraph convert["@app.convert()"] + Convert[OpenGraph] + end + + Collect --> |Raw data| Raw[(Local Storage)] + Raw --> |Raw data| Preproc + Raw --> |Raw data| Convert + +``` diff --git a/docs/development/graph.md b/docs/development/graph.md new file mode 100644 index 0000000..a0577f5 --- /dev/null +++ b/docs/development/graph.md @@ -0,0 +1,140 @@ +# Configuring graph properties + +This page explains how to configure the required OpenGraph properties for your collector. When creating a new collector, +you need to define three key components: + +- **Node types**: The different kinds of assets your collector will extract; +- **Node properties**: Common properties shared by all nodes from your service; +- **GUID generation**: How to uniquely identify each node. + +The cookiecutter template includes a minimal `graph.py` that demonstrates the pattern to define the required OpenGraph +properties. + +## 1. Define Node Types + +Node types represent the different kinds of assets your collector provides. Define them as a constant in the +`kinds/nodes.py` and `kinds/edges.py` files. These strings are used in the `kinds` field of your asset models and should +be unique across all collectors to prevent collisions. + +```python +# kinds/nodes.py example +COMPUTER = "jamf_Computer" +USER = "jamf_ComputerUser" +POLICY = "jamf_Policy" +``` + +```python +# kinds/edges.py example +CONTAINS = "jamf_Contains" +ASSIGNED_USER = "jamf_AssignedUser" +MEMBER_OF = "jamf_MemberOf" +``` + +!!! info "IMPORTANT" +Use descriptive names which clearly identify both the asset type and the source service (e.g., jamf_User instead of just +User). This also prevents potential collisions with other collectors. + +## 2. Define (required) node properties + +Common node properties are attributes that every node from your collector/source will have regardless of its specific +node type. These should be properties unique to your service/source. During the conversion process every resource will +be checked if at these properties are present. Think of attributes like a tenant name, AD domain, etc. By inheriting +BaseProperties, both `name` and `displayname` are already included as properties as they are required fields as part of +the OpenGraph standard. Additionally, a `last_seen` property will be added for every node with the timestamp set to +current date/time (ie. when the resource was converted). + +```py +@dataclass +class BaseNodeProperties(BaseProperties): + tenant: str + domain: str +``` + +Make sure to choose properties that: + +- Are present on **all** nodes from your service; +- Provide useful context for identification and filtering; + +Make sure *not* to choose properties that: + +- Are specific only to a few nodes. Inherit from BaseNodeProperties and add those properties to the specific node model + instead; + +## 3. Implement GUID Generation + +The GUID (Globally Unique Identifier) uniquely identifies each node. You must override the guid() static method and the +id computed property. This generates a repeatable unique identifier that other collectors can also use to refer to your +nodes without requiring any context/awareness or lookup database to generate edges. + +### Override the guid() static method + +Choose properties that uniquely identify your resource within the service: + +```py +@staticmethod +def guid( + id: str, + node_type: str, + tenant: str, +) -> str: + return BaseNode.guid(id, node_type, tenant) + +``` + +### Override the id property + +Implement __post_init__ to set the node ID based on your custom guid() method with the appropriate properties: + +```py +def __post_init__(self): + self.id = self.guid( + str(self.properties.id), self.kinds[0], self.properties.tenant + ) +``` + +### GUID Selection Guidelines + +The properties you use for GUID generation should: + +- Uniquely identify the resource across all instances; +- Be stable and not change over time; +- Be deterministic, the same input should always procude the same GUID; +- Include a service specific identifier if available. + +Note: You can also generate a GUID based on pre-existing resource ID if available. + +## Complete example + +Here is a complete example of the Jamf collector: + +```py +@dataclass +class JAMFNodeProperties(BaseProperties): + tenant: str + id: int + tier: int + environmentid: str + + +@dataclass +class JAMFNode(BaseNode): + properties: JAMFNodeProperties + id: str = field(init=False) + + @staticmethod + def guid( + id: str, + node_type: str, + tenant: str, + ) -> str: + return BaseNode.guid(id, node_type, tenant) + + def __post_init__(self): + self.id = self.guid( + str(self.properties.id), self.kinds[0], self.properties.tenant + ) +``` + +## Profit! + +Your collector is now able to collect raw resources, perform data validation and perform the conversion to OpenGraph. diff --git a/docs/development/modelling.md b/docs/development/modelling.md new file mode 100644 index 0000000..320d2da --- /dev/null +++ b/docs/development/modelling.md @@ -0,0 +1,107 @@ +# Modelling resources + +This page explains how OpenHound models resources using Pydantic classes and the custom @app.asset decorator. The +cookiecutter template includes a minimal `asset.py` that demonstrates the pattern to define a resource model for data +validation and OpenGraph node/edge mapping. Each `@app.asset` (as defined in source.py) should specify a model as part +of the `columns` configuration, ex `@app.resource(name="asset", columns=Asset)`. + +## Pydantic resource model + +Individual resources are Pydantic models that extend the `BaseAsset`. These models represent the fields that your +resource yields and provides a mapping into OpenGraph nodes and +edges. In the example below, the id, name and hostname fields are required. If the DLT resource does not return any of +these fields, the application will exit with an error. More information on how to define fields using Pydantic can be +found [here](https://docs.pydantic.dev/latest/concepts/fields/). + +```py + +@dataclass +class AssetProperties(BaseNodeProperties): + """Properties for the Asset node. + + Attributes: + hostname: The asset hostname. + """ + hostname: str + + +@app.asset( + node=NodeDef( + kind=nk.Asset, + description="Asset node", + icon="computer", + properties=AssetProperties, + ), + edges=[ + EdgeDef( + start=nk.Asset, + end=nk.Group, + kind=ek.MemberOf, + description="Asset belongs to group", + traversable=False + ) + ] +) +class Asset(BaseAsset): + id: int # ID should be an integer + name: str # Name should be a string + hostname: str # hostname should be a string + groups: list[str] # contains a list of groups (value string) + + @property + def as_node(self): + properties = AssetProperties( + name=self.name, displayname=self.name, hostname=self.hostname + ) + return Node(properties=properties) + + @property + def _groups_memberships(self): + # Assuming the groups are also retrieved and the group name is the reference used + # to generate the ID + for group in self.groups: + start = EdgePath(value=self.as_node.id, match_by="id") + end = EdgePath(value=group.id, match_by="id") + yield Edge(kind=ek.MemberOf, start=start, end=end) + + @property + def edges(self): + # Can yield multiple different edges + yield from self._groups_memberships + yield from ... + yield from ... + + +``` + +**Key points:** + +- The `@app.asset` decorator registers the model as an OpenGraph resource. +- The (`id`, `name`, `hostname`) fields should be included in the data yielded by the `@app.resource`. The fields + defined by our model are not optional and should match the corresponding data types. +- `as_node` maps the fields into a node as used by OpenGraph. +- `edges` yields relationships to other nodes (empty in the template). + +## Extending node properties + +OpenGraph nodes have standard properties (like `name` and `displayname`). You can extend +them with resource-specific fields by defining your own ExtendedProperties and inheriting `BaseNodeProperties`. + +```py +@dataclass +class AssetProperties(BaseNodeProperties): + hostname: str + +``` + +This allows you to add additional data to a node's properties without changing the base OpenGraph schema. + +## TLDR; workflow + +1. For every resource, create a dedicated model in `models/` and decorate it with `@app.asset`. +2. Define resource-specific fields for the Pydantic model. These can either be required or optional fields. +3. Extend `BaseNodeProperties` if you want to include custom node properties for the resource. +4. Implement `as_node` and `edges` for OpenGraph output. The pipeline will refuse to start unless these methods have + been implemented. + +Next up, and the final step, is configuring the "core" [opengraph model](graph.md) for your source. diff --git a/docs/development/opengraph-models.md b/docs/development/opengraph-models.md new file mode 100644 index 0000000..c0dc3d6 --- /dev/null +++ b/docs/development/opengraph-models.md @@ -0,0 +1,35 @@ +# OpenGraph Model +This page describes the core OpenGraph (Pydantic) models used to represent and ingest OpenGraph data into BloodHound. These models validate the nodes and edges exported by each collector, ensuring data consistency before ingestion. The graph model defines how objects (nodes) and their relationships (edges) are structured, along with metadata about the collection process. + +# Graph +A graph is a collection of nodes and edges representing the complete OpenGraph dataset. Each graph contains metadata describing the collection context, including the collector type, version and collection methods used. The graph structure consists of `GraphEntries` which contains the nodes and edges, `MetaData` which provides information about how the data was collected and `CollectorProperties` which define the specific collection methods. + +::: openhound.core.models.graph + options: + show_root_heading: false + members: + - Graph + - GraphEntries + - MetaData + - MetaDataCollector + - CollectorProperties + +# Nodes +A node represents an object in the graph with a set of properties/attributes. Each node contains a list of `kinds` which specify its type, a few examples are "Computer", "User", "JamfUser" etc. The first value of `kinds` represents the primary kind in BloodHound, which will also be used as the visual "title" of a node. Any additional kind can be used inside a Cypher query as part of an optional filter. Each node is expected to contain at least `displayname` and `name` as part of it's properties. Additionally each node will include an automatically generated `last_seen` as a default property. + +::: openhound.core.models.entries + options: + show_root_heading: false + members: + - Node + - NodeProperties + +# Edges +An edge represents a directional relationship between two nodes in the graph. Each edge contains a `kind` which specifies the relationship type, a few examples are "MemberOf", "AdminTo", "HasSession" etc. Edges can also have properties/attributes that provide additional context about the relationship. In Cypher queries, edges are used to traverse the graph and discover attack paths between nodes. Edges require no properties, but will include `last_seen` as a default property. Edges may also contain a `composed` property indicating the edge is derived from multiple relationships and a `traversable` property indicating whether the edge can be used in an attack path. + +::: openhound.core.models.entries + options: + show_root_heading: false + members: + - Edge + - EdgeProperties diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..933e548 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,74 @@ +# OpenHound + +OpenHound is a standardized framework for building OpenGraph collectors and converters. Built +on [DLT](https://dlthub.com/docs/intro) (Data Load Tool), it provides a +consistent workflow for collecting, processing, and converting data from any source into BloodHound-compatible graphs. +OpenHound enforces a collect-first, convert-later pipeline. Raw data collected from a source is always stored before +transformation and ensures reproducibility. Custom decorators simplify collector development with minimal boilerplate, +while CLI commands and graph documentation are automatically generated for every source. + +## How it works + +```mermaid +flowchart LR + + + Collect([Collect]):::step + Preprocess([Preprocess]):::step + Convert([Convert]):::step + + Raw[(JSONL/Parquet)]:::storage + DuckDB[(DuckDB)]:::optional + + OpenGraph[OpenGraph Files]:::output + BloodHound[BloodHound API]:::output + + Collect --> Raw + Raw --> Preprocess + Preprocess -.-> DuckDB + DuckDB -.-> Convert + Preprocess --> Convert + Convert --> OpenGraph + +``` + +**Collect**: +OpenHound uses DLT to collect resources from various services. Resources are parsed using a Pydantic model and stored as +JSONL/Parquet on disk during the collection phase. + +**Pre-process**: +A DuckDB database can be (optionally) populated to store resources for OpenGraph conversion. The database can be used as +a lookup to find, for example, all resources a particular user/group has permissions to. + +**Convert**: +The raw resources are read from disk and converted to OpenGraph nodes and edges. The generated local OpenGraph JSON +files are automatically split into multiple files based on your configured (entry) +size limit and resource type. + +# DLT (Data Load Tool) + +DLT Is an open-source Python library to load data from various data sources into well-structured datasets. The dlt +library solves a lot of the issues faced when building a custom data collector for BloodHound. DLT includes features +like: + +- Schema validation: Automatically catch potential data formatting issues from your source and before exporting your + graph; +- Incremental loading: Only process what has changed since your last run; +- Pre-built connectors: Already contains pre-built connectors and an easy to use (generic) HTTP connector which deals + with pagination automatically 💫; +- Multi-processing: Parallelize resource collection; +- Config management: Simple configuration management for your custom sources, which are read from both environment + variables and/or centralized config files; + +## Supported sources (current state) + +- Okta +- Github +- Jamf + +## Supported destinations + +- File Export: Generates local OpenGraph JSON files which are automatically split into multiple files based on your + configured (entry) size limit and resource type. +- Ingest API: Generates the same OpenGraph format but uploads the content via the BloodHound BHE API without the storing + files on disk first. This feature is only supported for BloodHound enterprise customers. diff --git a/docs/javascript/custom.mjs b/docs/javascript/custom.mjs new file mode 100644 index 0000000..fbd8aa4 --- /dev/null +++ b/docs/javascript/custom.mjs @@ -0,0 +1,109 @@ +const shadowRoots = new Map(); + +const originalAttachShadow = Element.prototype.attachShadow; +Element.prototype.attachShadow = function(init) { + const root = originalAttachShadow.call(this, { ...init, mode: 'open' }); + shadowRoots.set(this, root); + return root; +}; + +function initPanzoom(svg, host) { + if (svg.dataset.panzoom) return; + svg.dataset.panzoom = 'true'; + + const MIN_SCALE = 0.2, MAX_SCALE = 10; + let scale = 1, tx = 0, ty = 0; + let dragging = false, startX = 0, startY = 0; + + function applyTransform() { + svg.style.transformOrigin = '0 0'; + svg.style.transform = `translate(${tx}px, ${ty}px) scale(${scale})`; + } + + function clamp() { + // Host dimensions — the visible container + const hw = host.clientWidth; + const hh = host.clientHeight; + // Scaled SVG dimensions based on its natural (attribute) size + const sw = svg.width.baseVal.value * scale; + const sh = svg.height.baseVal.value * scale; + + // If SVG is smaller than container, lock to 0 (top-left) + // If SVG is larger than container, allow panning up to the point + // where the opposite edge aligns with the container edge + tx = sw < hw ? 0 : Math.min(0, Math.max(tx, hw - sw)); + ty = sh < hh ? 0 : Math.min(0, Math.max(ty, hh - sh)); + } + + svg.addEventListener('wheel', (e) => { + e.preventDefault(); + + const rect = host.getBoundingClientRect(); + // Mouse position relative to the host container + const mx = e.clientX - rect.left; + const my = e.clientY - rect.top; + + const factor = e.deltaMode === 1 ? 0.05 : 0.002; + const newScale = Math.min(Math.max(scale * (1 - e.deltaY * factor), MIN_SCALE), MAX_SCALE); + + // Zoom toward cursor + tx = mx - (mx - tx) * (newScale / scale); + ty = my - (my - ty) * (newScale / scale); + scale = newScale; + + clamp(); + applyTransform(); + }, { passive: false }); + + svg.addEventListener('pointerdown', (e) => { + dragging = true; + startX = e.clientX - tx; + startY = e.clientY - ty; + svg.setPointerCapture(e.pointerId); + svg.style.cursor = 'grabbing'; + }); + + svg.addEventListener('pointermove', (e) => { + if (!dragging) return; + tx = e.clientX - startX; + ty = e.clientY - startY; + clamp(); + applyTransform(); + }); + + svg.addEventListener('pointerup', () => { + dragging = false; + svg.style.cursor = 'grab'; + }); + + // The host must clip the SVG — without this the transform bleeds outside + host.style.overflow = 'hidden'; + host.style.position = 'relative'; + host.style.display = 'block'; + + svg.style.cursor = 'grab'; + + // Default zoom-in to 150% + scale = 1; + clamp(); + applyTransform(); +} + +const observer = new MutationObserver(() => { + document.querySelectorAll('.mermaid').forEach(el => { + const root = shadowRoots.get(el); + if (!root) return; + + if (!root.querySelector('.custom-injected')) { + const link = document.createElement('link'); + link.rel = 'stylesheet'; + link.href = '/stylesheets/mermaid-custom.css'; + link.classList.add('custom-injected'); + root.appendChild(link); + } + + root.querySelectorAll('svg:not([data-panzoom]):not(svg svg)').forEach(svg => initPanzoom(svg, el)); + }); +}); + +observer.observe(document.body, { childList: true, subtree: true }); diff --git a/docs/javascript/mermaid.mjs b/docs/javascript/mermaid.mjs new file mode 100644 index 0000000..a69a122 --- /dev/null +++ b/docs/javascript/mermaid.mjs @@ -0,0 +1,42 @@ +import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@11.12.2/dist/mermaid.esm.mjs'; +import elkLayouts from 'https://cdn.jsdelivr.net/npm/@mermaid-js/layout-elk@0/dist/mermaid-layout-elk.esm.min.mjs'; + +mermaid.registerIconPacks([ + { + name: 'fa', + loader: () => { + return fetch('https://unpkg.com/@iconify-json/fa6-solid@1/icons.json') + .then((res) => { + return res.json(); + }); + }, + }, + { + name: 'fab', + loader: () => { + return fetch('https://unpkg.com/@iconify-json/fa6-brands@1/icons.json') + .then((res) => { + return res.json(); + }); + }, + }, + { + name: 'logos', + loader: () => { + return fetch('https://unpkg.com/@iconify-json/logos@1/icons.json') + .then((res) => { + return res.json(); + }); + }, + }, +]); + + +mermaid.registerLayoutLoaders(elkLayouts); +mermaid.initialize({ + startOnLoad: true, + layout: "elk", + theme: 'base' +}); + +window.mermaid = mermaid; diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 0000000..13e5cf1 --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,45 @@ +# Logging + +OpenHound provides several logging configurations based on the deployment method and enables automatic file rotation. + +## Logging modes +Based on the detected runtime OpenHound will automatically apply custom logging handlers and formats. The following modes are currently available. + +### Container +The container mode is used when running OpenHound in a container or Kubernetes environment. This is detected by the LOG_CONTAINER (manually set) environment variable and/or KUBERNETES_SERVICE_HOST environment variable. All logs will be formatted in JSON and send to stdout for easy parsing by container orchestrators/log shippers with container support. Logs will not be written to disk. + +### CLI +The CLI mode is used when running OpenHound in a terminal, ie. when TTY detected. OpenHound will automatically output errors to stderr using Rich-formatted logs with colors and enhanced tracebacks for better readability. Additionally, a JSON-formatted log will be written to `openhound.log`. Both handlers will be used when running in CLI mode. + +### Service +TODO: The service mode is used when running OpenHound as a service using the `openhound start service` command. A JSON-formatted log will be written to `openhound.log`. + +## Log rotation +OpenHound implements both **time-based** and **size-based** log rotation. When a log is rotated, the date/time will be appended to the file name, ex. `openhound.log.2026-02-19_00` or `openhound.log.2026-02-19`. When rotation occurs based on the file size and happens within the same period/interval, the minutes and seconds are also added for uniqueness ex. `openhound.log.2026-02-19_00-15-23`. + + +## Configuration +Set logging parameters in `.dlt/config.toml`: + +```toml +[runtime] +log_level = "INFO" # Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + +# The time based rotation settings +log_rotate_when = "midnight" # S for seconds, H for hours, D for days and 'midnight' for rotating at midnight +log_interval = 1 # Rotate every X unit of seconds, hours, days etc. Ignored when rotate_when is 'midnight' + +# The size based rotation settings +log_max_bytes = 10485760 # ex 10_485_760 for 10MB, 0 means rotate by time only +log_backup_count = 14 # the amount of files to keep before deleting the oldest +``` + +Or set the configuration via environment variables: + +```bash +export RUNTIME__LOG_LEVEL="INFO" +export RUNTIME__LOG_MAX_BYTES="10485760" +export RUNTIME__LOG_BACKUP_COUNT="14" +export RUNTIME__LOG_ROTATE_WHEN="midnight" +export RUNTIME__LOG_INTERVAL="1" +``` diff --git a/docs/sources/faker/collection/assets/FakeComputer.md b/docs/sources/faker/collection/assets/FakeComputer.md new file mode 100644 index 0000000..ed494d9 --- /dev/null +++ b/docs/sources/faker/collection/assets/FakeComputer.md @@ -0,0 +1,26 @@ +# FakeComputer +This section describes the exported OpenGraph asset(s) for the FakeComputer class. Each resource wrapped with the @app.asset decorator will export documentation for an OpenGraph node, multiple edges or a combination of both. + + +## Node +| Name | Icon | +|------|------| +| [RAND_Computer](../../graph/nodes/RAND_Computer.md) | :fontawesome-solid-computer: | + + + + + +## Edges + +| Start | End | Kind | Description | +|-------|-----|------|-------------| +| [RAND_Computer](../../graph/nodes/RAND_Computer.md) | [RAND_Computer](../../graph/nodes/RAND_Computer.md) | [RAND_ExampleReference](../../graph/edges/RAND_ExampleReference.md) | Binding DummyComputer to another DummyComputer | + + + + + +## Resource attributes +This section describes the data collected and available fields as present in the exported jsonl/parquet files. + diff --git a/docs/sources/faker/collection/overview.md b/docs/sources/faker/collection/overview.md new file mode 100644 index 0000000..438b0e5 --- /dev/null +++ b/docs/sources/faker/collection/overview.md @@ -0,0 +1,39 @@ +# Overview + +This page documents the `faker` OpenHound source, including its architecture, resource relationships and exported OpenGraph assets. The source extracts data from faker and transforms it into the standardized OpenGraph format. Use the visual diagram below to understand how nodes relate to each other, then explore the exported DLT [resources](pipeline.md) and invidual assets. + +## Visual overview +The diagram below shows the relationships between OpenGraph nodes and assets that are part of the faker source. This diagram is automatically generated by each resource/asset that is wrapped with the `@app.asset` decorator. + +```mermaid +flowchart LR + + RAND_Computer["fa:fa-computer"]:::bhNode + + RAND_Computer -- RAND_ExampleReference --> RAND_Computer +``` + +## Node Icons +The following table shows the Font Awesome icon used for each node type in the visual diagram above: + + +| Node Type | Icon | Font Awesome Class | +|------|-----------|-------------------| +| RAND_Computer | :fontawesome-solid-computer: | `fa-computer` | + + + +## Exported OpenGraph assets + +The following table lists all OpenGraph assets produced by the faker source. Each asset represents a node or edge as part of the OpenGraph output. + +| Class | Description | Node | Edges | +|------|-------------|-------|-------| +|[FakeComputer](assets/FakeComputer.md) | Randomly generated computer resources, returns node and edges | RAND_Computer | 1 | + + + +**Next Steps:** + +- Explore individual faker [resources](pipeline.md) to see what data / API endpoints are used for extraction. +- Review asset schemas for detailed field information for each individual resource. \ No newline at end of file diff --git a/docs/sources/faker/collection/pipeline.md b/docs/sources/faker/collection/pipeline.md new file mode 100644 index 0000000..1d7a13b --- /dev/null +++ b/docs/sources/faker/collection/pipeline.md @@ -0,0 +1,52 @@ +# Pipeline + +This page lists all pipeline functions that make up the `faker` collector. + +## Sources +Sources are the top-level entry points that define the configuration and which resources/transformers to process. + +This section includes the API reference for the source function, including all parameters required to run the pipeline. The parameters may also include references to credentials or settings stored inside your DLT configuration, which can be loaded via environment variables or the .dlt/secrets.toml file. + +!!! info "Configuration Required" + Some source parameters may require configuration values that should be set via: + + - **Environment variables** (recommended for secrets); + - **`.dlt/config.toml`** for non-sensitive configuratio;n + - **`.dlt/secrets.toml`** for API keys, tokens, and credentials + + See the [DLT credentials documentation](https://dlthub.com/docs/walkthroughs/add_credentials) for details on managing credentials. + + + +::: openhound_faker.source.source + options: + show_source_link: true + show_root_heading: true + heading_level: 3 + + + +## Resources + +Resources are the individual extraction functions which are part of a DLT pipeline. Within the context of OpenHound, resources are functions that call individual API endpoints to fetch users, computers, roles etc. Each `@app.resource` represents a specific data item that can be extracted and loaded. + +**As part of this collector:** + +- Some resources return OpenGraph assets that can be loaded directly into BloodHound. These are wrapped with the @app.asset decorator; +- Other resources provide supplementary data used for enrichment, transformations or other supporting operations + + +::: openhound_faker.source.computers + options: + show_source_link: true + show_root_heading: true + heading_level: 3 + + + +## Transformers + +Transformers are similar to resources but are used for data transformation and enrichment. They typically receive the output of a resource and yield additional records. An example could be fetching user details via a seperate API endpoint based on a user ID returned by a resource. + + +*No transformer functions found.* diff --git a/docs/stylesheets/mermaid-custom.css b/docs/stylesheets/mermaid-custom.css new file mode 100644 index 0000000..22f96fb --- /dev/null +++ b/docs/stylesheets/mermaid-custom.css @@ -0,0 +1,43 @@ +.md-typeset__table { + width: 100%; +} + +.md-typeset__table table:not([class]) { + display: table +} + +.bhNode circle { + stroke-width: 4px !important; + stroke: black !important; + fill: white !important; +} + +.bhNode rect { + stroke-width: 5px !important; + stroke: black !important; + fill: white !important; + /* rx: 25px !important; */ + /* ry: 25px !important; */ + rx: 40px !important; + ry: 40px !important; +} + +.bhNode { + font-size: 30px; +} + +.bhNode .label div:first-child { + padding-bottom: 14px !important; + padding-top: 14px !important; + /* padding: 15px; */ + line-height: 0px !important; +} + +.bhNode .nodeLabel svg { + transform: scale(2.8) !important; + vertical-align: 0px !important; +} + +.flowchart-link { + stroke-width: 3px !important; /* Default is usually 1.5px */ +} diff --git a/pyproject.toml b/pyproject.toml index 97fd180..545a991 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,15 +6,15 @@ readme = "README.md" requires-python = ">=3.13" dependencies = [ "alive-progress>=3.3.0", - "dlt==1.22.2", - "duckdb==1.5.0", + "dlt==1.26.0", + "duckdb==1.5.2", "griffe>=1.15.0", "griffe-fieldz>=0.5.0", - "mkdocstrings[python]>=1.0.0", + "mkdocstrings[python]==1.0.4", "psutil>=7.2.1", - "pydantic==2.12.5", + "pydantic==2.13.3", "tqdm>=4.67.1", - "typer>=0.19.2", + "typer==0.25.1", "cookiecutter>=2.6.0", "pydantic-extra-types>=2.11.0", "jinja2>=3.1.6", @@ -67,14 +67,14 @@ local_scheme = "no-local-version" dev = [ "openhound-faker==0.0.4", "ipython>=9.12.0", - "pre-commit>=4.5.1", + "pre-commit==4.6.0", "pytest>=9.0.1", - "marimo>=0.23.0", - "altair>=6.0.0", - "fastapi>=0.129.0", - "zensical>=0.0.23", - "ruff>=0.15.4", - "mypy>=1.19.1", + "marimo==0.23.4", + "altair==6.1.0", + "fastapi==0.136.1", + "zensical>=0.0.38", + "ruff==0.15.12", + "mypy==1.20.2", "types-pyyaml>=6.0.12.20250915", "types-requests>=2.33.0.20260408", "httpx>=0.28.1", diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..9c28011 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,33 @@ +# Example skaffold to test/deploy the jamf collector +apiVersion: skaffold/v4beta13 +kind: Config +metadata: + name: openhound + +build: + local: + push: false + useBuildkit: true + artifacts: + - image: docker.io/specterops/openhound + docker: + dockerfile: Dockerfile + target: scheduler + buildArgs: + OPENHOUND_VERSION: 0.1.0 + secrets: + - id: ARTIFACT_TOKEN + env: ARTIFACT_TOKEN + +deploy: + helm: + releases: + - name: openhound-jamf + chartPath: deployments/helm/openhound + valuesFiles: + - deployments/helm/values.example.yaml + setValueTemplates: + image.repository: "{{.IMAGE_REPO_docker_io_specterops_openhound}}" + image.tag: "{{.IMAGE_TAG_docker_io_specterops_openhound}}@{{.IMAGE_DIGEST_docker_io_specterops_openhound}}" + setValues: + image.pullPolicy: IfNotPresent diff --git a/src/openhound/cli/create.py b/src/openhound/cli/create.py index d38c18a..e29dc27 100644 --- a/src/openhound/cli/create.py +++ b/src/openhound/cli/create.py @@ -1,16 +1,23 @@ import re +from enum import Enum from importlib.metadata import entry_points from pathlib import Path -from typing import Optional +from typing import Optional, Annotated import typer + +class Template(str, Enum): + mintlify = "mintlify" + mkdocs = "mkdocs" + + create_app = typer.Typer( help="Create a new OpenHound collector based on the cookiecutter template" ) -def generate_docs(base_path: Path, group: str = "openhound.sources") -> None: +def generate_docs(base_path: Path, template: Template, group: str = "openhound.sources") -> None: """Loads the collector extensions via entrypoints and uses griffe to generate OpenHound collector docs. Args: @@ -54,42 +61,47 @@ def generate_docs(base_path: Path, group: str = "openhound.sources") -> None: @create_app.command() def docs( - output_dir: Path = typer.Argument( - Path("./docs"), - help="Where to create the docs for extensions", - file_okay=False, - dir_okay=True, - exists=True, - ), + output_dir: Path = typer.Argument( + Path("./docs"), + help="Where to create the docs for extensions", + file_okay=False, + dir_okay=True, + exists=True, + ), + template: Annotated[Template, typer.Option( + help="Which template to use for docs generation") + ] = Template.mkdocs, + ): """Generate OpenHound collector docs based on @app.asset decorators and docstrings in each collector. Args: output_dir (Path): Base path where to create the collector docs (default: ./docs). + template (Template): Template to use for docs generation """ - generate_docs(output_dir) + generate_docs(output_dir, template) @create_app.command() def collector( - output_dir: Path = typer.Argument( - help="Where to create the collector", - file_okay=False, - dir_okay=True, - ), - config: Optional[Path] = typer.Option( - None, - "--config", - "-c", - help="Path to config file (YAML/JSON)", - exists=True, - file_okay=True, - dir_okay=False, - ), - template: str = typer.Option( - "gh:SpecterOps/OpenHound-template", "--template", "-t", help="Template to use" - ), + output_dir: Path = typer.Argument( + help="Where to create the collector", + file_okay=False, + dir_okay=True, + ), + config: Optional[Path] = typer.Option( + None, + "--config", + "-c", + help="Path to config file (YAML/JSON)", + exists=True, + file_okay=True, + dir_okay=False, + ), + template: str = typer.Option( + "gh:SpecterOps/OpenHound-template", "--template", "-t", help="Template to use" + ), ): """Generate a new OpenHound collector using a cookiecutter template. diff --git a/src/openhound/docs/pipeline.py b/src/openhound/docs/pipeline.py index b9652a0..ff30480 100644 --- a/src/openhound/docs/pipeline.py +++ b/src/openhound/docs/pipeline.py @@ -1,6 +1,7 @@ import importlib from collections import defaultdict from dataclasses import dataclass, field +from enum import Enum from pathlib import Path import griffe @@ -9,6 +10,11 @@ ROOT_PATH = Path(__file__).resolve().parents[1] +class Template(str, Enum): + mintlify = "mintlify" + mkdocs = "mkdocs" + + @dataclass class Collector: assets: list = field(default_factory=list) @@ -181,6 +187,7 @@ def __init__( sources: list[dict] | None = None, resources: list[dict] | None = None, transformers: list[dict] | None = None, + template: Template = Template.mkdocs, template_dir: Path = Path("docs/templates"), ): self.name = name @@ -190,7 +197,7 @@ def __init__( self.resources = resources or [] self.transformers = transformers or [] self.env = Environment( - loader=FileSystemLoader(ROOT_PATH / template_dir), + loader=FileSystemLoader(ROOT_PATH / template_dir / template.value), extensions=["jinja2.ext.do"], ) diff --git a/src/openhound/docs/pipeline_back.py b/src/openhound/docs/pipeline_back.py new file mode 100644 index 0000000..ca09b8a --- /dev/null +++ b/src/openhound/docs/pipeline_back.py @@ -0,0 +1,588 @@ +import importlib +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +import griffe +from jinja2 import Environment, FileSystemLoader + +ROOT_PATH = Path(__file__).resolve().parents[1] + + +@dataclass +class Collector: + assets: list = field(default_factory=list) + sources: list = field(default_factory=list) + resources: list = field(default_factory=list) + transformers: list = field(default_factory=list) + + +@dataclass +class DocumentedField: + name: str + type_name: str + description: str | None = None + + +class GraphResourceDecorator(griffe.Extension): + def __init__(self) -> None: + super().__init__() + self.collectors: dict[str, Collector] = defaultdict(Collector) + self.griffe_data = None + + @staticmethod + def _collector(args) -> dict[str, str]: + """Parses the collector name and description/help as a string. + + Args: + args: The function's original arguments. + + Returns: + A dictionary containing the collector name and help description. + """ + return { + "name": args[0].replace("'", ""), + "help": args[1].value.replace("'", ""), + } + + @staticmethod + def _description(arg, cls) -> dict[str, str]: + """Parses the asset description as a string. + + Args: + arg: The decorator argument for 'description'. + cls: The griffe Class being processed (unused, kept for uniform signature). + + Returns: + A dictionary containing the asset description. + """ + return {"description": arg.value.replace("'", "")} + + @staticmethod + def _resolve(expr, cls): + """Resolve a decorator argument to its actual runtime string value""" + + # Plain text string + if isinstance(expr, str): + return expr.replace("'", "") + + # ExprAttribute, ie. module constants like nk.USER + if isinstance(expr, griffe.ExprAttribute): + alias_name = expr.values[0].name # the module, ex. "nk" + attr_name = expr.values[1].name # the attribute, ex. "USER" + + alias_obj = cls.module.members.get(alias_name) + if alias_obj is not None and hasattr(alias_obj, "target_path"): + mod = importlib.import_module(alias_obj.target_path) + return getattr(mod, attr_name) + + if isinstance(expr, griffe.ExprName): + return expr.canonical_path.replace("'", "") + + # A fallback + return str(expr).replace("'", "") + + @staticmethod + def _parse_node_properties(properties_cls: griffe.Class, docstring_style: Literal['google'] = "google") -> dict: + """Parses the node properties as a dictionary. + + Args: + properties_cls: The Griffe class being processed. + """ + + descriptions: dict[str, str] = {} + all_attributes = {} + for attribute, content in properties_cls.attributes.items(): + all_attributes[attribute] = {'type': content.annotation.canonical_name} + + sections = properties_cls.docstring.parse(docstring_style) + for section in sections: + if section.kind is not griffe.DocstringSectionKind.attributes: + continue + for docs_attribute in section.value: + descriptions[docs_attribute.name] = (docs_attribute.description or "").strip() + + return descriptions + + @staticmethod + def _node(arg, cls) -> dict: + """Parse a ``NodeDef(...)`` as a dictionary. Returns the kind, description, icon.""" + node = {} + for argument in arg.value.arguments: + if argument.name in ['kind', 'description', 'icon']: + node[argument.canonical_name] = GraphResourceDecorator._resolve( + argument.value, cls + ) + if argument.name == 'properties': + node['properties'] = GraphResourceDecorator._parse_node_properties(argument.value.resolved) + + return {"node": node} + + @staticmethod + def _edges(arg, cls) -> dict: + """Parse the ``EdgeDef(...)`` edges as a dictionary. Returns the start, end and kind.""" + + # TODO: Make these conditional with custom parsers based on the argument + edges = [] + for elem in arg.value.elements: + edge = {} + for a in elem.arguments: + parsed_value = GraphResourceDecorator._resolve(a.value, cls) + if a.canonical_name == "traversable": + parsed_value = True if parsed_value == "True" else False + edge[a.canonical_name] = parsed_value + edges.append(edge) + return {"edges": edges} + + def on_function(self, *, func: griffe.Function, **kwargs) -> None: + """Generates an overview of source, resources and transformers. + + Args: + func (griffe.Function): The griffe function being processed + """ + collector_decorators = ("app.source", "app.resource", "app.transformer") + matched = None + for decorator in func.decorators: + for suffix in collector_decorators: + if decorator.callable_path.endswith(suffix): + matched = decorator + break + + if matched: + collector_meta = self._collector(func.module.members["app"].value.arguments) + collector = self.collectors[collector_meta["name"]] + func_name = func.name + for arg in matched.value.arguments: + if arg.canonical_name == "name": + func_name = str(arg.value).strip("'\"") + break + + entry = { + "function_name": func.name, + "module_path": func.path, + "name": func_name, + } + + # TODO: This can probably be simplified + if matched.callable_path.endswith("app.source"): + collector.sources.append(entry) + elif matched.callable_path.endswith("app.resource"): + collector.resources.append(entry) + elif matched.callable_path.endswith("app.transformer"): + collector.transformers.append(entry) + + def on_class( + self, + *, + cls: griffe.Class, + **kwargs, + ) -> None: + + parsers = { + "node": lambda a: self._node(a, cls), + "description": lambda a: self._description(a, cls), + "edges": lambda a: self._edges(a, cls), + } + + for decorator in cls.decorators: + if not decorator.callable_path.endswith("app.asset"): + continue + + resource_as_dict = {} + collector_meta = self._collector(cls.module.members["app"].value.arguments) + collector = self.collectors[collector_meta["name"]] + + for args in decorator.value.arguments: + parser = parsers.get(args.canonical_name) + if parser: + result = parser(args) + resource_as_dict = { + **resource_as_dict, + **result, + "class": cls.name, + "path": cls.path, + } + collector.assets.append(resource_as_dict) + + +class CustomCollectorDocs: + def __init__( + self, + name: str, + base_docs_dir: Path, + assets: list[dict], + sources: list[dict] | None = None, + resources: list[dict] | None = None, + transformers: list[dict] | None = None, + descriptions_dir: Path | None = None, + template_dir: Path = Path("docs/templates"), + ): + self.name = name + self.base_docs_dir = base_docs_dir + self.descriptions_dir = descriptions_dir or (base_docs_dir / "descriptions") + self.assets = assets + self.sources = sources or [] + self.resources = resources or [] + self.transformers = transformers or [] + self.env = Environment( + loader=FileSystemLoader(ROOT_PATH / template_dir), + extensions=["jinja2.ext.do"], + ) + self.env.globals["escape_markdown_cell"] = self.escape_markdown_cell + self._griffe_class_cache: dict[str, griffe.Class | None] = {} + + @staticmethod + def escape_markdown_cell(value: Any) -> str: + if value is None: + return "" + return str(value).replace("|", r"\|").replace("\n", "
") + + def _load_griffe_class(self, class_path: str) -> griffe.Class | None: + if class_path in self._griffe_class_cache: + return self._griffe_class_cache[class_path] + + try: + loaded_obj = griffe.load( + class_path, + resolve_aliases=True, + resolve_external=True, + ) + except ImportError: + self._griffe_class_cache[class_path] = None + return None + + resolved_class = loaded_obj if isinstance(loaded_obj, griffe.Class) else None + self._griffe_class_cache[class_path] = resolved_class + + return resolved_class + + @staticmethod + def _format_type_name(annotation: Any) -> str: + if annotation is None or annotation is type(None): + return "None" + + if isinstance(annotation, str): + return annotation + + if getattr(annotation, "__module__", "") == "builtins" and hasattr( + annotation, "__name__" + ): + return annotation.__name__ + + rendered = str(annotation) + # TODO: what the fuck is this + # if rendered.startswith(""): + # return rendered.removeprefix("").split(".")[-1] + + return rendered.replace("typing.", "").replace("NoneType", "None") + + def _field_descriptions( + self, cls: griffe.Class, include_inherited: bool = False + ) -> dict[str, str]: + descriptions: dict[str, str] = {} + + # TODO: Slop + if include_inherited: + classes = [*reversed(cls.mro()), cls] + else: + classes = [cls] + + for mro_cls in classes: + docstring = mro_cls.docstring + if docstring is None: + continue + + sections = docstring.parse("google") + for section in sections: + if section.kind is not griffe.DocstringSectionKind.attributes: + continue + for attribute in section.value: + descriptions[attribute.name] = (attribute.description or "").strip() + + return descriptions + + def _documented_fields( + self, cls: griffe.Class, include_inherited: bool = False + ) -> list[DocumentedField]: + descriptions = self._field_descriptions(cls, include_inherited=include_inherited) + members = cls.all_members if include_inherited else cls.members + + documented_fields: list[DocumentedField] = [] + for field_name, member in members.items(): + labels = getattr(member, "labels", set()) + annotation = getattr(member, "annotation", None) + + if "instance-attribute" not in labels: + continue + if annotation is None: + continue + + documented_fields.append( + DocumentedField( + name=field_name, + type_name=self._format_type_name(annotation), + description=descriptions.get(field_name) or None, + ) + ) + + return documented_fields + + def render_class_table(self, class_path: str, include_inherited: bool = False) -> str: + template = self.env.get_template("class_table.md.j2") + resolved_class = self._load_griffe_class(class_path) + if resolved_class is None: + return template.render(fields=[]) + + # TODO: Handle multiple cases for: + # dataclasses, pydantic models + + return template.render( + fields=self._documented_fields( + resolved_class, include_inherited=include_inherited + ) + ) + + def _description_file_content(self, category: str, name: str) -> str: + description_path = self.descriptions_dir / self.name / category / f"{name}.md" + if description_path.is_file(): + return description_path.read_text() + return "" + + @property + def collector_template(self) -> str: + """Renders the collector overview template + + Returns: + str: The rendered template as a string + """ + template = self.env.get_template("overview.md.j2") + result = template.render(name=self.name, graph_resources=self.assets) + return result + + @property + def pipeline_template(self) -> str: + """Renders the DLT source/resource/transformer function inventory. + + Note: Only works when wrapped with the OpenHound @app.source/@resource/@transformer decorators + + Returns: + str: The rendered template as a string. + """ + template = self.env.get_template("pipeline.md.j2") + return template.render( + name=self.name, + sources=self.sources, + resources=self.resources, + transformers=self.transformers, + ) + + @property + def asset_templates(self) -> list[dict[str, str]]: + """Renders the individual assets used by a collector + + Returns: + list[dict[str, str]]: Returns a list of rendered templates + """ + results = [] + template = self.env.get_template("asset.md.j2") + for asset in self.assets: + graph_resource = {**asset} + graph_resource["resource_attributes_table"] = self.render_class_table( + graph_resource["path"] + ) + if graph_resource.get("node", {}).get("properties"): + graph_resource["node_properties_table"] = self.render_class_table( + graph_resource["node"]["properties"], include_inherited=True + ) + + result = template.render(graph_resource=graph_resource) + results.append({**graph_resource, "render": result}) + return results + + @property + def _node_index(self) -> dict[str, dict]: + """Creates a mapping of unique node kinds with their properties and incoming/outgoing edges + + Returns: + dict[str, dict]: Node details by node name/id + """ + all_nodes: dict[str, dict] = {} + + # Iterate over all assets and populate the all_nodes index with all the unique node types + for asset in self.assets: + node = asset.get("node") + if not node or not node.get("kind"): + continue + + kind = node["kind"] + if kind not in all_nodes: + all_nodes[kind] = { + "kind": kind, + "icon": node["icon"], + "properties": node.get("properties"), + "color": node.get("color", "#FFFFFF"), + "produced_by": asset["class"], + "incoming": [], + "outgoing": [], + } + + # TODO: This can probably be done more efficiently compared to iterating over assets twice + for asset in self.assets: + for edge in asset.get("edges", []): + start = edge["start"] + end = edge["end"] + if start in all_nodes and edge not in all_nodes[start]["outgoing"]: + all_nodes[start]["outgoing"].append(edge) + if end in all_nodes and edge not in all_nodes[end]["incoming"]: + all_nodes[end]["incoming"].append(edge) + + return all_nodes + + @property + def _edge_index(self) -> dict[str, dict]: + """Creates a mapping of unique edge kinds with their properties and all node pairs that use them. + + Each entry is enriched with start/end node icons from the node index so + templates do not need to perform separate lookups. + + Returns: + dict[str, dict]: Edge details keyed by edge kind. + """ + all_edges: dict[str, dict] = {} + node_index = self._node_index + + for asset in self.assets: + for edge in asset.get("edges", []): + kind = edge.get("kind") + if not kind: + continue + + if kind not in all_edges: + all_edges[kind] = { + "kind": kind, + "description": edge.get("description", ""), + "traversable": edge.get("traversable", False), + "instances": [], + "produced_by": [], + } + + instance = { + "start": edge["start"], + "end": edge["end"], + "start_icon": node_index.get(edge["start"], {}).get( + "icon", "circle" + ), + "end_icon": node_index.get(edge["end"], {}).get("icon", "circle"), + } + if instance not in all_edges[kind]["instances"]: + all_edges[kind]["instances"].append(instance) + + if asset["class"] not in all_edges[kind]["produced_by"]: + all_edges[kind]["produced_by"].append(asset["class"]) + + return all_edges + + @property + def node_templates(self) -> list[dict[str, str]]: + """Renders a node document for each node in the node index + + Returns: + list[dict[str, str]]: Each result has contains the node kind and rendered document. + """ + results = [] + template = self.env.get_template("node.md.j2") + index = self._node_index + for kind, node_data in index.items(): + node = {**node_data} + if node.get("properties"): + node["properties_table"] = self.render_class_table( + node["properties"], include_inherited=True + ) + + render = template.render( + name=self.name, + node=node, + node_index=index, + node_description=self._description_file_content("nodes", kind), + ) + results.append({"kind": kind, "render": render}) + return results + + @property + def edge_templates(self) -> list[dict[str, str]]: + """Renders an edge document for each unique edge kind in the edge index. + + Returns: + list[dict[str, str]]: Each entry contains the edge kind and rendered document. + """ + results = [] + template = self.env.get_template("edge.md.j2") + for kind, edge_data in self._edge_index.items(): + render = template.render( + name=self.name, + edge=edge_data, + edge_description=self._description_file_content("edges", kind), + ) + results.append({"kind": kind, "render": render}) + return results + + @staticmethod + def safe_create(base_dir: Path, target_dir: Path, parents: bool = False) -> None: + """Safely creates a directory within a base directory + + Args: + base_dir (Path): The base directory + target_dir (Path): The target directory to create + parents (bool, optional): Whether to create parent directories. Defaults to False. + + Raises: + ValueError: If the target directory is outside the base directory + """ + if target_dir.resolve().is_relative_to(base_dir.resolve()): + target_dir.mkdir(parents=parents, exist_ok=True) + + else: + raise ValueError(f"Detected path traversal attempt for {str(target_dir)}") + + @staticmethod + def safe_write(base_dir: Path, target_dir: Path, text: str) -> None: + """Safely writes text to a file within a base directory + + Args: + base_dir (Path): The base directory + target_dir (Path): The target file to write + text (str): The text to write to the file + + Raises: + ValueError: If the target file is outside the base directory + """ + if target_dir.resolve().is_relative_to(base_dir.resolve()): + target_dir.write_text(text) + + else: + raise ValueError(f"Detected path traversal attempt for {str(target_dir)}") + + def to_markdown(self, output_path: Path): + collection_path = output_path / "collection" + graph_path = output_path / "graph" + self.safe_create(self.base_docs_dir, (collection_path / "assets"), parents=True) + self.safe_create(self.base_docs_dir, (graph_path / "nodes"), parents=True) + self.safe_create(self.base_docs_dir, (graph_path / "edges"), parents=True) + + (collection_path / "overview.md").write_text(self.collector_template) + self.safe_write( + self.base_docs_dir, collection_path / "pipeline.md", self.pipeline_template + ) + + for asset in self.asset_templates: + resource_path = collection_path / "assets" / f"{asset['class']}.md" + self.safe_write(self.base_docs_dir, resource_path, asset["render"]) + + for node in self.node_templates: + node_path = graph_path / "nodes" / f"{node['kind']}.md" + self.safe_write(self.base_docs_dir, node_path, node["render"]) + + for edge in self.edge_templates: + edge_path = graph_path / "edges" / f"{edge['kind']}.md" + self.safe_write(self.base_docs_dir, edge_path, edge["render"]) diff --git a/src/openhound/docs/templates/asset.md.j2 b/src/openhound/docs/templates/mintlify/asset.md.j2 similarity index 78% rename from src/openhound/docs/templates/asset.md.j2 rename to src/openhound/docs/templates/mintlify/asset.md.j2 index 2b40aec..937d535 100644 --- a/src/openhound/docs/templates/asset.md.j2 +++ b/src/openhound/docs/templates/mintlify/asset.md.j2 @@ -8,14 +8,10 @@ This section describes the exported OpenGraph asset(s) for the {{ graph_resource | [{{ graph_resource.node.kind }}](../../graph/nodes/{{ graph_resource.node.kind}}.md) | :fontawesome-solid-{{ graph_resource.node.icon }}: | {% endif %} -{% if graph_resource.node and "properties" in graph_resource.node %} -??? Properties +{% if graph_resource.node and "node_properties_table" in graph_resource %} +## Node properties - ::: {{ graph_resource.node.properties }} - options: - show_docstring_attributes: true - inherited_members: true - members_order: source +{{ graph_resource.node_properties_table }} {% endif %} @@ -34,4 +30,4 @@ This section describes the exported OpenGraph asset(s) for the {{ graph_resource ## Resource attributes This section describes the data collected and available fields as present in the exported jsonl/parquet files. -::: {{ graph_resource.path }} +{{ graph_resource.resource_attributes_table }} diff --git a/src/openhound/docs/templates/class_table.md.j2 b/src/openhound/docs/templates/mintlify/class_table.md.j2 similarity index 100% rename from src/openhound/docs/templates/class_table.md.j2 rename to src/openhound/docs/templates/mintlify/class_table.md.j2 diff --git a/src/openhound/docs/templates/mintlify/edge.md.j2 b/src/openhound/docs/templates/mintlify/edge.md.j2 new file mode 100644 index 0000000..ac8e4f9 --- /dev/null +++ b/src/openhound/docs/templates/mintlify/edge.md.j2 @@ -0,0 +1,43 @@ +--- +tags: + - edge + - {% if edge.traversable -%}traversable{% else %}non-traversable{% endif %} +--- + +# {{ edge.kind }} + +This page describes the **{{ edge.kind }}** OpenGraph edge type as exported by the {{ name }} source. This edge was generated via the {% for asset in edge.produced_by %}[{{ asset }}](../../collection/assets/{{ asset }}.md){% if not loop.last %}, {% endif %}{% endfor %} assets. + +{% if edge_description %} +{{ edge_description }} +{% endif %} + +## Source and destination nodes + +The following node pairs are connected via the **{{ edge.kind }}** edge: + +| Start | Kind | End | +|-------|-----------|-------| +{% for instance in edge.instances -%} +| [{{ instance.start }}](../../graph/nodes/{{ instance.start }}.md) | {{ edge.kind }} | [{{ instance.end }}](../../graph/nodes/{{ instance.end }}.md) | +{% endfor %} + +{% if edge.instances %} +```mermaid +flowchart LR +{%- set seen_kinds = [] %} +{%- for instance in edge.instances %} +{%- if instance.start not in seen_kinds %} +{%- do seen_kinds.append(instance.start) %} + {{ instance.start }}["{{ instance.start }}"]:::bhNode +{%- endif %} +{%- if instance.end not in seen_kinds %} +{%- do seen_kinds.append(instance.end) %} + {{ instance.end }}["{{ instance.end }}"]:::bhNode +{%- endif %} +{%- endfor %} +{%- for instance in edge.instances %} + {{ instance.start }} -- {{ edge.kind }} --> {{ instance.end }} +{%- endfor %} +``` +{% endif %} diff --git a/src/openhound/docs/templates/test b/src/openhound/docs/templates/mintlify/node.md.j2 similarity index 54% rename from src/openhound/docs/templates/test rename to src/openhound/docs/templates/mintlify/node.md.j2 index d678f98..ad76815 100644 --- a/src/openhound/docs/templates/test +++ b/src/openhound/docs/templates/mintlify/node.md.j2 @@ -1,28 +1,42 @@ +--- +icon: fontawesome/solid/{{ node.icon }} +tags: + - node +--- + # :fontawesome-solid-{{ node.icon }}: {{ node.kind }} -This page describes the **{{ node.kind }}** OpenGraph node type as exported by the {{ name }} source. This node was generated from the [{{ node.produced_by }}](../assets/{{ node.produced_by }}.md) asset. +This page describes the **{{ node.kind }}** OpenGraph node type as exported by the {{ name }} source. This node was generated from the [{{ node.produced_by }}](../../collection/assets/{{ node.produced_by }}.md) asset. + +{% if node_description %} +{{ node_description }} +{% endif %} + +{% if node.properties_fields %} +## Node properties -## Produced by +{{ node.properties_table }} -The following OpenHound assets produce the **{{ node.kind }}** node. +{% endif %} +## Edges {% if node.outgoing or node.incoming %} ```mermaid flowchart LR -{%- set ns = namespace(seen_kinds=[]) %} -{%- seen_kinds.append(node.kind) %} - {{ node.kind }}["fa:fa-{{ node.icon }}"]:::bhNode +{%- set seen_kinds = [] %} +{% do seen_kinds.append(node.kind) %} + {{ node.kind }}["{{ node.kind }}"]:::bhNode {%- for edge in node.outgoing %} -{%- if edge.end not in ns.seen_kinds %} -{%- set ns.seen_kinds = ns.seen_kinds + [edge.end] %} - {{ edge.end }}["fa:fa-{{ edge.end_icon }}"]:::bhNode +{%- if edge.end not in seen_kinds %} +{%- do seen_kinds.append(edge.end) %} + {% if edge.end in node_index %}{{ edge.end }}["{{ edge.end }}"]:::bhNode{% endif %} {%- endif %} {%- endfor %} {%- for edge in node.incoming %} -{%- if edge.start not in ns.seen_kinds %} -{%- set ns.seen_kinds = ns.seen_kinds + [edge.start] %} - {{ edge.start }}["fa:fa-{{ edge.start_icon }}"]:::bhNode +{%- if edge.start not in seen_kinds %} +{%- do seen_kinds.append(edge.start) %} + {% if edge.start in node_index %}{{ edge.start }}["{{ edge.start }}"]:::bhNode{% endif %} {%- endif %} {%- endfor %} {%- for edge in node.outgoing %} @@ -34,15 +48,12 @@ flowchart LR ``` {% endif %} - -## Edges - ### Outgoing {% if node.outgoing %} | Start | End | Kind | Description | |-------|-----|------|-------------| {% for edge in node.outgoing -%} -| [{{ edge.start }}]({{ edge.start }}.md) | [{{ edge.end }}]({{ edge.end }}.md) | {{ edge.kind }} | {{ edge.description }} | +| [{{ edge.start }}]({{ edge.start }}.md) | [{{ edge.end }}]({{ edge.end }}.md) | [{{ edge.kind }}](../../graph/edges/{{ edge.kind }}.md) | {{ edge.description }} | {% endfor %} {% else %} No outgoing edges. @@ -53,7 +64,7 @@ No outgoing edges. | Start | End | Kind | Description | |-------|-----|------|-------------| {% for edge in node.incoming -%} -| [{{ edge.start }}]({{ edge.start }}.md) | [{{ edge.end }}]({{ edge.end }}.md) | {{ edge.kind }} | {{ edge.description }} | +| [{{ edge.start }}]({{ edge.start }}.md) | [{{ edge.end }}]({{ edge.end }}.md) | [{{ edge.kind }}](../../graph/edges/{{ edge.kind }}.md) | {{ edge.description }} | {% endfor %} {% else %} No incoming edges. diff --git a/src/openhound/docs/templates/mintlify/overview.md.j2 b/src/openhound/docs/templates/mintlify/overview.md.j2 new file mode 100644 index 0000000..42ba5d2 --- /dev/null +++ b/src/openhound/docs/templates/mintlify/overview.md.j2 @@ -0,0 +1,49 @@ +# Overview + +This page documents the `{{ name }}` OpenHound source, including its architecture, resource relationships and exported OpenGraph assets. The source extracts data from {{ name }} and transforms it into the standardized OpenGraph format. Use the visual diagram below to understand how nodes relate to each other, then explore the exported DLT [resources](resources.md) and invidual assets. + +## Visual overview +The diagram below shows the relationships between OpenGraph nodes and assets that are part of the {{ name }} source. This diagram is automatically generated by each resource/asset that is wrapped with the `@app.asset` decorator. + +```mermaid +flowchart LR +{% for r in graph_resources %} +{%- if r.node and r.node.icon and r.node.kind %} + {{ r.node.kind }}["{{ r.node.kind }}"]:::bhNode +{% endif %} +{%- endfor %} +{%- for r in graph_resources %} +{%- for e in r.get('edges', []) %} + {{ e.start }} -- {{e.kind}} --> {{e.end}} +{%- endfor %} +{%- endfor %} +``` + +## Node Icons +The following table shows the Font Awesome icon associated with each node type: + + +| Node Type | Icon | Font Awesome Class | +|------|-----------|-------------------| +{% for r in graph_resources -%} +{% if r.node and r.node.icon and r.node.kind -%} +| {{ r.node.kind }} | :fontawesome-solid-{{ r.node.icon }}: | `fa-{{ r.node.icon }}` | +{% endif -%} +{% endfor %} + + +## Exported OpenGraph assets + +The following table lists all OpenGraph assets produced by the {{ name }} source. Each asset represents a node or edge as part of the OpenGraph output. + +| Class | Description | Node | Edges | +|------|-------------|-------|-------| +{% for r in graph_resources -%} +|[{{ r.class }}](assets/{{ r.class }}.md) | {{ r.description }} | {% if r.node %}{{ r.node.kind }}{% endif %} | {{ r.get('edges', []) | length }} | +{% endfor %} + + +**Next Steps:** + +- Explore individual {{name}} [resources](pipeline.md) to see what data / API endpoints are used for extraction. +- Review asset schemas for detailed field information for each individual resource. diff --git a/src/openhound/docs/templates/pipeline.md.j2 b/src/openhound/docs/templates/mintlify/pipeline.md.j2 similarity index 100% rename from src/openhound/docs/templates/pipeline.md.j2 rename to src/openhound/docs/templates/mintlify/pipeline.md.j2 diff --git a/src/openhound/docs/templates/mkdocs/asset.md.j2 b/src/openhound/docs/templates/mkdocs/asset.md.j2 new file mode 100644 index 0000000..937d535 --- /dev/null +++ b/src/openhound/docs/templates/mkdocs/asset.md.j2 @@ -0,0 +1,33 @@ +# {{ graph_resource.class }} +This section describes the exported OpenGraph asset(s) for the {{ graph_resource.class }} class. Each resource wrapped with the @app.asset decorator will export documentation for an OpenGraph node, multiple edges or a combination of both. + +{% if graph_resource.node %} +## Node +| Name | Icon | +|------|------| +| [{{ graph_resource.node.kind }}](../../graph/nodes/{{ graph_resource.node.kind}}.md) | :fontawesome-solid-{{ graph_resource.node.icon }}: | +{% endif %} + +{% if graph_resource.node and "node_properties_table" in graph_resource %} +## Node properties + +{{ graph_resource.node_properties_table }} + +{% endif %} + +{% if graph_resource.edges %} +## Edges + +| Start | End | Kind | Description | +|-------|-----|------|-------------| +{% for edge in graph_resource.edges -%} +| [{{ edge.start }}](../../graph/nodes/{{ edge.start }}.md) | [{{ edge.end }}](../../graph/nodes/{{ edge.end }}.md) | [{{ edge.kind }}](../../graph/edges/{{ edge.kind }}.md) | {{ edge.description }} | +{% endfor %} +{% endif %} + + + +## Resource attributes +This section describes the data collected and available fields as present in the exported jsonl/parquet files. + +{{ graph_resource.resource_attributes_table }} diff --git a/src/openhound/docs/templates/mkdocs/class_table.md.j2 b/src/openhound/docs/templates/mkdocs/class_table.md.j2 new file mode 100644 index 0000000..e145939 --- /dev/null +++ b/src/openhound/docs/templates/mkdocs/class_table.md.j2 @@ -0,0 +1,5 @@ +| Field | Type | Description | +|-------|------|-------------| +{% for documented_field in fields -%} +| `{{ escape_markdown_cell(documented_field.name) }}` | `{{ escape_markdown_cell(documented_field.type_name) }}` | {{ escape_markdown_cell(documented_field.description or "-") }} | +{% endfor %} diff --git a/src/openhound/docs/templates/edge.md.j2 b/src/openhound/docs/templates/mkdocs/edge.md.j2 similarity index 95% rename from src/openhound/docs/templates/edge.md.j2 rename to src/openhound/docs/templates/mkdocs/edge.md.j2 index 3c495f7..9f05484 100644 --- a/src/openhound/docs/templates/edge.md.j2 +++ b/src/openhound/docs/templates/mkdocs/edge.md.j2 @@ -8,7 +8,9 @@ tags: This page describes the **{{ edge.kind }}** OpenGraph edge type as exported by the {{ name }} source. This edge was generated via the {% for asset in edge.produced_by %}[{{ asset }}](../../collection/assets/{{ asset }}.md){% if not loop.last %}, {% endif %}{% endfor %} assets. ---8<-- "{{ edge.kind }}.md" +{% if edge_description %} +{{ edge_description }} +{% endif %} ## Source and destination nodes diff --git a/src/openhound/docs/templates/node.md.j2 b/src/openhound/docs/templates/mkdocs/node.md.j2 similarity index 81% rename from src/openhound/docs/templates/node.md.j2 rename to src/openhound/docs/templates/mkdocs/node.md.j2 index 758269b..d69c245 100644 --- a/src/openhound/docs/templates/node.md.j2 +++ b/src/openhound/docs/templates/mkdocs/node.md.j2 @@ -8,17 +8,14 @@ tags: This page describes the **{{ node.kind }}** OpenGraph node type as exported by the {{ name }} source. This node was generated from the [{{ node.produced_by }}](../../collection/assets/{{ node.produced_by }}.md) asset. ---8<-- "{{ node.kind }}.md" +{% if node_description %} +{{ node_description }} +{% endif %} -{% if node.properties %} +{% if node.properties_fields %} ## Node properties -{#??? Properties#} -::: {{ node.properties }} - options: - show_docstring_attributes: true - inherited_members: true - members_order: source +{{ node.properties_table }} {% endif %} @@ -56,7 +53,7 @@ flowchart LR | Start | End | Kind | Description | |-------|-----|------|-------------| {% for edge in node.outgoing -%} -| [{{ edge.start }}]({{ edge.start }}.md) | [{{ edge.end }}]({{ edge.end }}.md) | [{{ edge.kind }}](../../graph/edges/{{ edge.kind }}.md) | {{ edge.description }} | +| [{{ edge.start }}]({{ edge.start }}.md) | [{{ edge.end }}]({{ edge.end }}.md) | [{{ edge.kind }}](../edges/{{ edge.kind }}.md) | {{ edge.description }} | {% endfor %} {% else %} No outgoing edges. @@ -67,7 +64,7 @@ No outgoing edges. | Start | End | Kind | Description | |-------|-----|------|-------------| {% for edge in node.incoming -%} -| [{{ edge.start }}]({{ edge.start }}.md) | [{{ edge.end }}]({{ edge.end }}.md) | [{{ edge.kind }}](../../graph/edges/{{ edge.kind }}.md) | {{ edge.description }} | +| [{{ edge.start }}]({{ edge.start }}.md) | [{{ edge.end }}]({{ edge.end }}.md) | [{{ edge.kind }}](../edges/{{ edge.kind }}.md) | {{ edge.description }} | {% endfor %} {% else %} No incoming edges. diff --git a/src/openhound/docs/templates/overview.md.j2 b/src/openhound/docs/templates/mkdocs/overview.md.j2 similarity index 96% rename from src/openhound/docs/templates/overview.md.j2 rename to src/openhound/docs/templates/mkdocs/overview.md.j2 index d8486dc..c45cb87 100644 --- a/src/openhound/docs/templates/overview.md.j2 +++ b/src/openhound/docs/templates/mkdocs/overview.md.j2 @@ -1,6 +1,6 @@ # Overview -This page documents the `{{ name }}` OpenHound source, including its architecture, resource relationships and exported OpenGraph assets. The source extracts data from {{ name }} and transforms it into the standardized OpenGraph format. Use the visual diagram below to understand how nodes relate to each other, then explore the exported DLT [resources](resources.md) and invidual assets. +This page documents the `{{ name }}` OpenHound source, including its architecture, resource relationships and exported OpenGraph assets. The source extracts data from {{ name }} and transforms it into the standardized OpenGraph format. Use the visual diagram below to understand how nodes relate to each other, then explore the exported DLT [resources](pipeline.md) and invidual assets. ## Visual overview The diagram below shows the relationships between OpenGraph nodes and assets that are part of the {{ name }} source. This diagram is automatically generated by each resource/asset that is wrapped with the `@app.asset` decorator. diff --git a/src/openhound/docs/templates/mkdocs/pipeline.md.j2 b/src/openhound/docs/templates/mkdocs/pipeline.md.j2 new file mode 100644 index 0000000..eadc54e --- /dev/null +++ b/src/openhound/docs/templates/mkdocs/pipeline.md.j2 @@ -0,0 +1,67 @@ +# Pipeline + +This page lists all pipeline functions that make up the `{{ name }}` collector. + +## Sources +Sources are the top-level entry points that define the configuration and which resources/transformers to process. + +This section includes the API reference for the source function, including all parameters required to run the pipeline. The parameters may also include references to credentials or settings stored inside your DLT configuration, which can be loaded via environment variables or the .dlt/secrets.toml file. + +!!! info "Configuration Required" + Some source parameters may require configuration values that should be set via: + + - **Environment variables** (recommended for secrets); + - **`.dlt/config.toml`** for non-sensitive configuratio;n + - **`.dlt/secrets.toml`** for API keys, tokens, and credentials + + See the [DLT credentials documentation](https://dlthub.com/docs/walkthroughs/add_credentials) for details on managing credentials. + + +{% if sources %} +{% for source in sources | sort(attribute="name") -%} +::: {{ source.module_path }} + options: + show_source_link: true + show_root_heading: true + heading_level: 3 +{% endfor %} +{% else %} +*No source functions found.* +{% endif %} + +## Resources + +Resources are the individual extraction functions which are part of a DLT pipeline. Within the context of OpenHound, resources are functions that call individual API endpoints to fetch users, computers, roles etc. Each `@app.resource` represents a specific data item that can be extracted and loaded. + +**As part of this collector:** + +- Some resources return OpenGraph assets that can be loaded directly into BloodHound. These are wrapped with the @app.asset decorator; +- Other resources provide supplementary data used for enrichment, transformations or other supporting operations + +{% if resources %} +{% for resource in resources | sort(attribute="name") -%} +::: {{ resource.module_path }} + options: + show_source_link: true + show_root_heading: true + heading_level: 3 +{% endfor %} +{% else %} +*No resource functions found.* +{% endif %} + +## Transformers + +Transformers are similar to resources but are used for data transformation and enrichment. They typically receive the output of a resource and yield additional records. An example could be fetching user details via a seperate API endpoint based on a user ID returned by a resource. + +{% if transformers %} +{% for transformer in transformers | sort(attribute="name") -%} +::: {{ transformer.module_path }} + options: + show_source_link: true + show_root_heading: true + heading_level: 3 +{% endfor %} +{% else %} +*No transformer functions found.* +{% endif %} diff --git a/uv.lock b/uv.lock index 7ee7eea..34909ce 100644 --- a/uv.lock +++ b/uv.lock @@ -1286,7 +1286,7 @@ dev = [ { name = "ruff", specifier = ">=0.15.4" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250915" }, { name = "types-requests", specifier = ">=2.33.0.20260408" }, - { name = "zensical", specifier = ">=0.0.23" }, + { name = "zensical", specifier = ">=0.0.38" }, ] [[package]] @@ -2110,6 +2110,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, ] +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + [[package]] name = "tomlkit" version = "0.14.0" @@ -2311,7 +2347,7 @@ wheels = [ [[package]] name = "zensical" -version = "0.0.32" +version = "0.0.38" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -2320,19 +2356,20 @@ dependencies = [ { name = "pygments" }, { name = "pymdown-extensions" }, { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/94/4a49ca9329136445f4111fda60e4bfcbe68d95e18e9aa02e4606fba5df4a/zensical-0.0.32.tar.gz", hash = "sha256:0f857b09a2b10c99202b3712e1ffc4d1d1ffa4c7c2f1aa0fafb1346b2d8df604", size = 3891955, upload-time = "2026-04-07T11:41:29.203Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/e1/dd03762447f1c2a4c8aff08e8f047ec17c73421714a0600ef71c361a5934/zensical-0.0.32-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7ed181c76c03fec4c2dd5db207810044bf9c3fa87097fbdbabd633661e20fc70", size = 12416474, upload-time = "2026-04-07T11:40:55.888Z" }, - { url = "https://files.pythonhosted.org/packages/f5/a6/2f1babb00842c6efa5ae755b3ab414e4688ae8e47bdd2e785c0c37ef625d/zensical-0.0.32-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8cde82bf256408f75ae2b07bffcaac7d080b6aad5f7acf210c438cb7413c3081", size = 12292801, upload-time = "2026-04-07T11:40:59.648Z" }, - { url = "https://files.pythonhosted.org/packages/2d/f1/d32706de06fd30fb07ae514222a79dd17d4578cd1634e5b692e0c790a61e/zensical-0.0.32-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60e60e2358249b2a2c5e1c5c04586d8dbba27e577441cc9dd32fe8d879c6951e", size = 12658847, upload-time = "2026-04-07T11:41:02.347Z" }, - { url = "https://files.pythonhosted.org/packages/e7/42/a3daf4047c86382749a59795c4e7acd59952b4f6f37f329cd2d41cc37a0f/zensical-0.0.32-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ec79b4304009138e7a38ebe24e8a8e9dbc15d38922185f8a84470a7757d7b73f", size = 12604777, upload-time = "2026-04-07T11:41:05.227Z" }, - { url = "https://files.pythonhosted.org/packages/59/11/4af61d3fb07713cd3f77981c1b3017a60c2b210b36f1b04353f9116d03ca/zensical-0.0.32-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fc92fa7d0860ec6d95426a5f545cfc5493c60f8ab44fcc11611a4251f34f1b70", size = 12956242, upload-time = "2026-04-07T11:41:07.58Z" }, - { url = "https://files.pythonhosted.org/packages/8c/34/e9b5f4376bbf460f8c07a77af59bd169c7c68ed719a074e6667ba41109f8/zensical-0.0.32-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07f69019396060e310c9c3b18747ce8982ad56d67fbab269b61e74a6a5bdcb4a", size = 12701954, upload-time = "2026-04-07T11:41:10.532Z" }, - { url = "https://files.pythonhosted.org/packages/d2/43/a52e5dcb324f38a1d22f7fafd4eec273385d04de52a7ab5ac7b444cf2bdc/zensical-0.0.32-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d096c9ed20a48e5ff095eca218eef94f67e739cdf0abf7e1f7e232e78f6d980c", size = 12835464, upload-time = "2026-04-07T11:41:13.152Z" }, - { url = "https://files.pythonhosted.org/packages/a7/95/bede89ecb4932bbd29db7b61bf530a962aed09d3a8d5aa71a64af1d4920f/zensical-0.0.32-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:bf5576b7154bde18cebd9a7b065d3ab8b334c6e73d5b2e83abe2b17f9d00a992", size = 12876574, upload-time = "2026-04-07T11:41:16.085Z" }, - { url = "https://files.pythonhosted.org/packages/9e/e8/9b25fda22bf729ca2598cc42cefe9b20e751d12d23e35c70ea0c7939d20a/zensical-0.0.32-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:f33905a1e0b03a2ad548554a157b7f7c398e6f41012d1e755105ae2bc60eab8a", size = 13022702, upload-time = "2026-04-07T11:41:18.947Z" }, - { url = "https://files.pythonhosted.org/packages/f6/35/0c6d0b57187bd470a05e8a391c0edd1d690eb429e12b9755c99cf60a370e/zensical-0.0.32-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0a73a53b1dd41fd239875a3cb57c4284747989c45b6933f18e9b51f1b5f3d8ef", size = 12975593, upload-time = "2026-04-07T11:41:21.436Z" }, - { url = "https://files.pythonhosted.org/packages/ee/2d/4e88bcefc33b7af22f0637fd002d3cf5384e8354f0a7f8a9dbfcd40cfa24/zensical-0.0.32-cp310-abi3-win32.whl", hash = "sha256:f8cb579bdb9b56f1704b93f4e17b42895c8cb466e8eec933fbe0153b5b1e3459", size = 12012163, upload-time = "2026-04-07T11:41:23.975Z" }, - { url = "https://files.pythonhosted.org/packages/8a/ae/a80a2f15fd10201fe3dfd6b5cdf85351165f820cf5b29e3c3b24092c158c/zensical-0.0.32-cp310-abi3-win_amd64.whl", hash = "sha256:6d662f42b5d0eadfac6d281e9d86574bc7a9f812f1ed496335d15f2d581d4b28", size = 12205948, upload-time = "2026-04-07T11:41:27.056Z" }, + { name = "tomli" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/3d/96301349abd6e425b580f33474a51a5b6d68332ed538b8b6000497883794/zensical-0.0.38.tar.gz", hash = "sha256:e6fbf98dd851f5772d84648443e44fc8d8194ba0e09ec75c267fa033f6a0e43c", size = 3912956, upload-time = "2026-04-30T12:05:02.704Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/4d/6c7111f9885dd128b7caf742a160041b01d53bd61e501b8ec19c597fe699/zensical-0.0.38-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c1d498eecfba2d876ef6fb535fe867af5d752ea38551faab4bc70fd5f25ed5aa", size = 12666775, upload-time = "2026-04-30T12:04:21.522Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8a/d1a8359b5308cf4b0859741acbc7e5cd90641d1e4591e3bd3ca688bb8038/zensical-0.0.38-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:edb2e54f1d299a0b5b177fc55d15e198ccb0bf143991bb2f4b2d8db0a6c3b932", size = 12528871, upload-time = "2026-04-30T12:04:25.419Z" }, + { url = "https://files.pythonhosted.org/packages/34/8b/6a47e5065bd9baf161785f1afd2c6e67dd3a7eafccb7ed06e0c7efd7b424/zensical-0.0.38-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8adc9e87d2d5921d9aa4204c4f7488b6349efd57916680d4905414e6461c942b", size = 12925558, upload-time = "2026-04-30T12:04:29.073Z" }, + { url = "https://files.pythonhosted.org/packages/62/2a/62338132326dbc81bfd45d3ba47440dbd689be6c2cccf75f0005c6d0183d/zensical-0.0.38-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9576d21e3d5d6d6208df0873231838a3e42f05ba95316e4129df26a20edb8226", size = 12887161, upload-time = "2026-04-30T12:04:32.118Z" }, + { url = "https://files.pythonhosted.org/packages/04/b3/f4f0af1eb6caf2d163fb9ba97da4592c74f26fe77309093bec35d8dbab5c/zensical-0.0.38-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d649045e59b6ecb0f543fddeed5b0dc4dab3fdeb0dae791d71b2be29335dd603", size = 13252488, upload-time = "2026-04-30T12:04:35.558Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e4/d5329e20c9417ca4789150cf78c994e2489c0c8fd92f10d93fe13c9d71da/zensical-0.0.38-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:196aa6ffaf2e80a173233e5e639227e59437a2dc31849051901a9456960f5f1a", size = 12955366, upload-time = "2026-04-30T12:04:39.159Z" }, + { url = "https://files.pythonhosted.org/packages/38/26/11ca657164a2ca9347ffe665b57f5e788b628b6f21e7cf171cda7295a730/zensical-0.0.38-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3a0d173f4402a6201d990f05cb766aad872f222fffd9022d42421b331f69c60c", size = 13101610, upload-time = "2026-04-30T12:04:42.531Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c7/0247c1efff36914b8a720dbe4accc5e1065d4ae986a81c71fb69cb1cc3e8/zensical-0.0.38-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c6056675c5f9e2e00afe6770232213e7dcf07e7e87a5e278d0dd7dbbd8b52316", size = 13159871, upload-time = "2026-04-30T12:04:46.169Z" }, + { url = "https://files.pythonhosted.org/packages/b5/ec/5ff0d64e58f2f498ba1696de3dccf147aec024f374ece4ae55f1313ad3c2/zensical-0.0.38-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:e447ca87827b7db7802a4b071247fb72968ab482f611eb8a951917f63b7784b2", size = 13311076, upload-time = "2026-04-30T12:04:49.826Z" }, + { url = "https://files.pythonhosted.org/packages/78/80/8bd9054e15ac992c911a87a9d2651aa3468bc370ad97084f9902f2c9f7e0/zensical-0.0.38-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b913573ec99171534f51f0a5ab2032eee5416981ba2fe502601c5ac5a6da898", size = 13237935, upload-time = "2026-04-30T12:04:53.104Z" }, + { url = "https://files.pythonhosted.org/packages/63/75/d81ca979bc770c0d678717687b9b9fdf1e3afc0e3d52b05092a0391866c8/zensical-0.0.38-cp310-abi3-win32.whl", hash = "sha256:a2eebc767037943f93fa6f5b74f409ad2ca53d1eda7776092ebb455d7b42eb67", size = 12228161, upload-time = "2026-04-30T12:04:56.641Z" }, + { url = "https://files.pythonhosted.org/packages/14/09/52965dcb9bbae6883a1981a23d926b6410fdf61bd83f399fc9acda5ccb98/zensical-0.0.38-cp310-abi3-win_amd64.whl", hash = "sha256:e91412a38c4a7099e498b656eaf858b1f9d6c3b09dab05a4bdc65a6c3b9a45a1", size = 12469561, upload-time = "2026-04-30T12:04:59.632Z" }, ] diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 0000000..d981377 --- /dev/null +++ b/zensical.toml @@ -0,0 +1,46 @@ +[project] +site_name = "OpenHound" +description = "OpenGraph collectors using OpenHound" +site_url = "https://specterops.github.io/OpenHound" +extra_css = ["stylesheets/mermaid-custom.css"] +extra_javascript = ["javascript/custom.mjs", "javascript/mermaid.mjs"] +repo_url = "https://github.com/specterops/openhound" +repo_name = "specterops/openhound" + +[project.theme.icon] +repo = "fontawesome/brands/github" + + +[project.theme] +features = [ + "navigation.tracking" +] + +[project.plugins.mkdocstrings.handlers.python.options] +heading_level = 3 +show_root_toc_entry = false + +[[project.plugins.mkdocstrings.handlers.python.options.extensions]] +[project.plugins.mkdocstrings.handlers.python.options.extensions.griffe_fieldz] +include_inherited = true +include_private = false +add_fields_to = "docstring-attributes" +remove_fields_from_members = false +strip_annotated = false + +[project.markdown_extensions.toc] +permalink = true + +[project.markdown_extensions.admonition] + +[project.markdown_extensions.pymdownx.details] + +[project.markdown_extensions.pymdownx.emoji] +emoji_index = "zensical.extensions.emoji.twemoji" +emoji_generator = "zensical.extensions.emoji.to_svg" + + +[project.markdown_extensions.pymdownx.superfences] +custom_fences = [ + { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" } +]