diff --git a/apis/connectedendpoints/composition.yaml b/apis/connectedendpoints/composition.yaml new file mode 100644 index 0000000..8cc38ae --- /dev/null +++ b/apis/connectedendpoints/composition.yaml @@ -0,0 +1,16 @@ +# Composition for ConnectedEndpoints: validate this fragment and report on its own status. +# Uses the same function image as Fabric and Device, which dispatches on the +# composite kind. +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: connectedendpoints-avd +spec: + compositeTypeRef: + apiVersion: avd.netclab.dev/v1alpha1 + kind: ConnectedEndpoints + mode: Pipeline + pipeline: + - step: validate-input + functionRef: + name: netclab-function-avd diff --git a/apis/connectedendpoints/xrd.yaml b/apis/connectedendpoints/xrd.yaml new file mode 100644 index 0000000..c4affc6 --- /dev/null +++ b/apis/connectedendpoints/xrd.yaml @@ -0,0 +1,103 @@ +# CompositeResourceDefinition for what connects to the fabric. +# +# Carries `connected_endpoints_keys.key` lists -- `servers`, `firewalls` and the +# rest -- plus `port_profiles` and `network_ports`. Its own kind for the same +# reason as NetworkServices: whoever attaches servers is rarely whoever owns the +# fabric, and RBAC is granted per kind. +# +# Named ConnectedEndpoints, not Endpoints, because Endpoints is a core/v1 kind +# and `kubectl get endpoints` would become ambiguous. +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: connectedendpoints.avd.netclab.dev +spec: + scope: Namespaced + group: avd.netclab.dev + names: + kind: ConnectedEndpoints + plural: connectedendpoints + categories: + - crossplane + - netclab + defaultCompositionRef: + name: connectedendpoints-avd + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + design: + type: object + description: >- + The endpoints this input contributes. Structurally open: + the top-level key names come from connected_endpoints_keys, + so they are decided by the document's own content and no + OpenAPI schema can describe them. + x-kubernetes-preserve-unknown-fields: true + appliesTo: + type: object + description: >- + Devices that see this input. Defaults to every device in the + fabric. + properties: + all: + type: boolean + nodeSets: + type: array + description: Devices declared by the named NodeSets. + items: + type: string + hosts: + type: array + description: Devices named directly. + items: + type: string + matchHostnames: + type: array + description: >- + Regular expressions matched against device names. Same + semantics as AVD's own default_node_types, which matches + hostnames this way: the pattern is anchored for you, so + `dc1-leaf.*` matches the whole name. A pattern matching + no device is an error, not an empty set -- a pattern is + silent about both matching nothing and matching too much, + and the render is pushed as a full config replacement. + items: + type: string + required: + - design + status: + type: object + properties: + keys: + type: array + description: Top-level eos_designs keys this input contributes. + items: + type: string + devices: + type: array + description: >- + Devices this input resolved to. Reported rather than left to + be inferred, because appliesTo may be a pattern and a pattern + does not say what it matched. + items: + type: string + validation: + type: object + description: >- + Result of validating this fragment. Reported here rather than + on the Fabric so the team that owns this object sees its own + error on its own object. + properties: + ok: + type: boolean + message: + type: string + x-kubernetes-preserve-unknown-fields: true diff --git a/apis/device/xrd.yaml b/apis/device/xrd.yaml index 8c85079..1f46d8c 100644 --- a/apis/device/xrd.yaml +++ b/apis/device/xrd.yaml @@ -19,6 +19,9 @@ spec: names: kind: Device plural: devices + categories: + - crossplane + - netclab # Devices are composed by the Fabric function without an explicit composition # selector; pin the default so selection is deterministic. defaultCompositionRef: diff --git a/apis/fabric/xrd.yaml b/apis/fabric/xrd.yaml index 8f02562..441ebfd 100644 --- a/apis/fabric/xrd.yaml +++ b/apis/fabric/xrd.yaml @@ -22,6 +22,9 @@ spec: names: kind: Fabric plural: fabrics + categories: + - crossplane + - netclab defaultCompositionRef: name: fabric-avd versions: @@ -46,8 +49,45 @@ spec: description: >- Fabric-wide AVD eos_designs input document (node-type blocks, default_node_types, tenants, connected endpoints, ...). - Validated by pyavd; violations reported on status. + Validated by pyavd; violations reported on status. A fabric + may instead be assembled from input objects listed in + spec.requires, in which case this carries only what is + fabric-wide and the inputs carry the rest. x-kubernetes-preserve-unknown-fields: true + requires: + type: array + description: >- + The input objects composing this fabric. Only objects listed + here take part in the render, however they are labelled -- + which is what makes the rendered document a function of this + Fabric rather than of whatever else exists in the namespace, + and it matters because the render is pushed as a full config + replacement. An entry that does not resolve leaves the Fabric + not ready, naming the object it could not find. List order is + the merge order: a later input replaces an earlier one's keys, + which is how a setting is narrowed to part of the fabric. + items: + type: object + properties: + kind: + type: string + enum: + - NodeSet + - NetworkServices + - ConnectedEndpoints + - Settings + - Secret + name: + type: string + minLength: 1 + namespace: + type: string + description: >- + Namespace holding the object. Defaults to the Fabric's + own namespace. + required: + - kind + - name push: type: object description: >- diff --git a/apis/networkservices/composition.yaml b/apis/networkservices/composition.yaml new file mode 100644 index 0000000..81dc745 --- /dev/null +++ b/apis/networkservices/composition.yaml @@ -0,0 +1,16 @@ +# Composition for NetworkServices: validate this fragment and report on its own status. +# Uses the same function image as Fabric and Device, which dispatches on the +# composite kind. +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: networkservices-avd +spec: + compositeTypeRef: + apiVersion: avd.netclab.dev/v1alpha1 + kind: NetworkServices + mode: Pipeline + pipeline: + - step: validate-input + functionRef: + name: netclab-function-avd diff --git a/apis/networkservices/xrd.yaml b/apis/networkservices/xrd.yaml new file mode 100644 index 0000000..2a21b7f --- /dev/null +++ b/apis/networkservices/xrd.yaml @@ -0,0 +1,104 @@ +# CompositeResourceDefinition for a fabric's network services. +# +# Carries the tenants -- `network_services_keys.name`, `tenants` by default -- +# with their VRFs, SVIs and L2 VLANs. Its own kind because network services are +# owned by whoever runs the services, not by whoever owns the spines, and RBAC +# is granted per kind. +# +# It normally applies to every device: AVD decides per node which services land +# there, through `filter.tenants` and `filter.tags` on the node, so this input +# does not have to be scoped by hand. +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: networkservices.avd.netclab.dev +spec: + scope: Namespaced + group: avd.netclab.dev + names: + kind: NetworkServices + plural: networkservices + categories: + - crossplane + - netclab + defaultCompositionRef: + name: networkservices-avd + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + design: + type: object + description: >- + The tenants this input contributes. Structurally open: + the top-level key names come from network_services_keys, so + they are decided by the document's own content and no OpenAPI + schema can describe them. + x-kubernetes-preserve-unknown-fields: true + appliesTo: + type: object + description: >- + Devices that see this input. Defaults to every device in the + fabric. + properties: + all: + type: boolean + nodeSets: + type: array + description: Devices declared by the named NodeSets. + items: + type: string + hosts: + type: array + description: Devices named directly. + items: + type: string + matchHostnames: + type: array + description: >- + Regular expressions matched against device names. Same + semantics as AVD's own default_node_types, which matches + hostnames this way: the pattern is anchored for you, so + `dc1-leaf.*` matches the whole name. A pattern matching + no device is an error, not an empty set -- a pattern is + silent about both matching nothing and matching too much, + and the render is pushed as a full config replacement. + items: + type: string + required: + - design + status: + type: object + properties: + keys: + type: array + description: Top-level eos_designs keys this input contributes. + items: + type: string + devices: + type: array + description: >- + Devices this input resolved to. Reported rather than left to + be inferred, because appliesTo may be a pattern and a pattern + does not say what it matched. + items: + type: string + validation: + type: object + description: >- + Result of validating this fragment. Reported here rather than + on the Fabric so the team that owns this object sees its own + error on its own object. + properties: + ok: + type: boolean + message: + type: string + x-kubernetes-preserve-unknown-fields: true diff --git a/apis/nodeset/composition.yaml b/apis/nodeset/composition.yaml new file mode 100644 index 0000000..936031d --- /dev/null +++ b/apis/nodeset/composition.yaml @@ -0,0 +1,16 @@ +# Composition for NodeSet: validate this fragment and report on its own status. +# Uses the same function image as Fabric and Device, which dispatches on the +# composite kind. +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: nodeset-avd +spec: + compositeTypeRef: + apiVersion: avd.netclab.dev/v1alpha1 + kind: NodeSet + mode: Pipeline + pipeline: + - step: validate-input + functionRef: + name: netclab-function-avd diff --git a/apis/nodeset/xrd.yaml b/apis/nodeset/xrd.yaml new file mode 100644 index 0000000..6b4f284 --- /dev/null +++ b/apis/nodeset/xrd.yaml @@ -0,0 +1,126 @@ +# CompositeResourceDefinition for a set of fabric nodes. +# +# A NodeSet carries one node-type block's share of the eos_designs input -- the +# shape of a single group_vars file: `.defaults`, `.nodes[]`, +# `.node_groups[]`. One NodeSet is what `DC1_L3_LEAVES.yml` is in an AVD +# inventory, which is normally several `node_groups`, not one. +# +# It is the only kind that brings devices into existence. `spec.declares` is the +# fabric's device list, and there is no second list: a node named in a block the +# fabric does not declare is not a device. That is deliberate -- AVD inventories +# keep the inventory and the model as two lists that may disagree, and a device +# list in two places with nothing reconciling them is how `spec.push.hosts` +# went wrong. +# +# `spec.appliesTo` is a separate question from what a NodeSet declares: it says +# which devices *see* this input. The two coincide in simple topologies and +# diverge in a 5-stage CLOS, where a DC's super_spine block names four devices +# but is visible to every device of that DC. +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: nodesets.avd.netclab.dev +spec: + scope: Namespaced + group: avd.netclab.dev + names: + kind: NodeSet + plural: nodesets + categories: + - crossplane + - netclab + defaultCompositionRef: + name: nodeset-avd + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + design: + type: object + description: >- + This input's share of the eos_designs document, in the shape + of a group_vars file. Structurally open: the top-level key + names come from node_type_keys, so they are decided by the + document's own content and no OpenAPI schema can describe + them. Validated by pyavd; violations reported on status. + x-kubernetes-preserve-unknown-fields: true + declares: + type: array + description: >- + Devices this NodeSet brings into the fabric. Defaults to the + devices its node-type blocks name. Set it explicitly to + declare devices no block mentions -- their node type then + comes from default_node_types -- or to exclude a node the + blocks name but the fabric does not contain. + items: + type: string + appliesTo: + type: object + description: >- + Devices that see this input. Defaults to every device in the + fabric. + properties: + all: + type: boolean + nodeSets: + type: array + description: Devices declared by the named NodeSets. + items: + type: string + hosts: + type: array + description: Devices named directly. + items: + type: string + matchHostnames: + type: array + description: >- + Regular expressions matched against device names. Same + semantics as AVD's own default_node_types, which matches + hostnames this way: the pattern is anchored for you, so + `dc1-leaf.*` matches the whole name. A pattern matching + no device is an error, not an empty set -- a pattern is + silent about both matching nothing and matching too much, + and the render is pushed as a full config replacement. + items: + type: string + required: + - design + status: + type: object + properties: + keys: + type: array + description: Top-level eos_designs keys this input contributes. + items: + type: string + devices: + type: array + description: >- + Devices this input resolved to. Reported rather than left to + be inferred, because appliesTo may be a pattern and a pattern + does not say what it matched. + items: + type: string + deviceCount: + type: integer + description: Devices this input declares. + validation: + type: object + description: >- + Result of validating this fragment. Reported here rather than + on the Fabric so the team that owns this object sees its own + error on its own object. + properties: + ok: + type: boolean + message: + type: string + x-kubernetes-preserve-unknown-fields: true diff --git a/apis/settings/composition.yaml b/apis/settings/composition.yaml new file mode 100644 index 0000000..e3fd526 --- /dev/null +++ b/apis/settings/composition.yaml @@ -0,0 +1,16 @@ +# Composition for Settings: validate this fragment and report on its own status. +# Uses the same function image as Fabric and Device, which dispatches on the +# composite kind. +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: settings-avd +spec: + compositeTypeRef: + apiVersion: avd.netclab.dev/v1alpha1 + kind: Settings + mode: Pipeline + pipeline: + - step: validate-input + functionRef: + name: netclab-function-avd diff --git a/apis/settings/xrd.yaml b/apis/settings/xrd.yaml new file mode 100644 index 0000000..59c3c52 --- /dev/null +++ b/apis/settings/xrd.yaml @@ -0,0 +1,103 @@ +# CompositeResourceDefinition for fabric settings at any scope. +# +# Everything that is not node-scoped and not a service: routing protocol choices, +# `bgp_peer_groups`, `default_interfaces`, `aaa_settings`, `dns_settings`, +# `ntp_settings`, `management_eapi`. Fabric-wide settings live on the Fabric +# itself; this kind carries the same keys narrowed to part of the fabric, which +# is what an AVD inventory expresses by putting them in a DC or role group. +# +# The key categories across the input kinds are a convention, not a partition the +# schema could enforce -- eos_designs' top-level key names come from its own +# content. What the kinds separate is ownership. +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: settings.avd.netclab.dev +spec: + scope: Namespaced + group: avd.netclab.dev + names: + kind: Settings + plural: settings + categories: + - crossplane + - netclab + defaultCompositionRef: + name: settings-avd + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + design: + type: object + description: >- + The settings this input contributes, in the shape of a + group_vars file. Structurally open, like every input's design. + x-kubernetes-preserve-unknown-fields: true + appliesTo: + type: object + description: >- + Devices that see this input. Defaults to every device in the + fabric. + properties: + all: + type: boolean + nodeSets: + type: array + description: Devices declared by the named NodeSets. + items: + type: string + hosts: + type: array + description: Devices named directly. + items: + type: string + matchHostnames: + type: array + description: >- + Regular expressions matched against device names. Same + semantics as AVD's own default_node_types, which matches + hostnames this way: the pattern is anchored for you, so + `dc1-leaf.*` matches the whole name. A pattern matching + no device is an error, not an empty set -- a pattern is + silent about both matching nothing and matching too much, + and the render is pushed as a full config replacement. + items: + type: string + required: + - design + status: + type: object + properties: + keys: + type: array + description: Top-level eos_designs keys this input contributes. + items: + type: string + devices: + type: array + description: >- + Devices this input resolved to. Reported rather than left to + be inferred, because appliesTo may be a pattern and a pattern + does not say what it matched. + items: + type: string + validation: + type: object + description: >- + Result of validating this fragment. Reported here rather than + on the Fabric so the team that owns this object sees its own + error on its own object. + properties: + ok: + type: boolean + message: + type: string + x-kubernetes-preserve-unknown-fields: true diff --git a/function/fn.py b/function/fn.py index 53897b5..0ccc657 100644 --- a/function/fn.py +++ b/function/fn.py @@ -29,7 +29,16 @@ from .engine import ( InputValidationError, device_roles_from_design, - render_fabric_design, + hostnames_from_design, + render_structured_configs, +) +from .kinds import ( + KINDS, + Input, + hosts_in_blocks, + overwrites, + resolve, + unmatched_patterns, ) API_VERSION = "avd.netclab.dev/v1alpha1" @@ -94,10 +103,41 @@ async def RunFunction( # noqa: N802 (gRPC method name) self._reconcile_fabric(req, rsp, observed) elif kind == "Device": self._reconcile_device(req, rsp, observed) + elif kind in KINDS: + self._reconcile_input(rsp, observed) else: response.fatal(rsp, f"unsupported composite kind: {kind!r}") return rsp + # -- Inputs: a fragment of the design; they compose nothing --------------- + + def _reconcile_input(self, rsp: fnv1.RunFunctionResponse, observed: dict) -> None: + """Report what this fragment contributes, on its own object. + + An input composes nothing -- a Fabric collects it. This reconcile exists + so the team that owns the object sees its own shape here rather than + buried in someone else's Fabric status. + + It cannot report `status.devices`: an input does not know the fabric's + device list, so `appliesTo` only resolves where the inputs are collected. + The Fabric fills that in. Validation is deliberately not attempted + either -- whether pyavd can validate a fragment standalone is unsettled, + and a green validation that never ran is worse than none. + """ + spec = observed.get("spec") or {} + design = spec.get("design") or {} + status: dict = {"keys": sorted(design)} + + if observed.get("kind") == "NodeSet": + declared = spec.get("declares") + devices = set(declared) if declared is not None else hosts_in_blocks(design) + status["deviceCount"] = len(devices) + + resource.update_status(rsp.desired.composite, status) + response.normal( + rsp, f"{observed.get('kind')} contributes {len(design)} top-level key(s)" + ) + # -- Fabric: fabric-wide model -> one Device XR per host ------------------ def _reconcile_fabric( @@ -110,12 +150,24 @@ def _reconcile_fabric( namespace = meta.get("namespace", "default") xr_name = meta.get("name") or (fabric_name or "fabric").lower() - if not fabric_name or not design: - response.fatal(rsp, "spec.fabricName and spec.design are required") + requires = spec.get("requires") or [] + if not fabric_name or not (design or requires): + response.fatal(rsp, "spec.fabricName and spec.design or spec.requires are required") return + if requires: + all_inputs = self._collect(req, rsp, observed, requires, design, fabric_name) + if all_inputs is None: + return # gated -- _collect reported why + else: + # The released path: one fabric-wide document handed to every device, + # with AVD resolving roles from the node-type blocks. + document = dict(design) + document["fabric_name"] = fabric_name + all_inputs = {host: document for host in hostnames_from_design(document)} + try: - structured_configs = render_fabric_design(design, fabric_name) + structured_configs = render_structured_configs(all_inputs) except InputValidationError as err: resource.update_status( rsp.desired.composite, @@ -139,7 +191,12 @@ def _reconcile_fabric( "urlTemplate", "https://{hostname}.{namespace}.svc/command-api" ) - roles = device_roles_from_design(design) + # Roles come from each device's own view: with inputs, a leaf in DC1 sees + # DC1's node-type block and nothing of DC2's. + roles = { + host: device_roles_from_design(hostvars).get(host) or hostvars.get("type", "") + for host, hostvars in all_inputs.items() + } observed_devices = req.observed.resources # keyed by composition-resource-name (hostname) devices = [] for hostname, structured_config in structured_configs.items(): @@ -195,6 +252,139 @@ def _reconcile_fabric( rsp, f"Composed {len(structured_configs)} Device(s) for fabric {fabric_name}" ) + # -- Collecting the inputs a Fabric names --------------------------------- + + def _collect( # noqa: PLR0913 + self, + req: fnv1.RunFunctionRequest, + rsp: fnv1.RunFunctionResponse, + observed: dict, + requires: list[dict], + design: dict, + fabric_name: str, + ) -> dict[str, dict] | None: + """Ask Crossplane for the named inputs; layer them once they arrive. + + Returns per-device inputs, or ``None`` when the fabric must not render -- + the gate. That gate is not only a guard against a slow operator: + requirements are answered on the *next* reconcile, so the first one + always arrives with nothing at all, and rendering then would push a + fabric short of its inputs as a full config replacement. + """ + namespace = (observed.get("metadata") or {}).get("namespace", "default") + + # State the requirements on every reconcile. Crossplane fetches what the + # latest response asked for, so leaving them out once drops the inputs. + keys: list[tuple[str, dict]] = [] + for index, entry in enumerate(requires): + kind = entry["kind"] + key = f"{index:03d}-{kind.lower()}-{entry['name']}" + keys.append((key, entry)) + response.require_resources( + rsp, + name=key, + api_version="v1" if kind == "Secret" else API_VERSION, + kind=kind, + match_name=entry["name"], + namespace=entry.get("namespace", namespace), + ) + + pending: list[str] = [] + absent: list[str] = [] + inputs: list[Input] = [] + for key, entry in keys: + named = f"{entry['kind']}/{entry.get('namespace', namespace)}/{entry['name']}" + if entry["kind"] == "Secret": + # In the API from the first version so the mechanism can land + # without a schema change, but not implemented. Refuse rather + # than render a fabric whose credentials are silently absent. + response.fatal(rsp, f"Secret inputs are not implemented yet: {named}") + return None + if key not in req.required_resources: + pending.append(named) + continue + items = req.required_resources[key].items + if not items: + # Crossplane looked and found nothing. The proto distinguishes + # this from "not fetched yet" by sending an empty Resources, and + # that is what lets a Fabric tell "waiting" from "missing" -- + # the one thing this design was previously unable to do. + absent.append(named) + continue + inputs.append( + Input.from_xr(_normalize_numbers(resource.struct_to_dict(items[0].resource))) + ) + + if pending or absent: + detail = [] + if absent: + detail.append(f"not found: {', '.join(absent)}") + if pending: + detail.append(f"not fetched yet: {', '.join(pending)}") + message = "; ".join(detail) + response.set_conditions( + rsp, + resource.Condition( + typ="InputsResolved", + status="False", + reason="InputsMissing" if absent else "WaitingForInputs", + message=message[:400], + ), + ) + resource.update_status( + rsp.desired.composite, + {"fabricName": fabric_name, "validation": {"ok": False, "message": message}}, + ) + # Missing is a real problem; not-fetched-yet is the normal first pass. + report = response.warning if absent else response.normal + report(rsp, f"fabric {fabric_name} is waiting on inputs -- {message}") + return None + + # The Fabric's own design is the first input: fabric-wide, seen by every + # device, and declaring whatever devices its own blocks name so a Fabric + # that carries both a design and a requires list still has its devices. + document = dict(design) + document["fabric_name"] = fabric_name + inputs.insert( + 0, + Input( + name="fabric", + kind="Settings", + design=document, + all_devices=True, + declares=sorted(hosts_in_blocks(document)), + ), + ) + + if stray := unmatched_patterns(inputs): + listed = ", ".join(f"{name}: {pattern!r}" for name, pattern in stray) + response.fatal( + rsp, + f"appliesTo.matchHostnames matched no device ({listed}) -- " + f"a pattern that matches nothing is silent, so it is refused", + ) + return None + + if replaced := overwrites(inputs): + shown = ", ".join(f"{key} on {host} ({first} -> {second})" + for host, key, first, second in replaced[:5]) + response.warning( + rsp, + f"{len(replaced)} value(s) replaced by a later input: {shown}" + + (" ..." if len(replaced) > 5 else ""), + ) + + response.set_conditions( + rsp, + resource.Condition( + typ="InputsResolved", + status="True", + reason="AllInputsResolved", + message=f"{len(inputs)} input(s)", + ), + ) + return resolve(inputs) + # -- Device: validate + render one device's config ----------------------- def _reconcile_device( diff --git a/function/kinds.py b/function/kinds.py new file mode 100644 index 0000000..ebd5f04 --- /dev/null +++ b/function/kinds.py @@ -0,0 +1,205 @@ +"""The input-kind model: several XRs layered into per-device AVD inputs. + +A ``Fabric`` names its inputs in ``spec.requires``. Each input XR carries a +fragment of the eos_designs document in ``spec.design`` plus ``spec.appliesTo`` +saying which devices see it. Per device, the inputs that apply are layered in +``requires`` order with ``dict.update()`` -- Ansible's default +``hash_behaviour=replace``, which is what group_vars resolution does and what +``pyavd.get_avd_facts`` expects to be handed. + +**Nothing is merged.** Two NodeSets carrying the same node-type key never meet, +because no device sees both: in a dual-DC fabric a DC1 leaf sees DC1's +``l3leaf.defaults`` and a DC2 leaf sees DC2's. That is why this path needs +neither a fabric-wide document nor the fold in :mod:`function.xr`. + +Two things are separate that look like one: + +* which devices an input *declares* (``spec.declares``, plus the nodes its + blocks name) -- the union of these is the fabric's device list, and there is + no second list; +* which devices *see* it (``spec.appliesTo``). They coincide in simple + topologies and diverge in a 5-stage CLOS, where a DC's ``super_spine`` block + names four devices but is visible to every device of that DC. + +Measured against AVD's own corpus: the hostvars this produces are byte-identical +to faithfully reproduced Ansible for all 8 bundled examples and every eos_designs +molecule scenario -- 25 inventories, up to 501 devices. See +:mod:`function.verify_kinds`. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +KINDS = ("NodeSet", "NetworkServices", "ConnectedEndpoints", "Settings") + + +def matches(pattern: str, hostname: str) -> bool: + """AVD's own hostname-matching semantics, copied from the code not the docs. + + ``shared_utils/node_type.py`` resolves ``default_node_types`` with + ``search(f"^{regex}$", hostname)`` -- **AVD anchors the pattern for you**, so + ``dc1-leaf.*`` matches a whole name. The schema's description reads as though + the author must anchor it; the code does it for them. Copying the description + instead of the code would make the same pattern mean different things in the + two places. + """ + return re.search(f"^{pattern}$", hostname) is not None + + +def is_node_block(value: Any) -> bool: + """A node-type block is a dict carrying ``nodes`` and/or ``node_groups``.""" + return isinstance(value, dict) and ("nodes" in value or "node_groups" in value) + + +def hosts_in_blocks(design: dict) -> set[str]: + """Device names a design's node-type blocks mention.""" + hosts: set[str] = set() + for value in design.values(): + if not is_node_block(value): + continue + groups = list(value.get("node_groups") or []) + for nodes in [value.get("nodes") or []] + [g.get("nodes") or [] for g in groups]: + for node in nodes: + if isinstance(node, dict) and node.get("name"): + hosts.add(node["name"]) + return hosts + + +def classify(design: dict) -> str: + """Which kind a fragment belongs to. + + Advisory: the kinds exist for ownership (RBAC is granted per kind), not as a + partition the schema could enforce -- eos_designs' top-level key names come + from its own content, so no OpenAPI schema can describe them. + """ + if any(is_node_block(v) for v in design.values()): + return "NodeSet" + if {"tenants", "network_services_keys"} & design.keys(): + return "NetworkServices" + if { + "servers", "firewalls", "routers", "load_balancers", "storage_arrays", + "cpes", "workstations", "access_points", "phones", "printers", + "generic_devices", "port_profiles", "network_ports", + "connected_endpoints_keys", "custom_connected_endpoints_keys", + } & design.keys(): + return "ConnectedEndpoints" + return "Settings" + + +@dataclass +class Input: + """One input XR, reduced to what resolution needs.""" + + name: str + kind: str + design: dict + # spec.appliesTo -- the criteria are unioned; none set means every device + all_devices: bool = False + node_sets: list[str] = field(default_factory=list) + hosts: list[str] = field(default_factory=list) + match_hostnames: list[str] = field(default_factory=list) + # spec.declares -- devices this input brings into the fabric. Never a + # pattern: visibility may be matched, existence may not. A typo in a pattern + # would silently drop devices from the fabric. + declares: list[str] = field(default_factory=list) + + @classmethod + def from_xr(cls, xr: dict) -> "Input": + """Build from an XR as ``required_resources`` delivers it.""" + spec = xr.get("spec") or {} + applies = spec.get("appliesTo") or {} + design = spec.get("design") or {} + declares = list(spec.get("declares") or []) + kind = xr.get("kind") or classify(design) + if kind == "NodeSet" and not declares: + # A NodeSet that declares nothing explicitly declares what its + # blocks name -- the common case, where the two coincide. + declares = sorted(hosts_in_blocks(design)) + return cls( + name=(xr.get("metadata") or {}).get("name", ""), + kind=kind, + design=design, + all_devices=bool(applies.get("all")), + node_sets=list(applies.get("nodeSets") or []), + hosts=list(applies.get("hosts") or []), + match_hostnames=list(applies.get("matchHostnames") or []), + declares=declares, + ) + + def scope(self, declared_by: dict[str, set[str]], devices: set[str]) -> set[str]: + """Devices that see this input. The criteria are unioned.""" + if self.all_devices or not (self.node_sets or self.hosts or self.match_hostnames): + return devices + named: set[str] = set() + for name in self.node_sets: + named |= declared_by.get(name, set()) + named |= set(self.hosts) + named |= {h for h in devices for p in self.match_hostnames if matches(p, h)} + return devices & named + + +def resolve(inputs: list[Input]) -> dict[str, dict]: + """Layer ordered inputs into ``{hostname: hostvars}`` for ``get_avd_facts``. + + List order is precedence order: later inputs overwrite earlier ones key by + key, whole-key, exactly as Ansible resolves group_vars. An overwrite is + therefore intentional -- it is what the fabric owner declared by ordering -- + and belongs on status as a warning, never as an error. + """ + declared_by = {i.name: set(i.declares) for i in inputs if i.declares} + devices: set[str] = set() + for hosts in declared_by.values(): + devices |= hosts + + out: dict[str, dict] = {host: {} for host in devices} + for inp in inputs: + for host in inp.scope(declared_by, devices): + out[host].update(inp.design) + return out + + +def unmatched_patterns(inputs: list[Input]) -> list[tuple[str, str]]: + """``(input name, pattern)`` for every ``matchHostnames`` entry matching no device. + + A pattern is silent in both directions: a typo matches nothing and the input + quietly reaches no device, while a wide pattern quietly reaches devices it + was not meant to. The second is visible on status (`devices`); the first is + not, so the caller is expected to treat this as an error and refuse to + render -- the render is pushed as a full config replacement. + """ + devices: set[str] = set() + for inp in inputs: + devices |= set(inp.declares) + return [ + (inp.name, pattern) + for inp in inputs + for pattern in inp.match_hostnames + if not any(matches(pattern, host) for host in devices) + ] + + +def overwrites(inputs: list[Input]) -> list[tuple[str, str, str, str]]: + """``(device, key, earlier input, later input)`` for every value replaced. + + Ansible resolves these silently. Here the ordering is written down by a + person, so surfacing them is cheap and worth doing -- on status, as a + warning. + """ + declared_by = {i.name: set(i.declares) for i in inputs if i.declares} + devices: set[str] = set() + for hosts in declared_by.values(): + devices |= hosts + + seen: dict[tuple[str, str], tuple[str, Any]] = {} + found: list[tuple[str, str, str, str]] = [] + for inp in inputs: + for host in inp.scope(declared_by, devices): + for key, value in inp.design.items(): + previous = seen.get((host, key)) + if previous is not None and previous[1] != value: + found.append((host, key, previous[0], inp.name)) + seen[(host, key)] = (inp.name, value) + return found diff --git a/function/verify_kinds.py b/function/verify_kinds.py new file mode 100644 index 0000000..8485898 --- /dev/null +++ b/function/verify_kinds.py @@ -0,0 +1,311 @@ +"""Prove the input-kind model reproduces Ansible's variable resolution. + +Translates an AVD inventory into input XRs, resolves them with +:func:`function.kinds.resolve` -- which never looks at the inventory again -- +and compares the resulting hostvars against faithfully reproduced Ansible. + +Byte equality is the assertion. It is stricter than necessary (a hostvar AVD +never reads cannot change a rendered config) and that is deliberate: it fails +before a render can hide a difference. + +The translation reads the inventory; the resolution does not. Only the second +half is the model. What it establishes: + +* ``spec.requires`` order reproduces Ansible precedence. Ansible sorts groups by + (depth, name), which is a *global* total order, so one flat list restricted to + the inputs that apply to a device reproduces that device's view. +* ``appliesTo`` reproduces group membership without groups. +* One device list -- what NodeSets declare -- reproduces the inventory. + +Usage: + uv run avd-verify-kinds [EXAMPLE_DIR ...] # default: every bundled example +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from .ansible_inputs import ( + ALL_GROUP, + AnsibleInventory, + _load_group_vars, + _strip_ansible_keys, + _yaml_load, +) +from .kinds import Input, classify, hosts_in_blocks, is_node_block, resolve + +EXAMPLES_ROOT = Path("avd/ansible_collections/arista/avd/examples") +MOLECULE_ROOT = Path("avd/ansible_collections/arista/avd/extensions/molecule") + +# Examples this harness cannot translate, with the reason. Expected-failure +# semantics, as in verify_xr: one that starts passing IS reported, so a deferral +# can never rot silently. +DEFERRED: dict[str, str] = {} + +# The same, for --render. Resolution and rendering fail for different reasons: +# resolution is settled, rendering still runs into AVD features this path does +# not carry yet. +DEFERRED_RENDER: dict[str, str] = { + "cv-pathfinder": "ansible-vault secrets; credentials cannot live in an XR spec", +} + + +def inline_host_vars(inventory_file: Path) -> dict[str, dict]: + """Host variables written straight into the inventory. + + ``AnsibleInventory`` iterates the keys under ``hosts:`` and drops the values, + so ``dc1-spine1: {type: spine}`` is invisible to it. Several molecule + scenarios declare device types that way and nothing else does. + """ + found: dict[str, dict] = {} + + def walk(node: object) -> None: + if not isinstance(node, dict): + return + for host, hostvars in (node.get("hosts") or {}).items(): + if isinstance(hostvars, dict): + stripped = _strip_ansible_keys(hostvars) + if stripped: + found.setdefault(host, {}).update(stripped) + for child in (node.get("children") or {}).values(): + walk(child) + + for value in (_yaml_load(inventory_file) or {}).values(): + walk(value) + return found + + +def _layout(root: Path) -> tuple[Path, Path]: + """(directory holding group_vars, inventory file) for either layout.""" + if (root / "inventory.yml").is_file(): + return root, root / "inventory.yml" + return root / "inventory", root / "inventory" / "hosts.yml" + + +def ansible_hostvars(root: Path) -> dict[str, dict]: + """What ansible-playbook would hand to AVD, from every source it reads.""" + var_dir, inventory_file = _layout(root) + inventory = AnsibleInventory.from_file(inventory_file) + inline = inline_host_vars(inventory_file) + host_var_dir = var_dir / "host_vars" + files = ( + {f.stem: _strip_ansible_keys(_yaml_load(f)) for f in host_var_dir.glob("*.yml")} + if host_var_dir.is_dir() + else {} + ) + cache: dict[str, dict] = {} + out: dict[str, dict] = {} + for host in sorted(inventory.hosts()): + hostvars: dict = {} + for group in inventory.groups_for_host(host): + if group not in cache: + cache[group] = _load_group_vars(var_dir / "group_vars", group) + hostvars.update(cache[group]) + hostvars = _strip_ansible_keys(hostvars) + hostvars.update(inline.get(host, {})) + hostvars.update(files.get(host, {})) + out[host] = hostvars + return out + + +def inputs_from_inventory(root: Path) -> list[Input]: + """Translate an AVD inventory into ordered input XRs -- a migration. + + Each group_vars file becomes one or two inputs: a ``NodeSet`` for its + node-type blocks and one input for the rest. The split is not cosmetic -- + the two halves have different scopes whenever a block names fewer devices + than the group holds, which is what a 5-stage CLOS does. + """ + var_dir, inventory_file = _layout(root) + inventory = AnsibleInventory.from_file(inventory_file) + group_var_dir = var_dir / "group_vars" + every_device = inventory.hosts() + + groups = [] + if group_var_dir.is_dir(): + named = {p.stem for p in group_var_dir.glob("*.yml")} | { + d.name for d in group_var_dir.iterdir() if d.is_dir() + } + groups = [g for g in named if g in inventory.depth or g == ALL_GROUP] + # Ansible precedence: `all` first, then (depth, name). This is requires order. + ordered = sorted(groups, key=lambda g: (inventory.depth.get(g, 0), g)) + + def group_devices(group: str) -> set[str]: + if group == ALL_GROUP: + return set(every_device) + return {h for h in every_device if group in inventory.groups_for_host(h)} + + designs = {g: _strip_ansible_keys(_load_group_vars(group_var_dir, g)) for g in ordered} + declared_by: dict[str, set[str]] = {} + for group, design in designs.items(): + blocks = {k: v for k, v in design.items() if is_node_block(v)} + if blocks: + declared_by[group] = hosts_in_blocks(blocks) & set(every_device) + + # Devices no block mentions -- fixtures with no node-type blocks at all, and + # hosts that only the inventory knows about. Declared at the narrowest group + # holding each one rather than in a single fabric-wide NodeSet, so the + # resulting NodeSets line up with real groups and other inputs can name them. + device_sets: dict[str, set[str]] = {} + for host in sorted(set(every_device) - set().union(*declared_by.values() or [set()])): + deepest = inventory.groups_for_host(host)[-1] # already (depth, name) sorted + device_sets.setdefault(f"{deepest}-devices", set()).add(host) + declared_by.update(device_sets) + + inputs = [ + Input(name, "NodeSet", {}, node_sets=[name], declares=sorted(hosts)) + for name, hosts in sorted(device_sets.items()) + ] + + for group in ordered: + design = designs[group] + if not design: + continue + blocks = {k: v for k, v in design.items() if is_node_block(v)} + rest = {k: v for k, v in design.items() if not is_node_block(v)} + want = group_devices(group) + + def scoped(name: str, kind: str, payload: dict, want: set[str] = want) -> Input: + inp = Input(name=name, kind=kind, design=payload) + if want == set(every_device): + inp.all_devices = True + else: + cover = [n for n, hs in declared_by.items() if hs and hs <= want] + covered: set[str] = set() + for n in cover: + covered |= declared_by[n] + if covered == want: + inp.node_sets = sorted(cover) + else: + # No union of NodeSets is this group -- name the devices. + inp.hosts = sorted(want) + return inp + + if blocks: + node_set = scoped(group, "NodeSet", blocks) + node_set.declares = sorted(declared_by[group]) + inputs.append(node_set) + if rest: + inputs.append(scoped(f"{group}-settings" if blocks else group, classify(rest), rest)) + + # host_vars last, as Ansible does: inventory inline first, then files. + for host, design in sorted(inline_host_vars(inventory_file).items()): + inputs.append(Input(f"{host}-inline", classify(design), design, hosts=[host])) + host_var_dir = var_dir / "host_vars" + if host_var_dir.is_dir(): + for f in sorted(host_var_dir.glob("*.yml")): + design = _strip_ansible_keys(_yaml_load(f)) + if design: + inputs.append(Input(f.stem, classify(design), design, hosts=[f.stem])) + return inputs + + +def verify_one(root: Path) -> tuple[str, int]: + """Return (status, difference count). status in {ok, differs, error}.""" + try: + from_kinds = resolve(inputs_from_inventory(root)) + from_ansible = ansible_hostvars(root) + except Exception as err: # noqa: BLE001 - surface any translation failure + return f"error: {type(err).__name__}: {str(err)[:70]}", -1 + + notes: list[str] = [] + for host in sorted(set(from_kinds) | set(from_ansible)): + if host not in from_kinds: + notes.append(f"{host}: missing") + continue + if host not in from_ansible: + notes.append(f"{host}: extra") + continue + a, b = from_kinds[host], from_ansible[host] + notes += [f"{host}.{k}" for k in sorted(set(a) | set(b)) if a.get(k) != b.get(k)] + if notes: + return f"differs ({len(notes)}): {', '.join(notes[:3])}", len(notes) + return "ok", 0 + + +def render_one(root: Path) -> tuple[str, int]: + """Render through the kinds path and diff against the checked-in golden. + + This does **not** test the model -- if the hostvars match Ansible, and + :func:`verify_one` asserts they do, the render must match too. What it tests + is the pair (this path, this pyavd): an AVD upgrade that changes output slips + past resolution equivalence and fails here. That is the job `test_xr_fold` + does today through the fold, and this is its successor -- the fold reaches + 6 of 8 examples, this reaches 7. + """ + import yaml + + from .engine import render_structured_configs + from .verify_example import _diff + + golden = root / "intended" / "structured_configs" + if not golden.is_dir(): + return "no golden", -1 + try: + rendered = render_structured_configs(resolve(inputs_from_inventory(root))) + except Exception as err: # noqa: BLE001 - surface any AVD/render failure + return f"error: {type(err).__name__}: {str(err)[:70]}", -1 + + total = 0 + for hostname in sorted(rendered): + golden_file = golden / f"{hostname}.yml" + if not golden_file.is_file(): + continue # a scenario may render hosts it keeps no golden for + out: list[str] = [] + _diff(hostname, rendered[hostname], yaml.safe_load(golden_file.read_text()) or {}, out) + total += len(out) + return ("ok" if total == 0 else f"diff ({total})"), total + + +def _discover(root: Path) -> list[Path]: + return sorted(d for d in root.iterdir() if (d / "inventory.yml").is_file()) + + +def _discover_molecule(root: Path = MOLECULE_ROOT) -> list[Path]: + """Molecule scenarios carrying an inventory of their own. + + The wider corpus, and the harder one: these reach 501 devices and lean on + inventory-inline host vars, which the examples barely use. + """ + if not root.is_dir(): + return [] + return sorted( + d + for d in root.iterdir() + if (d / "inventory" / "hosts.yml").is_file() and (d / "inventory" / "group_vars").is_dir() + ) + + +def main() -> int: + args = [a for a in sys.argv[1:] if a != "--render"] + rendering = "--render" in sys.argv[1:] + roots = [Path(a) for a in args] or _discover(EXAMPLES_ROOT) + check = render_one if rendering else verify_one + deferrals = DEFERRED_RENDER if rendering else DEFERRED + + failures = deferred = 0 + for root in roots: + status, _ = check(root) + ok = status == "ok" + reason = deferrals.get(root.name) + if reason and not ok: + mark, deferred = "DEFER", deferred + 1 + status = f"deferred: {reason}" + elif reason and ok: + mark, failures = "XPASS", failures + 1 + status = "passes now -- remove from the DEFERRED map" + elif ok: + mark = "OK " + else: + mark, failures = "FAIL", failures + 1 + print(f"[{mark}] {root.name:26s} {status}") + + expected = len(roots) - deferred + what = "reproduce golden" if rendering else "resolve identically to Ansible" + print(f"\n{expected - failures}/{expected} inventories {what} ({deferred} deferred).") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 824f76a..4854142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ [project.scripts] avd-verify = "function.verify_example:main" avd-verify-xr = "function.verify_xr:main" +avd-verify-kinds = "function.verify_kinds:main" avd-function = "function.main:main" avd-topology = "function.netclab_topology:main" diff --git a/scripts/kind-up.sh b/scripts/kind-up.sh index 6213d31..725f1e9 100755 --- a/scripts/kind-up.sh +++ b/scripts/kind-up.sh @@ -101,7 +101,15 @@ done echo ">> build + push function image/xpkg (tag ${TAG})" docker build --provenance=false -t "${IMG}:${TAG}" . -crossplane xpkg build --package-root=package --embed-runtime-image="${IMG}:${TAG}" -o "function-avd-${TAG}.xpkg" +# --examples-root is stated even though `package/` is the package root: it +# defaults to ./examples, which is not "no examples" but *everything* under +# examples/ -- including examples/lab/topology.yaml, which is helm values with no +# `kind` and fails the build with "Object 'Kind' is missing". --ignore is no help, +# it does not reach --examples-root. CI and the release workflow have named it +# since v0.1.4 failed on exactly this; this script was the build path that did +# not, so a local bring-up broke the day the topology was committed. +crossplane xpkg build --package-root=package --examples-root=examples/fabric \ + --embed-runtime-image="${IMG}:${TAG}" -o "function-avd-${TAG}.xpkg" crossplane xpkg push -f "function-avd-${TAG}.xpkg" "localhost:${REG_PORT}/netclab/function-avd:${TAG}" echo ">> install Crossplane (chart ${XP_CHART})" @@ -122,10 +130,14 @@ spec: EOF kubectl --context "$CTX" wait --for=condition=Healthy function.pkg.crossplane.io/netclab-function-avd --timeout=180s -echo ">> install XRDs + Compositions (Fabric + Device)" -kubectl --context "$CTX" apply -f apis/fabric/xrd.yaml -f apis/device/xrd.yaml -kubectl --context "$CTX" wait --for=condition=Established xrd/fabrics.avd.netclab.dev xrd/devices.avd.netclab.dev --timeout=60s -kubectl --context "$CTX" apply -f apis/fabric/composition.yaml -f apis/device/composition.yaml +# Every API under apis/, not a named pair: the package root ships whatever is +# there, so a script naming two of them installs a cluster that does not match +# the package it was built from -- and the input kinds would be missing exactly +# where a Fabric that names them is being tested. +echo ">> install XRDs + Compositions (everything under apis/)" +kubectl --context "$CTX" apply -f apis/*/xrd.yaml +kubectl --context "$CTX" wait --for=condition=Established xrd --all --timeout=60s +kubectl --context "$CTX" apply -f apis/*/composition.yaml if [ "$WITH_NETCLAB" = "1" ]; then echo ">> provider-http ${PROVIDER_HTTP} (config push over eAPI)" diff --git a/tests/test_apis_consistency.py b/tests/test_apis_consistency.py new file mode 100644 index 0000000..bf4f09f --- /dev/null +++ b/tests/test_apis_consistency.py @@ -0,0 +1,81 @@ +"""The published API and the code that serves it do not drift apart. + +Offline. Cheap checks over `apis/`, each guarding a failure that is silent: + +* a new input kind gets an XRD but `fn.py` never learns to reconcile it (or the + reverse), and the XR sits unready with "unsupported composite kind"; +* an XRD ships without `categories`, so `kubectl get netclab` does not list it -- + the exact defect this repo carried in Fabric and Device until it was found by + running the command, not by reading the file; +* an XRD points `defaultCompositionRef` at a Composition that is not there, or at + one built for a different kind, so nothing selects it. + +None of these break a build. They break in a cluster, one release later. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from function.kinds import KINDS + +APIS = Path("apis") +# Fabric and Device are composed, not collected -- they are not input kinds. +COMPOSED_KINDS = {"Fabric", "Device"} + + +def _xrds() -> dict[str, dict]: + return {p.parent.name: yaml.safe_load(p.read_text()) for p in sorted(APIS.glob("*/xrd.yaml"))} + + +XRDS = _xrds() + + +def test_apis_directory_is_not_empty() -> None: + assert len(XRDS) >= 6, f"expected the six XRDs, found {sorted(XRDS)}" + + +@pytest.mark.parametrize("name", sorted(XRDS), ids=str) +def test_xrd_declares_categories(name: str) -> None: + """Every XRD is reachable by `kubectl get netclab` and `kubectl get crossplane`.""" + names = XRDS[name]["spec"]["names"] + assert names.get("categories") == ["crossplane", "netclab"], ( + f"{name}: categories are {names.get('categories')!r}; netclab-xp's twelve " + f"XRDs all carry ['crossplane', 'netclab'] and these must match" + ) + + +@pytest.mark.parametrize("name", sorted(XRDS), ids=str) +def test_xrd_has_its_composition(name: str) -> None: + """`defaultCompositionRef` resolves, and to a Composition for this kind.""" + xrd = XRDS[name]["spec"] + wanted = xrd["defaultCompositionRef"]["name"] + composition = yaml.safe_load((APIS / name / "composition.yaml").read_text()) + assert composition["metadata"]["name"] == wanted + assert composition["spec"]["compositeTypeRef"]["kind"] == xrd["names"]["kind"] + + +def test_input_kinds_match_the_function() -> None: + """The XRDs that exist and the kinds fn.py reconciles are the same set.""" + from_apis = {x["spec"]["names"]["kind"] for x in XRDS.values()} - COMPOSED_KINDS + assert from_apis == set(KINDS), ( + f"apis/ serves {sorted(from_apis)} but function.kinds.KINDS is " + f"{sorted(KINDS)} -- fn.py would answer 'unsupported composite kind'" + ) + + +def test_fabric_requires_accepts_every_input_kind_and_secret() -> None: + """A Fabric can name each input kind, plus a Secret carrying credentials. + + Secret is in the enum from the first version deliberately: adding it later + would be a schema change to a released API, and the mechanism it enables -- + a Secret layered like any other input -- needs no other schema footprint. + """ + spec = XRDS["fabric"]["spec"]["versions"][0]["schema"]["openAPIV3Schema"] + enum = spec["properties"]["spec"]["properties"]["requires"]["items"]["properties"]["kind"][ + "enum" + ] + assert set(enum) == set(KINDS) | {"Secret"} diff --git a/tests/test_fabric_collect.py b/tests/test_fabric_collect.py new file mode 100644 index 0000000..68da053 --- /dev/null +++ b/tests/test_fabric_collect.py @@ -0,0 +1,181 @@ +"""A Fabric collects the inputs it names, and refuses to render without them. + +Offline -- drives RunFunction directly, with no cluster and no Crossplane. The +gate is the most safety-critical piece in the collect path: requirements are +answered on the *next* reconcile, so the first one always arrives with nothing, +and a fabric rendered short of its inputs would be pushed to devices as a full +config replacement. +""" + +from __future__ import annotations + +import asyncio + +import pytest +from crossplane.function import resource +from crossplane.function.proto.v1 import run_function_pb2 as fnv1 + +from function.fn import FunctionRunner + +API = "avd.netclab.dev/v1alpha1" + + +def _run(req: fnv1.RunFunctionRequest) -> fnv1.RunFunctionResponse: + return asyncio.run(FunctionRunner().RunFunction(req, None)) + + +def _fabric(requires: list[dict], design: dict | None = None) -> dict: + return { + "apiVersion": API, + "kind": "Fabric", + "metadata": {"name": "fabric", "namespace": "avd"}, + "spec": {"fabricName": "FABRIC", "design": design or {}, "requires": requires}, + } + + +def _input_xr(kind: str, name: str, spec: dict) -> dict: + return { + "apiVersion": API, + "kind": kind, + "metadata": {"name": name, "namespace": "avd"}, + "spec": spec, + } + + +def _request(xr: dict, required: dict[str, list[dict]] | None = None) -> fnv1.RunFunctionRequest: + req = fnv1.RunFunctionRequest() + req.observed.composite.resource.CopyFrom(resource.dict_to_struct(xr)) + for key, objects in (required or {}).items(): + # An empty list is Crossplane saying "I looked and found nothing", which + # the proto distinguishes from a key that is absent entirely. + entry = req.required_resources[key] + for obj in objects: + entry.items.add().resource.CopyFrom(resource.dict_to_struct(obj)) + return req + + +def _condition(rsp: fnv1.RunFunctionResponse, typ: str): + return next((c for c in rsp.conditions if c.type == typ), None) + + +# A spine rather than a leaf, only because a leaf defaults to being a VTEP and +# would drag in the VXLAN pools -- this fixture is about the collect path, not +# about exercising AVD. +SPINES = _input_xr( + "NodeSet", + "spines", + { + # `type` rides in the same input: AVD needs it (or default_node_types) + # to know what the device is, and it applies to whoever sees this input. + "design": { + "type": "spine", + "spine": { + "defaults": {"loopback_ipv4_pool": "10.255.0.0/27"}, + "nodes": [{"name": "spine1", "id": 1, "bgp_as": 65100}], + }, + } + }, +) + + +def test_first_reconcile_asks_and_renders_nothing() -> None: + """Requirements are answered next time round, so the first pass is empty. + + This is the case the gate exists for: not a slow operator, but the protocol. + """ + rsp = _run(_request(_fabric([{"kind": "NodeSet", "name": "spines"}]))) + + assert set(rsp.requirements.resources) == {"000-nodeset-spines"} + selector = rsp.requirements.resources["000-nodeset-spines"] + assert (selector.kind, selector.match_name, selector.namespace) == ("NodeSet", "spines", "avd") + + assert not rsp.desired.resources, "nothing may be composed before the inputs arrive" + condition = _condition(rsp, "InputsResolved") + assert condition.reason == "WaitingForInputs" + + +def test_missing_input_is_distinguished_from_not_yet_fetched() -> None: + """An empty Resources means Crossplane looked and found nothing.""" + rsp = _run( + _request( + _fabric([{"kind": "NodeSet", "name": "spines"}]), + required={"000-nodeset-spines": []}, + ) + ) + + assert not rsp.desired.resources + assert _condition(rsp, "InputsResolved").reason == "InputsMissing" + assert "not found" in _condition(rsp, "InputsResolved").message + + +def test_resolved_inputs_compose_devices() -> None: + rsp = _run( + _request( + _fabric([{"kind": "NodeSet", "name": "spines"}]), + required={"000-nodeset-spines": [SPINES]}, + ) + ) + + assert _condition(rsp, "InputsResolved").status == fnv1.STATUS_CONDITION_TRUE + assert set(rsp.desired.resources) == {"spine1"} + + +def test_design_without_requires_still_composes() -> None: + """The released path is untouched: one document, handed to every device. + + Guards the refactor that put both paths through render_structured_configs -- + v0.1.6 is published and netclab-xp pins it, so this must keep working with no + inputs in sight. + """ + rsp = _run(_request(_fabric(requires=[], design=SPINES["spec"]["design"]))) + + assert not any(r.severity == fnv1.SEVERITY_FATAL for r in rsp.results) + assert set(rsp.desired.resources) == {"spine1"} + assert not rsp.requirements.resources, "a fabric with no requires asks for nothing" + + +def test_secret_input_is_refused_until_implemented() -> None: + """It is in the enum so the mechanism can land without a schema change. + + Rendering a fabric whose credentials are silently absent would push a config + without them, so refusing is the only safe placeholder. + """ + rsp = _run(_request(_fabric([{"kind": "Secret", "name": "creds"}]))) + + assert any(r.severity == fnv1.SEVERITY_FATAL for r in rsp.results) + assert not rsp.desired.resources + + +def test_pattern_matching_no_device_is_refused() -> None: + """A pattern is silent about matching nothing, so it cannot be allowed to.""" + settings = _input_xr( + "Settings", + "typo", + # The fabric holds only spine1, so this pattern matches nothing. + {"design": {"ntp_settings": {}}, "appliesTo": {"matchHostnames": ["leaf.*"]}}, + ) + rsp = _run( + _request( + _fabric( + [ + {"kind": "NodeSet", "name": "spines"}, + {"kind": "Settings", "name": "typo"}, + ] + ), + required={"000-nodeset-spines": [SPINES], "001-settings-typo": [settings]}, + ) + ) + + fatal = [r for r in rsp.results if r.severity == fnv1.SEVERITY_FATAL] + assert fatal and "matched no device" in fatal[0].message + assert not rsp.desired.resources + + +@pytest.mark.parametrize("kind", ["NodeSet", "NetworkServices", "ConnectedEndpoints", "Settings"]) +def test_input_kinds_reconcile_and_report_their_keys(kind: str) -> None: + """Each input reports its own shape on its own object, composing nothing.""" + rsp = _run(_request(_input_xr(kind, "an-input", {"design": {"ntp_settings": {}, "type": "x"}}))) + + assert not any(r.severity == fnv1.SEVERITY_FATAL for r in rsp.results) + status = resource.struct_to_dict(rsp.desired.composite.resource).get("status", {}) + assert status["keys"] == ["ntp_settings", "type"] diff --git a/tests/test_kinds_equivalence.py b/tests/test_kinds_equivalence.py new file mode 100644 index 0000000..27376c8 --- /dev/null +++ b/tests/test_kinds_equivalence.py @@ -0,0 +1,116 @@ +"""The input-kind model resolves exactly as Ansible does. + +Offline -- no cluster, no pyavd render. Guards :func:`function.kinds.resolve` +and the migration that feeds it, over AVD's own corpus: the 8 bundled examples +and every molecule scenario with an inventory of its own, up to 501 devices. + +This is the regression net for the collect path. It is stricter than a render +comparison on purpose: it fails on a hostvar difference even where AVD would +have rendered the same config, so a divergence cannot hide until it matters. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from function.kinds import Input, resolve +from function.verify_kinds import ( + DEFERRED, + DEFERRED_RENDER, + EXAMPLES_ROOT, + _discover, + _discover_molecule, + render_one, + verify_one, +) + +CORPUS = _discover(EXAMPLES_ROOT) + _discover_molecule() +EXAMPLES = _discover(EXAMPLES_ROOT) + + +@pytest.mark.parametrize("root", CORPUS, ids=lambda p: p.name) +def test_resolves_identically_to_ansible(root: Path) -> None: + status, _ = verify_one(root) + + if root.name in DEFERRED: + # Expected failure. Asserting it still fails keeps a deferral from + # rotting: if it starts resolving, this fails and says to drop it. + assert status != "ok", ( + f"{root.name} resolves now -- remove it from verify_kinds.DEFERRED " + f"(was deferred: {DEFERRED[root.name]})" + ) + return + + assert status == "ok", f"{root.name}: {status}" + + +@pytest.mark.parametrize("root", EXAMPLES, ids=lambda p: p.name) +def test_render_reproduces_golden(root: Path) -> None: + """Rendered configs still match the checked-in golden. + + Redundant as a check on the model -- matching hostvars render identically -- + and that is not what it is for. It is the guard against pyavd itself + changing: an AVD upgrade slips past resolution equivalence and fails here. + + Examples only. The molecule scenarios need AVD features this path does not + carry yet (templates loaded from files, ID pools, custom Python classes), so + they stay on the equivalence test until those land. + """ + status, _ = render_one(root) + + if root.name in DEFERRED_RENDER: + assert status != "ok", ( + f"{root.name} renders clean now -- remove it from " + f"verify_kinds.DEFERRED_RENDER (was: {DEFERRED_RENDER[root.name]})" + ) + return + + assert status == "ok", f"{root.name}: {status}" + + +def test_corpus_is_not_empty() -> None: + # The submodule is optional in a fresh worktree; an empty parametrisation + # would make this whole file pass while testing nothing. + assert len(CORPUS) >= 8, f"expected the AVD corpus, found {len(CORPUS)} inventories" + + +def test_later_input_overwrites_earlier() -> None: + """Precedence is list order, and replacement is whole-key.""" + inputs = [ + Input("nodes", "NodeSet", {"l3leaf": {"nodes": [{"name": "leaf1"}]}}, + node_sets=["nodes"], declares=["leaf1"]), + Input("base", "Settings", {"ntp_settings": {"servers": ["a"]}}, all_devices=True), + Input("narrow", "Settings", {"ntp_settings": {"servers": ["b"]}}, hosts=["leaf1"]), + ] + assert resolve(inputs)["leaf1"]["ntp_settings"] == {"servers": ["b"]} + + +def test_input_applies_only_where_scoped() -> None: + """A device sees an input only if appliesTo names it -- this is what + replaces group membership, and what keeps two DCs' node blocks apart.""" + inputs = [ + Input("dc1", "NodeSet", {"l3leaf": {"defaults": {"loopback_ipv4_pool": "10.0.0.0/24"}}}, + node_sets=["dc1"], declares=["leaf1"]), + Input("dc2", "NodeSet", {"l3leaf": {"defaults": {"loopback_ipv4_pool": "10.1.0.0/24"}}}, + node_sets=["dc2"], declares=["leaf2"]), + ] + out = resolve(inputs) + assert out["leaf1"]["l3leaf"]["defaults"]["loopback_ipv4_pool"] == "10.0.0.0/24" + assert out["leaf2"]["l3leaf"]["defaults"]["loopback_ipv4_pool"] == "10.1.0.0/24" + + +def test_undeclared_node_is_not_a_device() -> None: + """A block may name a node the fabric does not declare -- AVD's own + anta_runner does -- and it must not become a device.""" + inputs = [ + Input( + "leaves", + "NodeSet", + {"l3leaf": {"nodes": [{"name": "leaf1"}, {"name": "ghost"}]}}, + node_sets=["leaves"], + declares=["leaf1"], + ) + ] + assert set(resolve(inputs)) == {"leaf1"}