From c80d1699f73ecc24dbfec6bbe889f2af26eab3c9 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 12:24:07 -0500 Subject: [PATCH 01/13] docs: regenerate stale CRD reference The generated reference had drifted from the types. Domain was missing its refresh-attempt fields, HTTPProxy and TrafficProtectionPolicy were several changes behind, and Connector, ConnectorAdvertisement, ConnectorClass and LocationBinding had no page at all. Generator output with no hand edits, separated from the work that surfaced it. --- docs/api/connectoradvertisements.md | 360 ++++ docs/api/connectorclasses.md | 96 + docs/api/connectors.md | 579 ++++++ docs/api/domains.md | 38 + docs/api/httpproxies.md | 2513 ++++++++++++++++++------- docs/api/locationbindings.md | 256 +++ docs/api/trafficprotectionpolicies.md | 46 +- 7 files changed, 3173 insertions(+), 715 deletions(-) create mode 100644 docs/api/connectoradvertisements.md create mode 100644 docs/api/connectorclasses.md create mode 100644 docs/api/connectors.md create mode 100644 docs/api/locationbindings.md diff --git a/docs/api/connectoradvertisements.md b/docs/api/connectoradvertisements.md new file mode 100644 index 00000000..e5ae48bb --- /dev/null +++ b/docs/api/connectoradvertisements.md @@ -0,0 +1,360 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha1](#networkingdatumapiscomv1alpha1) + +# networking.datumapis.com/v1alpha1 + +Resource Types: + +- [ConnectorAdvertisement](#connectoradvertisement) + + + + +## ConnectorAdvertisement +[↩ Parent](#networkingdatumapiscomv1alpha1 ) + + + + + + +ConnectorAdvertisement is the Schema for the connectoradvertisements API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
apiVersionstringnetworking.datumapis.com/v1alpha1true
kindstringConnectorAdvertisementtrue
metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
specobject + Spec defines the desired state of a ConnectorAdvertisement
+ 		true
statusobject + Status defines the observed state of a ConnectorAdvertisement
+
+ Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Accepted]]]
+ 		false
+ + +### ConnectorAdvertisement.spec +[↩ Parent](#connectoradvertisement) + + + +Spec defines the desired state of a ConnectorAdvertisement + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
connectorRefobject + ConnectorRef references the Connector being advertised.
+ 		true
layer4[]object + Layer 4 services being advertised.
+ 		false
+ + +### ConnectorAdvertisement.spec.connectorRef +[↩ Parent](#connectoradvertisementspec) + + + +ConnectorRef references the Connector being advertised. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the referenced Connector.
+ 		true
+ + +### ConnectorAdvertisement.spec.layer4[index] +[↩ Parent](#connectoradvertisementspec) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the advertisement.
+ 		true
services[]object + Layer 4 services being advertised.
+ 		true
+ + +### ConnectorAdvertisement.spec.layer4[index].services[index] +[↩ Parent](#connectoradvertisementspeclayer4index) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
addressstring + Address of the service. + +Can be an IPv4, IPv6, or a DNS address. A DNS address may contain +wildcards. A DNS address acts as an allow list for what addresses the +connector will allow to be requested through it. + +DNS resolution is the responsibility of the connector.
+ 		true
ports[]object + Ports of the service.
+ 		true
+ + +### ConnectorAdvertisement.spec.layer4[index].services[index].ports[index] +[↩ Parent](#connectoradvertisementspeclayer4indexservicesindex) + + + +Layer4ServicePort represents a port for a Layer 4 service. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Named port for the service.
+ 		true
portinteger + Port number for the service.
+
+ Format: int32
+ Minimum: 1
+ Maximum: 65535
+ 		true
protocolstring + Protocol for port. Must be TCP or UDP, defaults to "TCP".
+
+ Default: TCP
+ 		true
+ + +### ConnectorAdvertisement.status +[↩ Parent](#connectoradvertisement) + + + +Status defines the observed state of a ConnectorAdvertisement + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
conditions[]object + Conditions describe the current conditions of the ConnectorAdvertisement. + +Known conditions: +- Accepted: indicates whether the referenced Connector has been resolved. + When Accepted is False, the reason will explain why the reference + could not be resolved (for example, ConnectorNotFound).
+ 		false
+ + +### ConnectorAdvertisement.status.conditions[index] +[↩ Parent](#connectoradvertisementstatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
+
+ Format: date-time
+ 		true
messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
+ 		true
reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
+ 		true
statusenum + status of the condition, one of True, False, Unknown.
+
+ Enum: True, False, Unknown
+ 		true
typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
+ 		true
observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
+
+ Format: int64
+ Minimum: 0
+ 		false
diff --git a/docs/api/connectorclasses.md b/docs/api/connectorclasses.md new file mode 100644 index 00000000..98306332 --- /dev/null +++ b/docs/api/connectorclasses.md @@ -0,0 +1,96 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha1](#networkingdatumapiscomv1alpha1) + +# networking.datumapis.com/v1alpha1 + +Resource Types: + +- [ConnectorClass](#connectorclass) + + + + +## ConnectorClass +[↩ Parent](#networkingdatumapiscomv1alpha1 ) + + + + + + +ConnectorClass is the Schema for the connectorclasses API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
apiVersionstringnetworking.datumapis.com/v1alpha1true
kindstringConnectorClasstrue
metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
specobject + Spec defines the desired state of a ConnectorClass
+ 		true
statusobject + Status defines the observed state of a ConnectorClass
+ 		false
+ + +### ConnectorClass.spec +[↩ Parent](#connectorclass) + + + +Spec defines the desired state of a ConnectorClass + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
controllerNamestring + ControllerName is the name of the controller responsible for this ConnectorClass.
+
+ Default: networking.datumapis.com/datum-connect
+ 		true
diff --git a/docs/api/connectors.md b/docs/api/connectors.md new file mode 100644 index 00000000..6cb3d00d --- /dev/null +++ b/docs/api/connectors.md @@ -0,0 +1,579 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha1](#networkingdatumapiscomv1alpha1) + +# networking.datumapis.com/v1alpha1 + +Resource Types: + +- [Connector](#connector) + + + + +## Connector +[↩ Parent](#networkingdatumapiscomv1alpha1 ) + + + + + + +Connector is the Schema for the connectors API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
apiVersionstringnetworking.datumapis.com/v1alpha1true
kindstringConnectortrue
metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
specobject + Spec defines the desired state of a Connector
+ 		true
statusobject + Status defines the observed state of a Connector
+
+ Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Accepted]]]
+ 		false
+ + +### Connector.spec +[↩ Parent](#connector) + + + +Spec defines the desired state of a Connector + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
connectorClassNamestring +
+ 		true
capabilities[]object + Capabilities desired to be supported by the connector. + +A connector may choose to not support all requested capabilities, and may +also choose to support additional capabilities not requested here. The +condition of each capability will reflect whether the capability is supported +or not in the ConnectorStatus.
+ 		false
+ + +### Connector.spec.capabilities[index] +[↩ Parent](#connectorspec) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
typestring + Type of capability
+ 		true
connectTCPobject +
+ 		false
+ + +### Connector.spec.capabilities[index].connectTCP +[↩ Parent](#connectorspeccapabilitiesindex) + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
disabledboolean +
+ 		false
+ + +### Connector.status +[↩ Parent](#connector) + + + +Status defines the observed state of a Connector + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
capabilities[]object + Capabilities describe the status of each capability of the connector.
+ 		false
conditions[]object + Conditions describe the current conditions of the HTTPProxy.
+ 		false
connectionDetailsobject + ConnectionDetails provide details on how to connect to the connector.
+
+ Validations:
  • !(self.type != 'PublicKey' && has(self.publicKey)): publicKey field must be nil if the type is not PublicKey
  • self.type == 'PublicKey' && has(self.publicKey): publicKey field must be specified if the type is PublicKey
    • +     		false
    leaseRefobject + LeaseRef references the Lease used to report connector liveness. + +The connector controller creates the Lease when a Connector is created +and records it here. Connector implementations (agents) are expected to +periodically renew the Lease to indicate liveness.
    +     		false
    + + +### Connector.status.capabilities[index] +[↩ Parent](#connectorstatus) + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    typestring + Type of capability
    +     		true
    conditions[]object + Conditions describe the current conditions of the capability.
    +     		false
    + + +### Connector.status.capabilities[index].conditions[index] +[↩ Parent](#connectorstatuscapabilitiesindex) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### Connector.status.conditions[index] +[↩ Parent](#connectorstatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### Connector.status.connectionDetails +[↩ Parent](#connectorstatus) + + + +ConnectionDetails provide details on how to connect to the connector. + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    typeenum + Type of connection details provided.
    +
    + Enum: PublicKey
    +     		true
    publicKeyobject + PublicKey connection details
    +     		false
    + + +### Connector.status.connectionDetails.publicKey +[↩ Parent](#connectorstatusconnectiondetails) + + + +PublicKey connection details + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addresses[]object + Addresses where the connector can be reached
    +     		true
    homeRelaystring + Home Relay server of the connector + +Must be a valid URL
    +
    + Validations:
  • isURL(self): Must be a URL.
    • +     		true
    discoveryModeenum + The mode used to discover the public key
    +
    + Enum: DNS
    + Default: DNS
    +     		false
    idstring + The public key to dial and connect to
    +     		false
    + + +### Connector.status.connectionDetails.publicKey.addresses[index] +[↩ Parent](#connectorstatusconnectiondetailspublickey) + + + +PublicKeyConnectorAddress defines an address and port for a connector. + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + IPv4 or IPv6 address.
    +
    + Validations:
  • isIP(self): Must be an IP address.
    • +     		true
    portinteger + Port where the connector can be reached.
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    +     		true
    + + +### Connector.status.leaseRef +[↩ Parent](#connectorstatus) + + + +LeaseRef references the Lease used to report connector liveness. + +The connector controller creates the Lease when a Connector is created +and records it here. Connector implementations (agents) are expected to +periodically renew the Lease to indicate liveness. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of the referent. +This field is effectively required, but due to backwards compatibility is +allowed to be empty. Instances of this type with an empty value here are +almost certainly wrong. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
    +
    + Default:
    +     		false
    diff --git a/docs/api/domains.md b/docs/api/domains.md index 78119a9d..6793b1db 100644 --- a/docs/api/domains.md +++ b/docs/api/domains.md @@ -94,6 +94,26 @@ DomainSpec defines the desired state of Domain Validations:
  • oldSelf == '' || self == oldSelf: A domain name is immutable and cannot be changed after creation
  • self.indexOf('.') != -1: Must have at least two segments separated by dots
    • true + + desiredRegistrationRefreshAttempt + string + + DesiredRegistrationRefreshAttempt is the desired time of the next registration refresh attempt.
    +
    + Validations:
  • oldSelf == null || self == null || self == oldSelf || self >= oldSelf + duration('5m'): must be at least 5m after the previous desiredRegistrationRefreshAttempt when changed
    • + Format: date-time
    + + false + + desiredVerificationRefreshAttempt + string + + DesiredVerificationRefreshAttempt is the desired time of the next verification refresh attempt.
    +
    + Validations:
  • oldSelf == null || self == null || self == oldSelf || self >= oldSelf + duration('5m'): must be at least 5m after the previous desiredVerificationRefreshAttempt when changed
    • + Format: date-time
    + + false @@ -369,6 +389,15 @@ Registration represents the registration information for a domain
    false + + lastRefreshAttempt + string + +
    +
    + Format: date-time
    + + false nextRefreshAttempt string @@ -811,6 +840,15 @@ DomainVerificationStatus represents the verification status of a domain
    false + + lastVerificationAttempt + string + +
    +
    + Format: date-time
    + + false nextVerificationAttempt string diff --git a/docs/api/httpproxies.md b/docs/api/httpproxies.md index 826c7aa0..633faf77 100644 --- a/docs/api/httpproxies.md +++ b/docs/api/httpproxies.md @@ -234,6 +234,16 @@ Supports http and https protocols, IPs or DNS addresses in the host, custom ports, and paths.
    true + + connector + object + + Connector references the Connector that should be used for this backend. + +For now, only a name reference is supported. In the future this can be +extended to selector-based matching to allow multiple connectors.
    + + false filters []object @@ -244,6 +254,46 @@ request is being forwarded to the backend defined here.
    Validations:
  • !(self.exists(f, f.type == 'RequestRedirect') && self.exists(f, f.type == 'URLRewrite')): May specify either requestRedirect or urlRewrite, but not both
  • self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1: RequestHeaderModifier filter cannot be repeated
  • self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1: ResponseHeaderModifier filter cannot be repeated
  • self.filter(f, f.type == 'RequestRedirect').size() <= 1: RequestRedirect filter cannot be repeated
  • self.filter(f, f.type == 'URLRewrite').size() <= 1: URLRewrite filter cannot be repeated
    • false + + tls + object + + TLS contains backend TLS configuration. + +When the backend endpoint uses HTTPS with an IP address, the Hostname field +must be specified for TLS certificate validation.
    + + false + + + + +### HTTPProxy.spec.rules[index].backends[index].connector +[↩ Parent](#httpproxyspecrulesindexbackendsindex) + + + +Connector references the Connector that should be used for this backend. + +For now, only a name reference is supported. In the future this can be +extended to selector-based matching to allow multiple connectors. + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of the referenced Connector.
    +     		true
    @@ -260,8 +310,8 @@ examples include request or response modification, implementing authentication strategies, rate-limiting, and traffic shaping. API guarantee/conformance is defined based on the type of the filter. - - + + @@ -309,9 +359,9 @@ Unknown values here must result in the implementation setting the Accepted Condition for the Route to `status: False`, with a Reason of `UnsupportedValue`. -
    +

    - Enum: RequestHeaderModifier, ResponseHeaderModifier, RequestMirror, RequestRedirect, URLRewrite, ExtensionRef
    + Enum: RequestHeaderModifier, ResponseHeaderModifier, RequestMirror, RequestRedirect, URLRewrite, ExtensionRef, CORS
     @@ -321,9 +371,7 @@ Reason of `UnsupportedValue`. CORS defines a schema for a filter that responds to the cross-origin request based on HTTP response header. -Support: Extended - -
    +Support: Extended
     @@ -340,6 +388,24 @@ This filter can be used multiple times within the same rule. Support: Implementation-specific
    + + + + + @@ -410,8 +476,6 @@ cross-origin request based on HTTP response header. Support: Extended - -
    true
    false
    false
    externalAuthobject + ExternalAuth configures settings related to sending request details +to an external auth service. The external service MUST authenticate +the request, and MAY authorize the request as well. + +If there is any problem communicating with the external service, +this filter MUST fail closed. + +Support: Extended + +
    +
    + Validations:
  • self.protocol == 'GRPC' ? has(self.grpc) : true: grpc must be specified when protocol is set to 'GRPC'
  • has(self.grpc) ? self.protocol == 'GRPC' : true: protocol must be 'GRPC' when grpc is set
  • self.protocol == 'HTTP' ? has(self.http) : true: http must be specified when protocol is set to 'HTTP'
  • has(self.http) ? self.protocol == 'HTTP' : true: protocol must be 'HTTP' when http is set
    • +     		false
    requestHeaderModifier object
    @@ -428,16 +492,14 @@ Support: Extended AllowCredentials indicates whether the actual cross-origin request allows to include credentials. -The only valid value for the `Access-Control-Allow-Credentials` response -header is true (case-sensitive). +When set to true, the gateway will include the `Access-Control-Allow-Credentials` +response header with value true (case-sensitive). -If the credentials are not allowed in cross-origin requests, the gateway -will omit the header `Access-Control-Allow-Credentials` entirely rather -than setting its value to false. +When set to false or omitted the gateway will omit the header +`Access-Control-Allow-Credentials` entirely (this is the standard CORS +behavior). Support: Extended
    -
    - Enum: true
    @@ -447,7 +509,7 @@ Support: Extended
    AllowHeaders indicates which HTTP request headers are supported for accessing the requested resource. -Header names are not case sensitive. +Header names are not case-sensitive. Multiple header names in the value of the `Access-Control-Allow-Headers` response header are separated by a comma (","). @@ -466,20 +528,25 @@ does not recognize by the client, it will also occur an error on the client side. A wildcard indicates that the requests with all HTTP headers are allowed. -The `Access-Control-Allow-Headers` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. - -When the `AllowCredentials` field is specified and `AllowHeaders` field -specified with the `*` wildcard, the gateway must specify one or more +If config contains the wildcard "*" in allowHeaders and the request is +not credentialed, the `Access-Control-Allow-Headers` response header +can either use the `*` wildcard or the value of +Access-Control-Request-Headers from the request. + +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Headers` response header. When +also the `AllowCredentials` field is true and `AllowHeaders` field +is specified with the `*` wildcard, the gateway must specify one or more HTTP headers in the value of the `Access-Control-Allow-Headers` response header. The value of the header `Access-Control-Allow-Headers` is same as the `Access-Control-Request-Headers` header provided by the client. If the header `Access-Control-Request-Headers` is not included in the request, the gateway will omit the `Access-Control-Allow-Headers` -response header, instead of specifying the `*` wildcard. A Gateway -implementation may choose to add implementation-specific default headers. +response header, instead of specifying the `*` wildcard. Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowHeaders cannot contain '*' alongside other methods
    • @@ -492,7 +559,7 @@ requested resource. Valid values are any method defined by RFC9110, along with the special value `*`, which represents all HTTP methods are allowed. -Method names are case sensitive, so these values are also case-sensitive. +Method names are case-sensitive, so these values are also case-sensitive. (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) Multiple method names in the value of the `Access-Control-Allow-Methods` @@ -512,18 +579,21 @@ is not included in the list of methods specified by the response header `Access-Control-Allow-Methods`, it will present an error on the client side. -The `Access-Control-Allow-Methods` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. +If config contains the wildcard "*" in allowMethods and the request is +not credentialed, the `Access-Control-Allow-Methods` response header +can either use the `*` wildcard or the value of +Access-Control-Request-Method from the request. -When the `AllowCredentials` field is specified and `AllowMethods` field +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Methods` response header. When +also the `AllowCredentials` field is true and `AllowMethods` field specified with the `*` wildcard, the gateway must specify one HTTP method in the value of the Access-Control-Allow-Methods response header. The value of the header `Access-Control-Allow-Methods` is same as the `Access-Control-Request-Method` header provided by the client. If the header `Access-Control-Request-Method` is not included in the request, the gateway will omit the `Access-Control-Allow-Methods` response header, -instead of specifying the `*` wildcard. A Gateway implementation may -choose to add implementation-specific default methods. +instead of specifying the `*` wildcard. Support: Extended

    @@ -577,10 +647,19 @@ cross-origin response headers. Alternatively, the gateway responds with the CORS headers. The cross-origin request fails on the client side. Therefore, the client doesn't attempt the actual cross-origin request. -The `Access-Control-Allow-Origin` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. +Conversely, if the request `Origin` matches one of the configured +allowed origins, the gateway sets the response header +`Access-Control-Allow-Origin` to the same value as the `Origin` +header provided by the client. + +When config has the wildcard ("*") in allowOrigins, and the request +is not credentialed (e.g., it is a preflight request), the +`Access-Control-Allow-Origin` response header either contains the +wildcard as well or the Origin from the request. -When the `AllowCredentials` field is specified and `AllowOrigins` field +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Origin` response header. When +also the `AllowCredentials` field is true and `AllowOrigins` field specified with the `*` wildcard, the gateway must return a single origin in the value of the `Access-Control-Allow-Origin` response header, instead of specifying the `*` wildcard. The value of the header @@ -588,6 +667,8 @@ instead of specifying the `*` wildcard. The value of the header the client. Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowOrigins cannot contain '*' alongside other origins
    • @@ -614,15 +695,18 @@ When an HTTP header name is specified using the `ExposeHeaders` field, this additional header will be exposed as part of the response to the client. -Header names are not case sensitive. +Header names are not case-sensitive. Multiple header names in the value of the `Access-Control-Expose-Headers` response header are separated by a comma (","). A wildcard indicates that the responses with all HTTP headers are exposed to clients. The `Access-Control-Expose-Headers` response header can only -use `*` wildcard as value when the `AllowCredentials` field is -unspecified. +use `*` wildcard as value when the request is not credentialed. + +When the `exposeHeaders` config field contains the "*" wildcard and +the request is credentialed, the gateway cannot use the `*` wildcard in +the `Access-Control-Expose-Headers` response header. Support: Extended
    @@ -639,7 +723,10 @@ The information provided by the `Access-Control-Allow-Methods` and client until the time specified by `Access-Control-Max-Age` elapses. The default value of `Access-Control-Max-Age` response header is 5 -(seconds).
    +(seconds). + +When the `MaxAge` field is unspecified, the gateway sets the response +header "Access-Control-Max-Age: 5" by default.

    Format: int32
    Default: 5
    @@ -699,15 +786,21 @@ When unspecified or empty string, core API group is inferred.
    false
    false
    false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth [↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) -RequestHeaderModifier defines a schema for a filter that modifies request -headers. +ExternalAuth configures settings related to sending request details +to an external auth service. The external service MUST authenticate +the request, and MAY authorize the request as well. -Support: Core +If there is any problem communicating with the external service, +this filter MUST fail closed. + +Support: Extended + + @@ -719,81 +812,104 @@ Support: Core - - + + + + + + + - + - - + + - - + + + + + + +
    add[]objectbackendRefobject - Add adds the given header(s) (name, value) to the request -before the action. It appends to any existing values associated -with the header name. + BackendRef is a reference to a backend to send authorization +requests to. -Input: - GET /foo HTTP/1.1 - my-header: foo +The backend must speak the selected protocol (GRPC or HTTP) on the +referenced port. -Config: - add: - - name: "my-header" - value: "bar,baz" +If the backend service requires TLS, use BackendTLSPolicy to tell the +implementation to supply the TLS details to be used to connect to that +backend.
    +
    + Validations:
  • (size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true: Must have port for Service reference
    • +     		true
    protocolenum + ExternalAuthProtocol describes which protocol to use when communicating with an +ext_authz authorization server. -Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz
    +When this is set to GRPC, each backend must use the Envoy ext_authz protocol +on the port specified in `backendRefs`. Requests and responses are defined +in the protobufs explained at: +https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto + +When this is set to HTTP, each backend must respond with a `200` status +code in on a successful authorization. Any other code is considered +an authorization failure. + +Feature Names: +GRPC Support - HTTPRouteExternalAuthGRPC +HTTP Support - HTTPRouteExternalAuthHTTP
    +
    + Enum: HTTP, GRPC
    		falsetrue
    remove[]stringforwardBodyobject - Remove the given header(s) from the HTTP request before the action. The -value of Remove is a list of HTTP header names. Note that the header -names are case-insensitive (see -https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + ForwardBody controls if requests to the authorization server should include +the body of the client request; and if so, how big that body is allowed +to be. -Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz +It is expected that implementations will buffer the request body up to +`forwardBody.maxSize` bytes. Bodies over that size must be rejected with a +4xx series error (413 or 403 are common examples), and fail processing +of the filter. -Config: - remove: ["my-header1", "my-header3"] +If unset, or `forwardBody.maxSize` is set to `0`, then the body will not +be forwarded. -Output: - GET /foo HTTP/1.1 - my-header2: bar
    +Feature Name: HTTPRouteExternalAuthForwardBody
    		 false
    set[]objectgrpcobject - Set overwrites the request with the given header (name, value) -before the action. - -Input: - GET /foo HTTP/1.1 - my-header: foo + GRPCAuthConfig contains configuration for communication with ext_authz +protocol-speaking backends. -Config: - set: - - name: "my-header" - value: "bar" +If unset, implementations must assume the default behavior for each +included field is intended.
    +     		false
    httpobject + HTTPAuthConfig contains configuration for communication with HTTP-speaking +backends. -Output: - GET /foo HTTP/1.1 - my-header: bar
    +If unset, implementations must assume the default behavior for each +included field is intended.
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier.add[index] -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestheadermodifier) +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth.backendRef +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexexternalauth) -HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. +BackendRef is a reference to a backend to send authorization +requests to. + +The backend must speak the selected protocol (GRPC or HTTP) on the +referenced port. + +If the backend service requires TLS, use BackendTLSPolicy to tell the +implementation to supply the TLS details to be used to connect to that +backend. @@ -808,33 +924,93 @@ HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - + - + + + + + + + + + + + + + + + +
    name string - Name is the name of the HTTP Header to be matched. Name matching MUST be -case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - -If multiple entries specify equivalent header names, the first entry with -an equivalent name MUST be considered for a match. Subsequent entries -with an equivalent header name MUST be ignored. Due to the -case-insensitivity of header names, "foo" and "Foo" are considered -equivalent.
    + Name is the name of the referent.
    		 true
    valuegroup string - Value is the value of HTTP Header to be matched.
    + Group is the group of the referent. For example, "gateway.networking.k8s.io". +When unspecified or empty string, core API group is inferred.
    +
    + Default:
    		truefalse
    kindstring + Kind is the Kubernetes resource kind of the referent. For example +"Service". + +Defaults to "Service" when not specified. + +ExternalName services can refer to CNAME DNS records that may live +outside of the cluster and as such are difficult to reason about in +terms of conformance. They also may not be safe to forward to (see +CVE-2021-25740 for more information). Implementations SHOULD NOT +support ExternalName Services. + +Support: Core (Services with a type other than ExternalName) + +Support: Implementation-specific (Services with type ExternalName)
    +
    + Default: Service
    +     		false
    namespacestring + Namespace is the namespace of the backend. When unspecified, the local +namespace is inferred. + +Note that when a namespace different than the local namespace is specified, +a ReferenceGrant object is required in the referent namespace to allow that +namespace's owner to accept the reference. See the ReferenceGrant +documentation for details. + +Support: Core
    +     		false
    portinteger + Port specifies the destination port number to use for this resource. +Port is required when the referent is a Kubernetes Service. In this +case, the port number is the service port number, not the target port. +For other resources, destination port might be derived from the referent +resource or this field.
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    +     		false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier.set[index] -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestheadermodifier) +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth.forwardBody +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexexternalauth) -HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. +ForwardBody controls if requests to the authorization server should include +the body of the client request; and if so, how big that body is allowed +to be. + +It is expected that implementations will buffer the request body up to +`forwardBody.maxSize` bytes. Bodies over that size must be rejected with a +4xx series error (413 or 403 are common examples), and fail processing +of the filter. + +If unset, or `forwardBody.maxSize` is set to `0`, then the body will not +be forwarded. + +Feature Name: HTTPRouteExternalAuthForwardBody @@ -846,44 +1022,36 @@ HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - - + + - - - - - - +
    namestringmaxSizeinteger - Name is the name of the HTTP Header to be matched. Name matching MUST be -case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + MaxSize specifies how large in bytes the largest body that will be buffered +and sent to the authorization server. If the body size is larger than +`maxSize`, then the body sent to the authorization server must be +truncated to `maxSize` bytes. -If multiple entries specify equivalent header names, the first entry with -an equivalent name MUST be considered for a match. Subsequent entries -with an equivalent header name MUST be ignored. Due to the -case-insensitivity of header names, "foo" and "Foo" are considered -equivalent.
    -     		true
    valuestring - Value is the value of HTTP Header to be matched.
    +Experimental note: This behavior needs to be checked against +various dataplanes; it may need to be changed. +See https://github.com/kubernetes-sigs/gateway-api/pull/4001#discussion_r2291405746 +for more. + +If 0, the body will not be sent to the authorization server.
    		truefalse
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) - +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth.grpc +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexexternalauth) -RequestMirror defines a schema for a filter that mirrors requests. -Requests are sent to the specified destination, but responses from -that destination are ignored. -This filter can be used multiple times within the same rule. Note that -not all implementations will be able to support mirroring to multiple -backends. +GRPCAuthConfig contains configuration for communication with ext_authz +protocol-speaking backends. -Support: Extended +If unset, implementations must assume the default behavior for each +included field is intended. @@ -895,97 +1063,109 @@ Support: Extended - - + + + + +
    backendRefobjectallowedHeaders[]string - BackendRef references a resource where mirrored requests are sent. + AllowedRequestHeaders specifies what headers from the client request +will be sent to the authorization server. -Mirrored requests must be sent only to a single destination endpoint -within this BackendRef, irrespective of how many endpoints are present -within this BackendRef. +If this list is empty, then all headers must be sent. -If the referent cannot be found, this BackendRef is invalid and must be -dropped from the Gateway. The controller must ensure the "ResolvedRefs" -condition on the Route status is set to `status: False` and not configure -this backend in the underlying implementation. +If the list has entries, only those entries must be sent.
    +     		false
    -If there is a cross-namespace reference to an *existing* object -that is not allowed by a ReferenceGrant, the controller must ensure the -"ResolvedRefs" condition on the Route is set to `status: False`, -with the "RefNotPermitted" reason and not configure this backend in the -underlying implementation. -In either error case, the Message of the `ResolvedRefs` Condition -should be used to provide more detail about the problem. +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth.http +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexexternalauth) -Support: Extended for Kubernetes Service -Support: Implementation-specific for any other resource
    -
    - Validations:
  • (size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true: Must have port for Service reference
    • + +HTTPAuthConfig contains configuration for communication with HTTP-speaking +backends. + +If unset, implementations must assume the default behavior for each +included field is intended. + + + + + + + + + + + + + + - + - - + + - - + +
    NameTypeDescriptionRequired
    allowedHeaders[]string + AllowedRequestHeaders specifies what additional headers from the client request +will be sent to the authorization server. + +The following headers must always be sent to the authorization server, +regardless of this setting: + +* `Host` +* `Method` +* `Path` +* `Content-Length` +* `Authorization` + +If this list is empty, then only those headers must be sent. + +Note that `Content-Length` has a special behavior, in that the length +sent must be correct for the actual request to the external authorization +server - that is, it must reflect the actual number of bytes sent in the +body of the request to the authorization server. + +So if the `forwardBody` stanza is unset, or `forwardBody.maxSize` is set +to `0`, then `Content-Length` must be `0`. If `forwardBody.maxSize` is set +to anything other than `0`, then the `Content-Length` of the authorization +request must be set to the actual number of bytes forwarded.
    		truefalse
    fractionobjectallowedResponseHeaders[]string - Fraction represents the fraction of requests that should be -mirrored to BackendRef. + AllowedResponseHeaders specifies what headers from the authorization response +will be copied into the request to the backend. -Only one of Fraction or Percent may be specified. If neither field -is specified, 100% of requests will be mirrored.
    -
    - Validations:
  • self.numerator <= self.denominator: numerator must be less than or equal to denominator
    • +If this list is empty, then all headers from the authorization server +except Authority or Host must be copied.
    		 false
    percentintegerpathstring - Percent represents the percentage of requests that should be -mirrored to BackendRef. Its minimum value is 0 (indicating 0% of -requests) and its maximum value is 100 (indicating 100% of requests). + Path sets the prefix that paths from the client request will have added +when forwarded to the authorization server. -Only one of Fraction or Percent may be specified. If neither field -is specified, 100% of requests will be mirrored.
    -
    - Format: int32
    - Minimum: 0
    - Maximum: 100
    +When empty or unspecified, no prefix is added. + +Valid values are the same as the "value" regex for path values in the `match` +stanza, and the validation regex will screen out invalid paths in the same way. +Even with the validation, implementations MUST sanitize this input before using it +directly.
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror.backendRef -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestmirror) - - - -BackendRef references a resource where mirrored requests are sent. - -Mirrored requests must be sent only to a single destination endpoint -within this BackendRef, irrespective of how many endpoints are present -within this BackendRef. - -If the referent cannot be found, this BackendRef is invalid and must be -dropped from the Gateway. The controller must ensure the "ResolvedRefs" -condition on the Route status is set to `status: False` and not configure -this backend in the underlying implementation. +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) -If there is a cross-namespace reference to an *existing* object -that is not allowed by a ReferenceGrant, the controller must ensure the -"ResolvedRefs" condition on the Route is set to `status: False`, -with the "RefNotPermitted" reason and not configure this backend in the -underlying implementation. -In either error case, the Message of the `ResolvedRefs` Condition -should be used to provide more detail about the problem. -Support: Extended for Kubernetes Service +RequestHeaderModifier defines a schema for a filter that modifies request +headers. -Support: Implementation-specific for any other resource +Support: Core @@ -997,88 +1177,81 @@ Support: Implementation-specific for any other resource - - - - - - - - - - - - + + - - + + - - + +
    namestring - Name is the name of the referent.
    -     		true
    groupstring - Group is the group of the referent. For example, "gateway.networking.k8s.io". -When unspecified or empty string, core API group is inferred.
    -
    - Default:
    -     		false
    kindstringadd[]object - Kind is the Kubernetes resource kind of the referent. For example -"Service". - -Defaults to "Service" when not specified. + Add adds the given header(s) (name, value) to the request +before the action. It appends to any existing values associated +with the header name. -ExternalName services can refer to CNAME DNS records that may live -outside of the cluster and as such are difficult to reason about in -terms of conformance. They also may not be safe to forward to (see -CVE-2021-25740 for more information). Implementations SHOULD NOT -support ExternalName Services. +Input: + GET /foo HTTP/1.1 + my-header: foo -Support: Core (Services with a type other than ExternalName) +Config: + add: + - name: "my-header" + value: "bar,baz" -Support: Implementation-specific (Services with type ExternalName)
    -
    - Default: Service
    +Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz
    		 false
    namespacestringremove[]string - Namespace is the namespace of the backend. When unspecified, the local -namespace is inferred. + Remove the given header(s) from the HTTP request before the action. The +value of Remove is a list of HTTP header names. Note that the header +names are case-insensitive (see +https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). -Note that when a namespace different than the local namespace is specified, -a ReferenceGrant object is required in the referent namespace to allow that -namespace's owner to accept the reference. See the ReferenceGrant -documentation for details. +Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz -Support: Core
    +Config: + remove: ["my-header1", "my-header3"] + +Output: + GET /foo HTTP/1.1 + my-header2: bar
    		 false
    portintegerset[]object - Port specifies the destination port number to use for this resource. -Port is required when the referent is a Kubernetes Service. In this -case, the port number is the service port number, not the target port. -For other resources, destination port might be derived from the referent -resource or this field.
    -
    - Format: int32
    - Minimum: 1
    - Maximum: 65535
    + Set overwrites the request with the given header (name, value) +before the action. + +Input: + GET /foo HTTP/1.1 + my-header: foo + +Config: + set: + - name: "my-header" + value: "bar" + +Output: + GET /foo HTTP/1.1 + my-header: bar
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror.fraction -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestmirror) - +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier.add[index] +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestheadermodifier) -Fraction represents the fraction of requests that should be -mirrored to BackendRef. -Only one of Fraction or Percent may be specified. If neither field -is specified, 100% of requests will be mirrored. +HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. @@ -1090,39 +1263,42 @@ is specified, 100% of requests will be mirrored. - - + + - - + + - +
    numeratorintegernamestring -
    -
    - Format: int32
    - Minimum: 0
    + Name is the name of the HTTP Header to be matched. Name matching MUST be +case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + +If multiple entries specify equivalent header names, the first entry with +an equivalent name MUST be considered for a match. Subsequent entries +with an equivalent header name MUST be ignored. Due to the +case-insensitivity of header names, "foo" and "Foo" are considered +equivalent.
    		 true
    denominatorintegervaluestring -
    -
    - Format: int32
    - Default: 100
    - Minimum: 1
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    		falsetrue
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestRedirect -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) - +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier.set[index] +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestheadermodifier) -RequestRedirect defines a schema for a filter that responds to the -request with an HTTP redirection. -Support: Core +HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. @@ -1134,116 +1310,152 @@ Support: Core - + - + - - + + - - - - + + +
    hostnamename string - Hostname is the hostname to be used in the value of the `Location` -header in the response. -When empty, the hostname in the `Host` header of the request is used. + Name is the name of the HTTP Header to be matched. Name matching MUST be +case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). -Support: Core
    +If multiple entries specify equivalent header names, the first entry with +an equivalent name MUST be considered for a match. Subsequent entries +with an equivalent header name MUST be ignored. Due to the +case-insensitivity of header names, "foo" and "Foo" are considered +equivalent.
    		falsetrue
    pathobjectvaluestring - Path defines parameters used to modify the path of the incoming request. -The modified path is then used to construct the `Location` header. When -empty, the request path is used as-is. + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + -Support: Extended
    -
    - Validations:
  • self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true: replaceFullPath must be specified when type is set to 'ReplaceFullPath'
  • has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true: type must be 'ReplaceFullPath' when replaceFullPath is set
  • self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'
  • has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set
    • +
    		false
    portintegertrue
    + + +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) + + + +RequestMirror defines a schema for a filter that mirrors requests. +Requests are sent to the specified destination, but responses from +that destination are ignored. + +This filter can be used multiple times within the same rule. Note that +not all implementations will be able to support mirroring to multiple +backends. + +Support: Extended + + + + + + + + + + + + + - + - - + + - +
    NameTypeDescriptionRequired
    backendRefobject - Port is the port to be used in the value of the `Location` -header in the response. + BackendRef references a resource where mirrored requests are sent. -If no port is specified, the redirect port MUST be derived using the -following rules: +Mirrored requests must be sent only to a single destination endpoint +within this BackendRef, irrespective of how many endpoints are present +within this BackendRef. -* If redirect scheme is not-empty, the redirect port MUST be the well-known - port associated with the redirect scheme. Specifically "http" to port 80 - and "https" to port 443. If the redirect scheme does not have a - well-known port, the listener port of the Gateway SHOULD be used. -* If redirect scheme is empty, the redirect port MUST be the Gateway - Listener port. +If the referent cannot be found, this BackendRef is invalid and must be +dropped from the Gateway. The controller must ensure the "ResolvedRefs" +condition on the Route status is set to `status: False` and not configure +this backend in the underlying implementation. -Implementations SHOULD NOT add the port number in the 'Location' -header in the following cases: +If there is a cross-namespace reference to an *existing* object +that is not allowed by a ReferenceGrant, the controller must ensure the +"ResolvedRefs" condition on the Route is set to `status: False`, +with the "RefNotPermitted" reason and not configure this backend in the +underlying implementation. -* A Location header that will use HTTP (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 80. -* A Location header that will use HTTPS (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 443. +In either error case, the Message of the `ResolvedRefs` Condition +should be used to provide more detail about the problem. -Support: Extended
    +Support: Extended for Kubernetes Service + +Support: Implementation-specific for any other resource

    - Format: int32
    - Minimum: 1
    - Maximum: 65535
    + Validations:
  • (size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true: Must have port for Service reference
    		• falsetrue
    schemeenumfractionobject - Scheme is the scheme to be used in the value of the `Location` header in -the response. When empty, the scheme of the request is used. - -Scheme redirects can affect the port of the redirect, for more information, -refer to the documentation for the port field of this filter. - -Note that values may be added to this enum, implementations -must ensure that unknown values will not cause a crash. - -Unknown values here must result in the implementation setting the -Accepted Condition for the Route to `status: False`, with a -Reason of `UnsupportedValue`. + Fraction represents the fraction of requests that should be +mirrored to BackendRef. -Support: Extended
    +Only one of Fraction or Percent may be specified. If neither field +is specified, 100% of requests will be mirrored.

    - Enum: http, https
    + Validations:
  • self.numerator <= self.denominator: numerator must be less than or equal to denominator
    		•  false
    statusCodepercent integer - StatusCode is the HTTP status code to be used in response. - -Note that values may be added to this enum, implementations -must ensure that unknown values will not cause a crash. - -Unknown values here must result in the implementation setting the -Accepted Condition for the Route to `status: False`, with a -Reason of `UnsupportedValue`. + Percent represents the percentage of requests that should be +mirrored to BackendRef. Its minimum value is 0 (indicating 0% of +requests) and its maximum value is 100 (indicating 100% of requests). -Support: Core
    +Only one of Fraction or Percent may be specified. If neither field +is specified, 100% of requests will be mirrored.

    - Enum: 301, 302
    - Default: 302
    + Format: int32
    + Minimum: 0
    + Maximum: 100
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestRedirect.path -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestredirect) +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror.backendRef +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestmirror) -Path defines parameters used to modify the path of the incoming request. -The modified path is then used to construct the `Location` header. When -empty, the request path is used as-is. +BackendRef references a resource where mirrored requests are sent. -Support: Extended +Mirrored requests must be sent only to a single destination endpoint +within this BackendRef, irrespective of how many endpoints are present +within this BackendRef. + +If the referent cannot be found, this BackendRef is invalid and must be +dropped from the Gateway. The controller must ensure the "ResolvedRefs" +condition on the Route status is set to `status: False` and not configure +this backend in the underlying implementation. + +If there is a cross-namespace reference to an *existing* object +that is not allowed by a ReferenceGrant, the controller must ensure the +"ResolvedRefs" condition on the Route is set to `status: False`, +with the "RefNotPermitted" reason and not configure this backend in the +underlying implementation. + +In either error case, the Message of the `ResolvedRefs` Condition +should be used to provide more detail about the problem. + +Support: Extended for Kubernetes Service + +Support: Implementation-specific for any other resource @@ -1255,65 +1467,88 @@ Support: Extended - - + + - + - + + + + + + + + + + +
    typeenumnamestring - Type defines the type of path modifier. Additional types may be -added in a future release of the API. - -Note that values may be added to this enum, implementations -must ensure that unknown values will not cause a crash. - -Unknown values here must result in the implementation setting the -Accepted Condition for the Route to `status: False`, with a -Reason of `UnsupportedValue`.
    -
    - Enum: ReplaceFullPath, ReplacePrefixMatch
    + Name is the name of the referent.
    		 true
    replaceFullPathgroup string - ReplaceFullPath specifies the value with which to replace the full path -of a request during a rewrite or redirect.
    + Group is the group of the referent. For example, "gateway.networking.k8s.io". +When unspecified or empty string, core API group is inferred.
    +
    + Default:
    		false
    replacePrefixMatchkind string - ReplacePrefixMatch specifies the value with which to replace the prefix -match of a request during a rewrite or redirect. For example, a request -to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch -of "/xyz" would be modified to "/xyz/bar". + Kind is the Kubernetes resource kind of the referent. For example +"Service". -Note that this matches the behavior of the PathPrefix match type. This -matches full path elements. A path element refers to the list of labels -in the path split by the `/` separator. When specified, a trailing `/` is -ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all -match the prefix `/abc`, but the path `/abcd` would not. +Defaults to "Service" when not specified. -ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. -Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in -the implementation setting the Accepted Condition for the Route to `status: False`. +ExternalName services can refer to CNAME DNS records that may live +outside of the cluster and as such are difficult to reason about in +terms of conformance. They also may not be safe to forward to (see +CVE-2021-25740 for more information). Implementations SHOULD NOT +support ExternalName Services. -Request Path | Prefix Match | Replace Prefix | Modified Path
    +Support: Core (Services with a type other than ExternalName) + +Support: Implementation-specific (Services with type ExternalName)
    +
    + Default: Service
    +     		false
    namespacestring + Namespace is the namespace of the backend. When unspecified, the local +namespace is inferred. + +Note that when a namespace different than the local namespace is specified, +a ReferenceGrant object is required in the referent namespace to allow that +namespace's owner to accept the reference. See the ReferenceGrant +documentation for details. + +Support: Core
    +     		false
    portinteger + Port specifies the destination port number to use for this resource. +Port is required when the referent is a Kubernetes Service. In this +case, the port number is the service port number, not the target port. +For other resources, destination port might be derived from the referent +resource or this field.
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].responseHeaderModifier -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror.fraction +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestmirror) -ResponseHeaderModifier defines a schema for a filter that modifies response -headers. +Fraction represents the fraction of requests that should be +mirrored to BackendRef. -Support: Extended +Only one of Fraction or Percent may be specified. If neither field +is specified, 100% of requests will be mirrored. @@ -1325,12 +1560,247 @@ Support: Extended - - + + + + + + + + + +
    add[]objectnumeratorinteger - Add adds the given header(s) (name, value) to the request -before the action. It appends to any existing values associated -with the header name. +
    +
    + Format: int32
    + Minimum: 0
    +     		true
    denominatorinteger +
    +
    + Format: int32
    + Default: 100
    + Minimum: 1
    +     		false
    + + +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestRedirect +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) + + + +RequestRedirect defines a schema for a filter that responds to the +request with an HTTP redirection. + +Support: Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    hostnamestring + Hostname is the hostname to be used in the value of the `Location` +header in the response. +When empty, the hostname in the `Host` header of the request is used. + +Support: Core
    +     		false
    pathobject + Path defines parameters used to modify the path of the incoming request. +The modified path is then used to construct the `Location` header. When +empty, the request path is used as-is. + +Support: Extended
    +
    + Validations:
  • self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true: replaceFullPath must be specified when type is set to 'ReplaceFullPath'
  • has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true: type must be 'ReplaceFullPath' when replaceFullPath is set
  • self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'
  • has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set
    • +     		false
    portinteger + Port is the port to be used in the value of the `Location` +header in the response. + +If no port is specified, the redirect port MUST be derived using the +following rules: + +* If redirect scheme is not-empty, the redirect port MUST be the well-known + port associated with the redirect scheme. Specifically "http" to port 80 + and "https" to port 443. If the redirect scheme does not have a + well-known port, the listener port of the Gateway SHOULD be used. +* If redirect scheme is empty, the redirect port MUST be the Gateway + Listener port. + +Implementations SHOULD NOT add the port number in the 'Location' +header in the following cases: + +* A Location header that will use HTTP (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 80. +* A Location header that will use HTTPS (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 443. + +Support: Extended
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    +     		false
    schemeenum + Scheme is the scheme to be used in the value of the `Location` header in +the response. When empty, the scheme of the request is used. + +Scheme redirects can affect the port of the redirect, for more information, +refer to the documentation for the port field of this filter. + +Note that values may be added to this enum, implementations +must ensure that unknown values will not cause a crash. + +Unknown values here must result in the implementation setting the +Accepted Condition for the Route to `status: False`, with a +Reason of `UnsupportedValue`. + +Support: Extended
    +
    + Enum: http, https
    +     		false
    statusCodeinteger + StatusCode is the HTTP status code to be used in response. + +Note that values may be added to this enum, implementations +must ensure that unknown values will not cause a crash. + +Unknown values here must result in the implementation setting the +Accepted Condition for the Route to `status: False`, with a +Reason of `UnsupportedValue`. + +Support: Core
    +
    + Enum: 301, 302, 303, 307, 308
    + Default: 302
    +     		false
    + + +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestRedirect.path +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestredirect) + + + +Path defines parameters used to modify the path of the incoming request. +The modified path is then used to construct the `Location` header. When +empty, the request path is used as-is. + +Support: Extended + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    typeenum + Type defines the type of path modifier. Additional types may be +added in a future release of the API. + +Note that values may be added to this enum, implementations +must ensure that unknown values will not cause a crash. + +Unknown values here must result in the implementation setting the +Accepted Condition for the Route to `status: False`, with a +Reason of `UnsupportedValue`.
    +
    + Enum: ReplaceFullPath, ReplacePrefixMatch
    +     		true
    replaceFullPathstring + ReplaceFullPath specifies the value with which to replace the full path +of a request during a rewrite or redirect.
    +     		false
    replacePrefixMatchstring + ReplacePrefixMatch specifies the value with which to replace the prefix +match of a request during a rewrite or redirect. For example, a request +to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch +of "/xyz" would be modified to "/xyz/bar". + +Note that this matches the behavior of the PathPrefix match type. This +matches full path elements. A path element refers to the list of labels +in the path split by the `/` separator. When specified, a trailing `/` is +ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all +match the prefix `/abc`, but the path `/abcd` would not. + +ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. +Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in +the implementation setting the Accepted Condition for the Route to `status: False`. + +Request Path | Prefix Match | Replace Prefix | Modified Path
    +     		false
    + + +### HTTPProxy.spec.rules[index].backends[index].filters[index].responseHeaderModifier +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) + + + +ResponseHeaderModifier defines a schema for a filter that modifies response +headers. + +Support: Extended + + + + + + + + + + + + + + @@ -1469,7 +1945,13 @@ equivalent.
    @@ -1583,25 +2065,554 @@ the implementation setting the Accepted Condition for the Route to `status: Fals Request Path | Prefix Match | Replace Prefix | Modified Path
    - + + +
    NameTypeDescriptionRequired
    add[]object + Add adds the given header(s) (name, value) to the request +before the action. It appends to any existing values associated +with the header name. Input: GET /foo HTTP/1.1 @@ -1428,7 +1898,13 @@ equivalent.
    		value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    		 true
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    		 true
    falsefalse
    + + +### HTTPProxy.spec.rules[index].backends[index].tls +[↩ Parent](#httpproxyspecrulesindexbackendsindex) + + + +TLS contains backend TLS configuration. + +When the backend endpoint uses HTTPS with an IP address, the Hostname field +must be specified for TLS certificate validation. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    hostnamestring + Hostname is used for TLS certificate validation when connecting to an +HTTPS backend. This hostname is used for: + +1. SNI (Server Name Indication) during the TLS handshake +2. Certificate validation - the certificate must be valid for this hostname + +This field is required when the backend endpoint uses HTTPS with an IP +address, as there is no hostname to extract from the endpoint URL. + +When the backend endpoint uses HTTPS with a DNS hostname, this field is +optional and defaults to the hostname from the endpoint URL.
    +     		false
    + + +### HTTPProxy.spec.rules[index].filters[index] +[↩ Parent](#httpproxyspecrulesindex) + + + +HTTPRouteFilter defines processing steps that must be completed during the +request or response lifecycle. HTTPRouteFilters are meant as an extension +point to express processing that may be done in Gateway implementations. Some +examples include request or response modification, implementing +authentication strategies, rate-limiting, and traffic shaping. API +guarantee/conformance is defined based on the type of the filter. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    typeenum + Type identifies the type of filter to apply. As with other API fields, +types are classified into three conformance levels: + +- Core: Filter types and their corresponding configuration defined by + "Support: Core" in this package, e.g. "RequestHeaderModifier". All + implementations must support core filters. + +- Extended: Filter types and their corresponding configuration defined by + "Support: Extended" in this package, e.g. "RequestMirror". Implementers + are encouraged to support extended filters. + +- Implementation-specific: Filters that are defined and supported by + specific vendors. + In the future, filters showing convergence in behavior across multiple + implementations will be considered for inclusion in extended or core + conformance levels. Filter-specific configuration for such filters + is specified using the ExtensionRef field. `Type` should be set to + "ExtensionRef" for custom filters. + +Implementers are encouraged to define custom implementation types to +extend the core API with implementation-specific behavior. + +If a reference to a custom filter type cannot be resolved, the filter +MUST NOT be skipped. Instead, requests that would have been processed by +that filter MUST receive a HTTP error response. + +Note that values may be added to this enum, implementations +must ensure that unknown values will not cause a crash. + +Unknown values here must result in the implementation setting the +Accepted Condition for the Route to `status: False`, with a +Reason of `UnsupportedValue`. + +
    +
    + Enum: RequestHeaderModifier, ResponseHeaderModifier, RequestMirror, RequestRedirect, URLRewrite, ExtensionRef, CORS
    +     		true
    corsobject + CORS defines a schema for a filter that responds to the +cross-origin request based on HTTP response header. + +Support: Extended
    +     		false
    extensionRefobject + ExtensionRef is an optional, implementation-specific extension to the +"filter" behavior. For example, resource "myroutefilter" in group +"networking.example.net"). ExtensionRef MUST NOT be used for core and +extended filters. + +This filter can be used multiple times within the same rule. + +Support: Implementation-specific
    +     		false
    externalAuthobject + ExternalAuth configures settings related to sending request details +to an external auth service. The external service MUST authenticate +the request, and MAY authorize the request as well. + +If there is any problem communicating with the external service, +this filter MUST fail closed. + +Support: Extended + +
    +
    + Validations:
  • self.protocol == 'GRPC' ? has(self.grpc) : true: grpc must be specified when protocol is set to 'GRPC'
  • has(self.grpc) ? self.protocol == 'GRPC' : true: protocol must be 'GRPC' when grpc is set
  • self.protocol == 'HTTP' ? has(self.http) : true: http must be specified when protocol is set to 'HTTP'
  • has(self.http) ? self.protocol == 'HTTP' : true: protocol must be 'HTTP' when http is set
    • +     		false
    requestHeaderModifierobject + RequestHeaderModifier defines a schema for a filter that modifies request +headers. + +Support: Core
    +     		false
    requestMirrorobject + RequestMirror defines a schema for a filter that mirrors requests. +Requests are sent to the specified destination, but responses from +that destination are ignored. + +This filter can be used multiple times within the same rule. Note that +not all implementations will be able to support mirroring to multiple +backends. + +Support: Extended
    +
    + Validations:
  • !(has(self.percent) && has(self.fraction)): Only one of percent or fraction may be specified in HTTPRequestMirrorFilter
    • +     		false
    requestRedirectobject + RequestRedirect defines a schema for a filter that responds to the +request with an HTTP redirection. + +Support: Core
    +     		false
    responseHeaderModifierobject + ResponseHeaderModifier defines a schema for a filter that modifies response +headers. + +Support: Extended
    +     		false
    urlRewriteobject + URLRewrite defines a schema for a filter that modifies a request during forwarding. + +Support: Extended
    +     		false
    + + +### HTTPProxy.spec.rules[index].filters[index].cors +[↩ Parent](#httpproxyspecrulesindexfiltersindex) + + + +CORS defines a schema for a filter that responds to the +cross-origin request based on HTTP response header. + +Support: Extended + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    allowCredentialsboolean + AllowCredentials indicates whether the actual cross-origin request allows +to include credentials. + +When set to true, the gateway will include the `Access-Control-Allow-Credentials` +response header with value true (case-sensitive). + +When set to false or omitted the gateway will omit the header +`Access-Control-Allow-Credentials` entirely (this is the standard CORS +behavior). + +Support: Extended
    +     		false
    allowHeaders[]string + AllowHeaders indicates which HTTP request headers are supported for +accessing the requested resource. + +Header names are not case-sensitive. + +Multiple header names in the value of the `Access-Control-Allow-Headers` +response header are separated by a comma (","). + +When the `AllowHeaders` field is configured with one or more headers, the +gateway must return the `Access-Control-Allow-Headers` response header +which value is present in the `AllowHeaders` field. + +If any header name in the `Access-Control-Request-Headers` request header +is not included in the list of header names specified by the response +header `Access-Control-Allow-Headers`, it will present an error on the +client side. + +If any header name in the `Access-Control-Allow-Headers` response header +does not recognize by the client, it will also occur an error on the +client side. + +A wildcard indicates that the requests with all HTTP headers are allowed. +If config contains the wildcard "*" in allowHeaders and the request is +not credentialed, the `Access-Control-Allow-Headers` response header +can either use the `*` wildcard or the value of +Access-Control-Request-Headers from the request. + +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Headers` response header. When +also the `AllowCredentials` field is true and `AllowHeaders` field +is specified with the `*` wildcard, the gateway must specify one or more +HTTP headers in the value of the `Access-Control-Allow-Headers` response +header. The value of the header `Access-Control-Allow-Headers` is same as +the `Access-Control-Request-Headers` header provided by the client. If +the header `Access-Control-Request-Headers` is not included in the +request, the gateway will omit the `Access-Control-Allow-Headers` +response header, instead of specifying the `*` wildcard. + +Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowHeaders cannot contain '*' alongside other methods
    • +     		false
    allowMethods[]enum + AllowMethods indicates which HTTP methods are supported for accessing the +requested resource. + +Valid values are any method defined by RFC9110, along with the special +value `*`, which represents all HTTP methods are allowed. + +Method names are case-sensitive, so these values are also case-sensitive. +(See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) + +Multiple method names in the value of the `Access-Control-Allow-Methods` +response header are separated by a comma (","). + +A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. +(See https://fetch.spec.whatwg.org/#cors-safelisted-method) The +CORS-safelisted methods are always allowed, regardless of whether they +are specified in the `AllowMethods` field. + +When the `AllowMethods` field is configured with one or more methods, the +gateway must return the `Access-Control-Allow-Methods` response header +which value is present in the `AllowMethods` field. + +If the HTTP method of the `Access-Control-Request-Method` request header +is not included in the list of methods specified by the response header +`Access-Control-Allow-Methods`, it will present an error on the client +side. + +If config contains the wildcard "*" in allowMethods and the request is +not credentialed, the `Access-Control-Allow-Methods` response header +can either use the `*` wildcard or the value of +Access-Control-Request-Method from the request. + +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Methods` response header. When +also the `AllowCredentials` field is true and `AllowMethods` field +specified with the `*` wildcard, the gateway must specify one HTTP method +in the value of the Access-Control-Allow-Methods response header. The +value of the header `Access-Control-Allow-Methods` is same as the +`Access-Control-Request-Method` header provided by the client. If the +header `Access-Control-Request-Method` is not included in the request, +the gateway will omit the `Access-Control-Allow-Methods` response header, +instead of specifying the `*` wildcard. + +Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowMethods cannot contain '*' alongside other methods
    • + Enum: GET, HEAD, POST, PUT, DELETE, CONNECT, OPTIONS, TRACE, PATCH, *
    +     		false
    allowOrigins[]string + AllowOrigins indicates whether the response can be shared with requested +resource from the given `Origin`. + +The `Origin` consists of a scheme and a host, with an optional port, and +takes the form `://(:)`. + +Valid values for scheme are: `http` and `https`. + +Valid values for port are any integer between 1 and 65535 (the list of +available TCP/UDP ports). Note that, if not included, port `80` is +assumed for `http` scheme origins, and port `443` is assumed for `https` +origins. This may affect origin matching. + +The host part of the origin may contain the wildcard character `*`. These +wildcard characters behave as follows: + +* `*` is a greedy match to the _left_, including any number of + DNS labels to the left of its position. This also means that + `*` will include any number of period `.` characters to the + left of its position. +* A wildcard by itself matches all hosts. + +An origin value that includes _only_ the `*` character indicates requests +from all `Origin`s are allowed. + +When the `AllowOrigins` field is configured with multiple origins, it +means the server supports clients from multiple origins. If the request +`Origin` matches the configured allowed origins, the gateway must return +the given `Origin` and sets value of the header +`Access-Control-Allow-Origin` same as the `Origin` header provided by the +client. + +The status code of a successful response to a "preflight" request is +always an OK status (i.e., 204 or 200). + +If the request `Origin` does not match the configured allowed origins, +the gateway returns 204/200 response but doesn't set the relevant +cross-origin response headers. Alternatively, the gateway responds with +403 status to the "preflight" request is denied, coupled with omitting +the CORS headers. The cross-origin request fails on the client side. +Therefore, the client doesn't attempt the actual cross-origin request. + +Conversely, if the request `Origin` matches one of the configured +allowed origins, the gateway sets the response header +`Access-Control-Allow-Origin` to the same value as the `Origin` +header provided by the client. + +When config has the wildcard ("*") in allowOrigins, and the request +is not credentialed (e.g., it is a preflight request), the +`Access-Control-Allow-Origin` response header either contains the +wildcard as well or the Origin from the request. + +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Origin` response header. When +also the `AllowCredentials` field is true and `AllowOrigins` field +specified with the `*` wildcard, the gateway must return a single origin +in the value of the `Access-Control-Allow-Origin` response header, +instead of specifying the `*` wildcard. The value of the header +`Access-Control-Allow-Origin` is same as the `Origin` header provided by +the client. + +Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowOrigins cannot contain '*' alongside other origins
    • +     		false
    exposeHeaders[]string + ExposeHeaders indicates which HTTP response headers can be exposed +to client-side scripts in response to a cross-origin request. + +A CORS-safelisted response header is an HTTP header in a CORS response +that it is considered safe to expose to the client scripts. +The CORS-safelisted response headers include the following headers: +`Cache-Control` +`Content-Language` +`Content-Length` +`Content-Type` +`Expires` +`Last-Modified` +`Pragma` +(See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) +The CORS-safelisted response headers are exposed to client by default. + +When an HTTP header name is specified using the `ExposeHeaders` field, +this additional header will be exposed as part of the response to the +client. + +Header names are not case-sensitive. + +Multiple header names in the value of the `Access-Control-Expose-Headers` +response header are separated by a comma (","). + +A wildcard indicates that the responses with all HTTP headers are exposed +to clients. The `Access-Control-Expose-Headers` response header can only +use `*` wildcard as value when the request is not credentialed. + +When the `exposeHeaders` config field contains the "*" wildcard and +the request is credentialed, the gateway cannot use the `*` wildcard in +the `Access-Control-Expose-Headers` response header. + +Support: Extended
    +     		false
    maxAgeinteger + MaxAge indicates the duration (in seconds) for the client to cache the +results of a "preflight" request. + +The information provided by the `Access-Control-Allow-Methods` and +`Access-Control-Allow-Headers` response headers can be cached by the +client until the time specified by `Access-Control-Max-Age` elapses. + +The default value of `Access-Control-Max-Age` response header is 5 +(seconds). + +When the `MaxAge` field is unspecified, the gateway sets the response +header "Access-Control-Max-Age: 5" by default.
    +
    + Format: int32
    + Default: 5
    + Minimum: 1
    +     		false
    + + +### HTTPProxy.spec.rules[index].filters[index].extensionRef +[↩ Parent](#httpproxyspecrulesindexfiltersindex) + + + +ExtensionRef is an optional, implementation-specific extension to the +"filter" behavior. For example, resource "myroutefilter" in group +"networking.example.net"). ExtensionRef MUST NOT be used for core and +extended filters. + +This filter can be used multiple times within the same rule. + +Support: Implementation-specific + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    groupstring + Group is the group of the referent. For example, "gateway.networking.k8s.io". +When unspecified or empty string, core API group is inferred.
    +     		true
    kindstring + Kind is kind of the referent. For example "HTTPRoute" or "Service".
    +     		true
    namestring + Name is the name of the referent.
    +     		true
    -### HTTPProxy.spec.rules[index].filters[index] -[↩ Parent](#httpproxyspecrulesindex) +### HTTPProxy.spec.rules[index].filters[index].externalAuth +[↩ Parent](#httpproxyspecrulesindexfiltersindex) -HTTPRouteFilter defines processing steps that must be completed during the -request or response lifecycle. HTTPRouteFilters are meant as an extension -point to express processing that may be done in Gateway implementations. Some -examples include request or response modification, implementing -authentication strategies, rate-limiting, and traffic shaping. API -guarantee/conformance is defined based on the type of the filter. +ExternalAuth configures settings related to sending request details +to an external auth service. The external service MUST authenticate +the request, and MAY authorize the request as well. + +If there is any problem communicating with the external service, +this filter MUST fail closed. + +Support: Extended - - + @@ -1613,144 +2624,104 @@ guarantee/conformance is defined based on the type of the filter. - - + + - - - - - - - + + - - - - - - + - + - - - - - - + - +
    typeenumbackendRefobject - Type identifies the type of filter to apply. As with other API fields, -types are classified into three conformance levels: - -- Core: Filter types and their corresponding configuration defined by - "Support: Core" in this package, e.g. "RequestHeaderModifier". All - implementations must support core filters. - -- Extended: Filter types and their corresponding configuration defined by - "Support: Extended" in this package, e.g. "RequestMirror". Implementers - are encouraged to support extended filters. - -- Implementation-specific: Filters that are defined and supported by - specific vendors. - In the future, filters showing convergence in behavior across multiple - implementations will be considered for inclusion in extended or core - conformance levels. Filter-specific configuration for such filters - is specified using the ExtensionRef field. `Type` should be set to - "ExtensionRef" for custom filters. - -Implementers are encouraged to define custom implementation types to -extend the core API with implementation-specific behavior. - -If a reference to a custom filter type cannot be resolved, the filter -MUST NOT be skipped. Instead, requests that would have been processed by -that filter MUST receive a HTTP error response. - -Note that values may be added to this enum, implementations -must ensure that unknown values will not cause a crash. + BackendRef is a reference to a backend to send authorization +requests to. -Unknown values here must result in the implementation setting the -Accepted Condition for the Route to `status: False`, with a -Reason of `UnsupportedValue`. +The backend must speak the selected protocol (GRPC or HTTP) on the +referenced port. -
    +If the backend service requires TLS, use BackendTLSPolicy to tell the +implementation to supply the TLS details to be used to connect to that +backend.

    - Enum: RequestHeaderModifier, ResponseHeaderModifier, RequestMirror, RequestRedirect, URLRewrite, ExtensionRef
    + Validations:
  • (size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true: Must have port for Service reference
    		•  true
    corsobject - CORS defines a schema for a filter that responds to the -cross-origin request based on HTTP response header. - -Support: Extended - -
    -     		false
    extensionRefobjectprotocolenum - ExtensionRef is an optional, implementation-specific extension to the -"filter" behavior. For example, resource "myroutefilter" in group -"networking.example.net"). ExtensionRef MUST NOT be used for core and -extended filters. + ExternalAuthProtocol describes which protocol to use when communicating with an +ext_authz authorization server. -This filter can be used multiple times within the same rule. +When this is set to GRPC, each backend must use the Envoy ext_authz protocol +on the port specified in `backendRefs`. Requests and responses are defined +in the protobufs explained at: +https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto -Support: Implementation-specific
    -     		false
    requestHeaderModifierobject - RequestHeaderModifier defines a schema for a filter that modifies request -headers. +When this is set to HTTP, each backend must respond with a `200` status +code in on a successful authorization. Any other code is considered +an authorization failure. -Support: Core
    +Feature Names: +GRPC Support - HTTPRouteExternalAuthGRPC +HTTP Support - HTTPRouteExternalAuthHTTP
    +
    + Enum: HTTP, GRPC
    		falsetrue
    requestMirrorforwardBody object - RequestMirror defines a schema for a filter that mirrors requests. -Requests are sent to the specified destination, but responses from -that destination are ignored. + ForwardBody controls if requests to the authorization server should include +the body of the client request; and if so, how big that body is allowed +to be. -This filter can be used multiple times within the same rule. Note that -not all implementations will be able to support mirroring to multiple -backends. +It is expected that implementations will buffer the request body up to +`forwardBody.maxSize` bytes. Bodies over that size must be rejected with a +4xx series error (413 or 403 are common examples), and fail processing +of the filter. -Support: Extended
    -
    - Validations:
  • !(has(self.percent) && has(self.fraction)): Only one of percent or fraction may be specified in HTTPRequestMirrorFilter
    • -     		false
    requestRedirectobject - RequestRedirect defines a schema for a filter that responds to the -request with an HTTP redirection. +If unset, or `forwardBody.maxSize` is set to `0`, then the body will not +be forwarded. -Support: Core
    +Feature Name: HTTPRouteExternalAuthForwardBody
    		 false
    responseHeaderModifiergrpc object - ResponseHeaderModifier defines a schema for a filter that modifies response -headers. + GRPCAuthConfig contains configuration for communication with ext_authz +protocol-speaking backends. -Support: Extended
    +If unset, implementations must assume the default behavior for each +included field is intended.
    		 false
    urlRewritehttp object - URLRewrite defines a schema for a filter that modifies a request during forwarding. + HTTPAuthConfig contains configuration for communication with HTTP-speaking +backends. -Support: Extended
    +If unset, implementations must assume the default behavior for each +included field is intended.
    		 false
    -### HTTPProxy.spec.rules[index].filters[index].cors -[↩ Parent](#httpproxyspecrulesindexfiltersindex) +### HTTPProxy.spec.rules[index].filters[index].externalAuth.backendRef +[↩ Parent](#httpproxyspecrulesindexfiltersindexexternalauth) -CORS defines a schema for a filter that responds to the -cross-origin request based on HTTP response header. +BackendRef is a reference to a backend to send authorization +requests to. -Support: Extended +The backend must speak the selected protocol (GRPC or HTTP) on the +referenced port. - +If the backend service requires TLS, use BackendTLSPolicy to tell the +implementation to supply the TLS details to be used to connect to that +backend. @@ -1762,247 +2733,173 @@ Support: Extended - - + + + + + + + - - + + - - + + - - + + + + +
    allowCredentialsbooleannamestring - AllowCredentials indicates whether the actual cross-origin request allows -to include credentials. - -The only valid value for the `Access-Control-Allow-Credentials` response -header is true (case-sensitive). - -If the credentials are not allowed in cross-origin requests, the gateway -will omit the header `Access-Control-Allow-Credentials` entirely rather -than setting its value to false. - -Support: Extended
    + Name is the name of the referent.
    +     		true
    groupstring + Group is the group of the referent. For example, "gateway.networking.k8s.io". +When unspecified or empty string, core API group is inferred.

    - Enum: true
    + Default:
    		false
    allowHeaders[]stringkindstring - AllowHeaders indicates which HTTP request headers are supported for -accessing the requested resource. - -Header names are not case sensitive. - -Multiple header names in the value of the `Access-Control-Allow-Headers` -response header are separated by a comma (","). - -When the `AllowHeaders` field is configured with one or more headers, the -gateway must return the `Access-Control-Allow-Headers` response header -which value is present in the `AllowHeaders` field. - -If any header name in the `Access-Control-Request-Headers` request header -is not included in the list of header names specified by the response -header `Access-Control-Allow-Headers`, it will present an error on the -client side. + Kind is the Kubernetes resource kind of the referent. For example +"Service". -If any header name in the `Access-Control-Allow-Headers` response header -does not recognize by the client, it will also occur an error on the -client side. +Defaults to "Service" when not specified. -A wildcard indicates that the requests with all HTTP headers are allowed. -The `Access-Control-Allow-Headers` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. +ExternalName services can refer to CNAME DNS records that may live +outside of the cluster and as such are difficult to reason about in +terms of conformance. They also may not be safe to forward to (see +CVE-2021-25740 for more information). Implementations SHOULD NOT +support ExternalName Services. -When the `AllowCredentials` field is specified and `AllowHeaders` field -specified with the `*` wildcard, the gateway must specify one or more -HTTP headers in the value of the `Access-Control-Allow-Headers` response -header. The value of the header `Access-Control-Allow-Headers` is same as -the `Access-Control-Request-Headers` header provided by the client. If -the header `Access-Control-Request-Headers` is not included in the -request, the gateway will omit the `Access-Control-Allow-Headers` -response header, instead of specifying the `*` wildcard. A Gateway -implementation may choose to add implementation-specific default headers. +Support: Core (Services with a type other than ExternalName) -Support: Extended
    +Support: Implementation-specific (Services with type ExternalName)
    +
    + Default: Service
    		 false
    allowMethods[]enumnamespacestring - AllowMethods indicates which HTTP methods are supported for accessing the -requested resource. - -Valid values are any method defined by RFC9110, along with the special -value `*`, which represents all HTTP methods are allowed. - -Method names are case sensitive, so these values are also case-sensitive. -(See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - -Multiple method names in the value of the `Access-Control-Allow-Methods` -response header are separated by a comma (","). - -A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. -(See https://fetch.spec.whatwg.org/#cors-safelisted-method) The -CORS-safelisted methods are always allowed, regardless of whether they -are specified in the `AllowMethods` field. - -When the `AllowMethods` field is configured with one or more methods, the -gateway must return the `Access-Control-Allow-Methods` response header -which value is present in the `AllowMethods` field. - -If the HTTP method of the `Access-Control-Request-Method` request header -is not included in the list of methods specified by the response header -`Access-Control-Allow-Methods`, it will present an error on the client -side. - -The `Access-Control-Allow-Methods` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. + Namespace is the namespace of the backend. When unspecified, the local +namespace is inferred. -When the `AllowCredentials` field is specified and `AllowMethods` field -specified with the `*` wildcard, the gateway must specify one HTTP method -in the value of the Access-Control-Allow-Methods response header. The -value of the header `Access-Control-Allow-Methods` is same as the -`Access-Control-Request-Method` header provided by the client. If the -header `Access-Control-Request-Method` is not included in the request, -the gateway will omit the `Access-Control-Allow-Methods` response header, -instead of specifying the `*` wildcard. A Gateway implementation may -choose to add implementation-specific default methods. +Note that when a namespace different than the local namespace is specified, +a ReferenceGrant object is required in the referent namespace to allow that +namespace's owner to accept the reference. See the ReferenceGrant +documentation for details. -Support: Extended
    -
    - Validations:
  • !('*' in self && self.size() > 1): AllowMethods cannot contain '*' alongside other methods
    • - Enum: GET, HEAD, POST, PUT, DELETE, CONNECT, OPTIONS, TRACE, PATCH, *
    +Support: Core
    		 false
    allowOrigins[]stringportinteger - AllowOrigins indicates whether the response can be shared with requested -resource from the given `Origin`. - -The `Origin` consists of a scheme and a host, with an optional port, and -takes the form `://(:)`. + Port specifies the destination port number to use for this resource. +Port is required when the referent is a Kubernetes Service. In this +case, the port number is the service port number, not the target port. +For other resources, destination port might be derived from the referent +resource or this field.
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    +     		false
    -Valid values for scheme are: `http` and `https`. -Valid values for port are any integer between 1 and 65535 (the list of -available TCP/UDP ports). Note that, if not included, port `80` is -assumed for `http` scheme origins, and port `443` is assumed for `https` -origins. This may affect origin matching. +### HTTPProxy.spec.rules[index].filters[index].externalAuth.forwardBody +[↩ Parent](#httpproxyspecrulesindexfiltersindexexternalauth) -The host part of the origin may contain the wildcard character `*`. These -wildcard characters behave as follows: -* `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. -* A wildcard by itself matches all hosts. -An origin value that includes _only_ the `*` character indicates requests -from all `Origin`s are allowed. +ForwardBody controls if requests to the authorization server should include +the body of the client request; and if so, how big that body is allowed +to be. -When the `AllowOrigins` field is configured with multiple origins, it -means the server supports clients from multiple origins. If the request -`Origin` matches the configured allowed origins, the gateway must return -the given `Origin` and sets value of the header -`Access-Control-Allow-Origin` same as the `Origin` header provided by the -client. +It is expected that implementations will buffer the request body up to +`forwardBody.maxSize` bytes. Bodies over that size must be rejected with a +4xx series error (413 or 403 are common examples), and fail processing +of the filter. -The status code of a successful response to a "preflight" request is -always an OK status (i.e., 204 or 200). +If unset, or `forwardBody.maxSize` is set to `0`, then the body will not +be forwarded. -If the request `Origin` does not match the configured allowed origins, -the gateway returns 204/200 response but doesn't set the relevant -cross-origin response headers. Alternatively, the gateway responds with -403 status to the "preflight" request is denied, coupled with omitting -the CORS headers. The cross-origin request fails on the client side. -Therefore, the client doesn't attempt the actual cross-origin request. +Feature Name: HTTPRouteExternalAuthForwardBody -The `Access-Control-Allow-Origin` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. + + + + + + + + + + + + + - - - - +
    NameTypeDescriptionRequired
    maxSizeinteger + MaxSize specifies how large in bytes the largest body that will be buffered +and sent to the authorization server. If the body size is larger than +`maxSize`, then the body sent to the authorization server must be +truncated to `maxSize` bytes. -When the `AllowCredentials` field is specified and `AllowOrigins` field -specified with the `*` wildcard, the gateway must return a single origin -in the value of the `Access-Control-Allow-Origin` response header, -instead of specifying the `*` wildcard. The value of the header -`Access-Control-Allow-Origin` is same as the `Origin` header provided by -the client. +Experimental note: This behavior needs to be checked against +various dataplanes; it may need to be changed. +See https://github.com/kubernetes-sigs/gateway-api/pull/4001#discussion_r2291405746 +for more. -Support: Extended
    +If 0, the body will not be sent to the authorization server.
    		 false
    exposeHeaders[]string - ExposeHeaders indicates which HTTP response headers can be exposed -to client-side scripts in response to a cross-origin request. +
    -A CORS-safelisted response header is an HTTP header in a CORS response -that it is considered safe to expose to the client scripts. -The CORS-safelisted response headers include the following headers: -`Cache-Control` -`Content-Language` -`Content-Length` -`Content-Type` -`Expires` -`Last-Modified` -`Pragma` -(See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) -The CORS-safelisted response headers are exposed to client by default. -When an HTTP header name is specified using the `ExposeHeaders` field, -this additional header will be exposed as part of the response to the -client. +### HTTPProxy.spec.rules[index].filters[index].externalAuth.grpc +[↩ Parent](#httpproxyspecrulesindexfiltersindexexternalauth) -Header names are not case sensitive. -Multiple header names in the value of the `Access-Control-Expose-Headers` -response header are separated by a comma (","). -A wildcard indicates that the responses with all HTTP headers are exposed -to clients. The `Access-Control-Expose-Headers` response header can only -use `*` wildcard as value when the `AllowCredentials` field is -unspecified. +GRPCAuthConfig contains configuration for communication with ext_authz +protocol-speaking backends. -Support: Extended
    - - false - - maxAge - integer +If unset, implementations must assume the default behavior for each +included field is intended. + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    allowedHeaders[]string - MaxAge indicates the duration (in seconds) for the client to cache the -results of a "preflight" request. + AllowedRequestHeaders specifies what headers from the client request +will be sent to the authorization server. -The information provided by the `Access-Control-Allow-Methods` and -`Access-Control-Allow-Headers` response headers can be cached by the -client until the time specified by `Access-Control-Max-Age` elapses. +If this list is empty, then all headers must be sent. -The default value of `Access-Control-Max-Age` response header is 5 -(seconds).
    -
    - Format: int32
    - Default: 5
    - Minimum: 1
    +If the list has entries, only those entries must be sent.
    		 false
    -### HTTPProxy.spec.rules[index].filters[index].extensionRef -[↩ Parent](#httpproxyspecrulesindexfiltersindex) - +### HTTPProxy.spec.rules[index].filters[index].externalAuth.http +[↩ Parent](#httpproxyspecrulesindexfiltersindexexternalauth) -ExtensionRef is an optional, implementation-specific extension to the -"filter" behavior. For example, resource "myroutefilter" in group -"networking.example.net"). ExtensionRef MUST NOT be used for core and -extended filters. -This filter can be used multiple times within the same rule. +HTTPAuthConfig contains configuration for communication with HTTP-speaking +backends. -Support: Implementation-specific +If unset, implementations must assume the default behavior for each +included field is intended. @@ -2014,27 +2911,60 @@ Support: Implementation-specific - - + + - + - - + + - + - + - +
    groupstringallowedHeaders[]string - Group is the group of the referent. For example, "gateway.networking.k8s.io". -When unspecified or empty string, core API group is inferred.
    + AllowedRequestHeaders specifies what additional headers from the client request +will be sent to the authorization server. + +The following headers must always be sent to the authorization server, +regardless of this setting: + +* `Host` +* `Method` +* `Path` +* `Content-Length` +* `Authorization` + +If this list is empty, then only those headers must be sent. + +Note that `Content-Length` has a special behavior, in that the length +sent must be correct for the actual request to the external authorization +server - that is, it must reflect the actual number of bytes sent in the +body of the request to the authorization server. + +So if the `forwardBody` stanza is unset, or `forwardBody.maxSize` is set +to `0`, then `Content-Length` must be `0`. If `forwardBody.maxSize` is set +to anything other than `0`, then the `Content-Length` of the authorization +request must be set to the actual number of bytes forwarded.
    		truefalse
    kindstringallowedResponseHeaders[]string - Kind is kind of the referent. For example "HTTPRoute" or "Service".
    + AllowedResponseHeaders specifies what headers from the authorization response +will be copied into the request to the backend. + +If this list is empty, then all headers from the authorization server +except Authority or Host must be copied.
    		truefalse
    namepath string - Name is the name of the referent.
    + Path sets the prefix that paths from the client request will have added +when forwarded to the authorization server. + +When empty or unspecified, no prefix is added. + +Valid values are the same as the "value" regex for path values in the `match` +stanza, and the validation regex will screen out invalid paths in the same way. +Even with the validation, implementations MUST sanitize this input before using it +directly.
    		truefalse
    @@ -2162,7 +3092,13 @@ equivalent.
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -2203,7 +3139,13 @@ equivalent.
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -2566,7 +3508,7 @@ Reason of `UnsupportedValue`. Support: Core

    - Enum: 301, 302
    + Enum: 301, 302, 303, 307, 308
    Default: 302
    false @@ -2768,7 +3710,13 @@ equivalent.
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -2809,7 +3757,13 @@ equivalent.
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -3049,7 +4003,13 @@ processing a repeated header, with special handling for "Set-Cookie".
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -3216,6 +4176,17 @@ This field will not contain custom hostnames defined in the HTTPProxy. See the `hostnames` field
    false + + canonicalHostname + string + + CanonicalHostname is the platform-managed stable hostname assigned to this +HTTPProxy (e.g., ".datumproxy.net"). Users may create external CNAME +or ALIAS records pointing to this hostname to route traffic through the +platform. The platform manages A/AAAA records for this hostname in the +datumproxy.net zone.
    + + false conditions []object @@ -3223,6 +4194,16 @@ the `hostnames` field
    Conditions describe the current conditions of the HTTPProxy.
    false + + hostnameStatuses + []object + + HostnameStatuses lists the per-hostname status for each hostname configured +on this HTTPProxy. Each entry includes verification and DNS record +programming conditions. Use this field instead of the deprecated Hostnames +field for detailed per-hostname lifecycle information.
    + + false hostnames []string @@ -3230,7 +4211,10 @@ the `hostnames` field
    Hostnames lists the hostnames that have been bound to the HTTPProxy. If this list does not match that defined in the HTTPProxy, see the -`HostnamesVerified` condition message for details.
    +`HostnamesVerified` condition message for details. + +Deprecated: Use HostnameStatuses for detailed per-hostname status. +This field will be removed in a future API version.
    false @@ -3281,6 +4265,121 @@ Examples: `1.2.3.4`, `128::1`, `my-ip-address`.
    +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### HTTPProxy.status.hostnameStatuses[index] +[↩ Parent](#httpproxystatus) + + + +HostnameStatus captures the per-hostname verification and DNS programming status. +Each hostname configured on an HTTPProxy has a corresponding entry tracking +its lifecycle from domain ownership verification through DNS record creation. + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    hostnamestring + Hostname is the fully qualified domain name being tracked. +Must be a valid RFC 1123 hostname without a trailing dot.
    +     		true
    conditions[]object + Conditions contains the current status conditions for this hostname. +Standard condition types include Verified and DNSRecordProgrammed.
    +     		false
    + + +### HTTPProxy.status.hostnameStatuses[index].conditions[index] +[↩ Parent](#httpproxystatushostnamestatusesindex) + + + Condition contains details for one aspect of the current state of this API Resource. diff --git a/docs/api/locationbindings.md b/docs/api/locationbindings.md new file mode 100644 index 00000000..d93dd036 --- /dev/null +++ b/docs/api/locationbindings.md @@ -0,0 +1,256 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha](#networkingdatumapiscomv1alpha) + +# networking.datumapis.com/v1alpha + +Resource Types: + +- [LocationBinding](#locationbinding) + + + + +## LocationBinding +[↩ Parent](#networkingdatumapiscomv1alpha ) + + + + + + +LocationBinding is the Schema for the locationbindings API. It is a +cluster-scoped projection of a cluster-scoped Location into a project's +virtual control plane, created once the location's class is supported, the +Location is Ready, and the corresponding ServiceAvailability is Available. + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    apiVersionstringnetworking.datumapis.com/v1alphatrue
    kindstringLocationBindingtrue
    metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
    specobject + LocationBindingSpec defines the desired state of LocationBinding.
    +     		false
    statusobject + LocationBindingStatus defines the observed state of LocationBinding.
    +     		false
    + + +### LocationBinding.spec +[↩ Parent](#locationbinding) + + + +LocationBindingSpec defines the desired state of LocationBinding. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    locationRefobject + LocationRef references the canonical cluster-scoped Location object.
    +     		true
    displayNamestring + DisplayName is a human-readable label for the location.
    +     		false
    locationClassNamestring + LocationClassName mirrors spec.locationClassName from the referenced Location.
    +     		false
    topologymap[string]string + Topology mirrors spec.topology from the referenced Location, containing +well-known keys like topology.datum.net/city-code and topology.datum.net/region.
    +     		false
    + + +### LocationBinding.spec.locationRef +[↩ Parent](#locationbindingspec) + + + +LocationRef references the canonical cluster-scoped Location object. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of the referent. +This field is effectively required, but due to backwards compatibility is +allowed to be empty. Instances of this type with an empty value here are +almost certainly wrong. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
    +
    + Default:
    +     		false
    + + +### LocationBinding.status +[↩ Parent](#locationbinding) + + + +LocationBindingStatus defines the observed state of LocationBinding. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    conditions[]object +
    +     		false
    + + +### LocationBinding.status.conditions[index] +[↩ Parent](#locationbindingstatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    diff --git a/docs/api/trafficprotectionpolicies.md b/docs/api/trafficprotectionpolicies.md index d710060e..907fa747 100644 --- a/docs/api/trafficprotectionpolicies.md +++ b/docs/api/trafficprotectionpolicies.md @@ -192,7 +192,7 @@ Core Rule Set (CRS). paranoia levels to use.

    Validations:
  • self.detection >= self.blocking: detection paranoia level must be greater than or equal to blocking paranoia level
    • - Default: map[]
    + Default: map[blocking:1 detection:1]
    false @@ -527,6 +527,43 @@ with a composite key made up of the AncestorRef and the ControllerName. PolicyAncestorStatus struct describes the status of.
    true + + conditions + []object + + Conditions describes the status of the Policy with respect to the given Ancestor. + + + +Notes for implementors: + +Conditions are a listType `map`, which means that they function like a +map with a key of the `type` field _in the k8s apiserver_. + +This means that implementations must obey some rules when updating this +section. + +* Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. +* Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. +* Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. +* Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. +* If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + +
    + + true controllerName string @@ -546,13 +583,6 @@ entries to status populated with their ControllerName are cleaned up when they a longer necessary.
    true - - conditions - []object - - Conditions describes the status of the Policy with respect to the given Ancestor.
    - - false From 73a100b1128d17427e25a687788c9638fe82f5e7 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 12:24:17 -0500 Subject: [PATCH 02/13] feat: add NetworkInterfaceClaim and NetworkInterface Compute asks for an interface the way a pod asks for storage. It writes a claim naming a network and what the interface needs, and the operator binds it to an interface carrying everything required to configure a NIC. Two properties follow. Compute stops reaching into networking internals, and an interface becomes a thing that outlives the instance using it, which is what makes a per-instance address and a retained address possible at once. Key changes: - A claim states intent: network, families, extra addresses by class, and reclaim policy. It carries no location, because the cell it lives in serves one. - An interface is the answer: addresses, gateway, MTU, and the claim holding it. Addresses live in spec because they are the desired NIC configuration rather than an observation. - A claim binds once, to the interface of its own name, and never searches. A replacement instance derives the same name and returns to the same addresses. - The fields addresses are allocated against are immutable, because the release path derives what to free from them. - A template lets a workload declare an interface once and materialize one claim per slot, as a StatefulSet does for storage. - Allocated and Programmed are reported apart, so an instance is never told its network is ready before it can carry traffic. --- PROJECT | 18 + api/v1alpha/groupversion_info.go | 4 + api/v1alpha/networkinterface_types.go | 329 ++++++++ api/v1alpha/networkinterfaceclaim_types.go | 253 ++++++ api/v1alpha/zz_generated.deepcopy.go | 351 ++++++++ ....datumapis.com_networkinterfaceclaims.yaml | 437 ++++++++++ ...rking.datumapis.com_networkinterfaces.yaml | 385 +++++++++ config/crd/kustomization.yaml | 2 + docs/api/networkinterfaceclaims.md | 580 ++++++++++++++ docs/api/networkinterfaces.md | 580 ++++++++++++++ docs/enhancements/network-interfaces.md | 756 ++++++++++++++++++ 11 files changed, 3695 insertions(+) create mode 100644 api/v1alpha/networkinterface_types.go create mode 100644 api/v1alpha/networkinterfaceclaim_types.go create mode 100644 config/crd/bases/networking.datumapis.com_networkinterfaceclaims.yaml create mode 100644 config/crd/bases/networking.datumapis.com_networkinterfaces.yaml create mode 100644 docs/api/networkinterfaceclaims.md create mode 100644 docs/api/networkinterfaces.md create mode 100644 docs/enhancements/network-interfaces.md diff --git a/PROJECT b/PROJECT index 59ebc91d..706f9797 100644 --- a/PROJECT +++ b/PROJECT @@ -185,4 +185,22 @@ resources: kind: ConnectorClass path: go.datum.net/network-services-operator/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: datumapis.com + group: networking + kind: NetworkInterface + path: go.datum.net/network-services-operator/api/v1alpha + version: v1alpha +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: datumapis.com + group: networking + kind: NetworkInterfaceClaim + path: go.datum.net/network-services-operator/api/v1alpha + version: v1alpha version: "3" diff --git a/api/v1alpha/groupversion_info.go b/api/v1alpha/groupversion_info.go index 5d70324a..bf4230a5 100644 --- a/api/v1alpha/groupversion_info.go +++ b/api/v1alpha/groupversion_info.go @@ -38,6 +38,10 @@ func addKnownTypes(scheme *runtime.Scheme) error { &NetworkBindingList{}, &NetworkContext{}, &NetworkContextList{}, + &NetworkInterface{}, + &NetworkInterfaceList{}, + &NetworkInterfaceClaim{}, + &NetworkInterfaceClaimList{}, &NetworkPolicy{}, &NetworkPolicyList{}, &Subnet{}, diff --git a/api/v1alpha/networkinterface_types.go b/api/v1alpha/networkinterface_types.go new file mode 100644 index 00000000..4d3563f8 --- /dev/null +++ b/api/v1alpha/networkinterface_types.go @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NetworkInterfaceReclaimPolicy decides what becomes of an interface, and the +// addresses it holds, when the claim bound to it is deleted. +// +// The two policies differ only when a workload goes away: on scale-down, on +// deletion, or whenever a claim is removed. While a workload is running, or +// while an instance is being replaced, both policies keep the addresses. +// +// +kubebuilder:validation:Enum=Delete;Retain +type NetworkInterfaceReclaimPolicy string + +const ( + // NetworkInterfaceReclaimPolicyDelete deletes the interface and returns its + // addresses to IPAM. A workload recreated later gets new addresses. + NetworkInterfaceReclaimPolicyDelete NetworkInterfaceReclaimPolicy = "Delete" + + // NetworkInterfaceReclaimPolicyRetain keeps the interface and its addresses + // after the claim is gone. The interface returns to the Available phase and + // keeps holding the addresses, so a later claim of the same name binds the + // same interface and the workload comes back on the same addresses. The + // addresses stay reserved, and billable, for as long as the interface + // exists. Deleting the interface releases its claim on them but does not + // return them to the pool today, so a retained address is reclaimed by an + // operator rather than automatically. + NetworkInterfaceReclaimPolicyRetain NetworkInterfaceReclaimPolicy = "Retain" +) + +// NetworkInterfacePhase reports whether an interface is held by a claim. +// +// +kubebuilder:validation:Enum=Available;Bound +type NetworkInterfacePhase string + +const ( + // NetworkInterfacePhaseAvailable means the interface still holds its addresses + // but no claim is bound to it. Retained interfaces wait here for a claim of + // the matching name. + NetworkInterfacePhaseAvailable NetworkInterfacePhase = "Available" + + // NetworkInterfacePhaseBound means the claim named in spec.claimRef holds the + // interface. + NetworkInterfacePhaseBound NetworkInterfacePhase = "Bound" +) + +const ( + // NetworkInterfaceAllocated reports that every address the interface must + // carry is allocated and recorded in spec. + NetworkInterfaceAllocated = "Allocated" + + // NetworkInterfaceProgrammed reports that the data plane carries the + // interface's addresses. Traffic flows only once this is true. + NetworkInterfaceProgrammed = "Programmed" +) + +// NetworkInterfaceAddress is an address the interface holds inside its network. +// These are the addresses configured on the NIC itself, and they always carry a +// prefix length. +type NetworkInterfaceAddress struct { + // family is the address family of this entry. + // + // +kubebuilder:validation:Required + Family IPFamily `json:"family"` + + // address is the address the interface holds, in CIDR notation, such as + // 10.128.0.2/32 or 2001:db8:a001::1/128. + // + // For IPv6 this may be a block delegated to the interface rather than a + // single address, such as 2001:db8:a001::/96. The interface owns the whole + // block and assigns within it. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=45 + Address string `json:"address"` + + // gateway is the next hop the interface routes through for this family, such + // as 10.128.0.1. It is resolved from the subnet backing the network in this + // location, so nothing has to read the subnet to configure the NIC. It is + // empty until that subnet exists. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxLength=45 + Gateway string `json:"gateway,omitempty"` + + // primary marks the address projected into single-address fields, such as an + // instance's reported network IP. + // + // Exactly one address is primary for the interface as a whole, not one per + // family. It is the address of the first family the claim listed in + // spec.ipFamilies. + // + // +kubebuilder:validation:Optional + Primary bool `json:"primary,omitempty"` + + // class is the IPAM class this address was allocated from, such as + // private-ipv6. It is empty for the addresses a claim requests by family + // rather than by class. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxLength=63 + Class string `json:"class,omitempty"` +} + +// NetworkInterfaceExternalAddress is an address reachable from outside the +// network, mapped onto an address the interface holds inside it. A public IPv4 +// address in front of a private address is the usual case. +// +// Unlike an interface address, an external address is a bare address with no +// prefix length, such as 203.0.113.10, because nothing configures it on the +// NIC. The data plane maps it onto the interface address of the same family. +type NetworkInterfaceExternalAddress struct { + // family is the address family of this entry. + // + // +kubebuilder:validation:Required + Family IPFamily `json:"family"` + + // address is the externally reachable address, such as 203.0.113.10. It + // carries no prefix length. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=45 + Address string `json:"address"` + + // class is the IPAM class this address was allocated from, such as + // public-ipv4. It matches the class the claim requested in spec.addresses. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Class string `json:"class"` +} + +// NetworkInterfaceClaimRef identifies the claim holding an interface. +type NetworkInterfaceClaimRef struct { + // name is the name of the NetworkInterfaceClaim, in the same namespace as the + // interface. A claim name stays with the workload slot it serves, so a + // replacement instance binds this same interface and its addresses. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` +} + +// LocalNetworkInterfaceRef references a NetworkInterface in the same namespace. +type LocalNetworkInterfaceRef struct { + // name is the network interface name. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` +} + +// NetworkInterfaceAttachmentRef references the provider resource realizing an +// interface on the data plane, such as an instance NIC attachment. It tells an +// operator what is carrying the interface, and it is written by the provider +// rather than by a user. +type NetworkInterfaceAttachmentRef struct { + // apiGroup is the API group of the referent, such as + // compute.datumapis.com. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + APIGroup string `json:"apiGroup"` + + // kind is the kind of the referent. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Kind string `json:"kind"` + + // name is the name of the referent. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` +} + +// NetworkInterfaceSpec defines the desired state of NetworkInterface. It is +// written by the operator when a claim is fulfilled, and it carries everything +// a provider needs to configure a NIC without reading any other resource. +type NetworkInterfaceSpec struct { + // network is the network this interface belongs to, in the same namespace as + // the interface. It comes from the claim and does not change. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:message="network is immutable and cannot be changed after creation",rule="self == oldSelf" + Network LocalNetworkRef `json:"network"` + + // claimRef is the claim currently holding this interface. It is empty while a + // retained interface waits, unbound, for a claim of its name to return. + // + // +kubebuilder:validation:Optional + ClaimRef *NetworkInterfaceClaimRef `json:"claimRef,omitempty"` + + // interfaceName is the device name the interface presents to the guest + // operating system, such as eth0 or eth1. It comes from the claim. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=15 + // +kubebuilder:default="eth0" + InterfaceName string `json:"interfaceName,omitempty"` + + // mtu is the MTU, in bytes, the interface must be configured with. It is + // resolved from the network, so a provider never has to read the network to + // configure the NIC. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Minimum=1300 + // +kubebuilder:validation:Maximum=8856 + MTU int32 `json:"mtu,omitempty"` + + // addresses are the addresses the interface holds inside its network, at most + // one per address family, exactly one of them primary. Each carries a prefix + // length and, once the location has a subnet, the gateway to route through. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxItems=4 + // +kubebuilder:validation:XValidation:message="Exactly one address must be primary",rule="size(self) == 0 || self.filter(a, has(a.primary) && a.primary).size() == 1" + // +kubebuilder:validation:XValidation:message="Only one address may be held per address family",rule="self.all(a, self.exists_one(b, b.family == a.family))" + Addresses []NetworkInterfaceAddress `json:"addresses,omitempty"` + + // externalAddresses are the addresses the interface is reachable at from + // outside the network, each mapped onto the interface address of the same + // family. They come from the classes the claim requested, and they are absent + // for a workload that only needs private addressing. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxItems=4 + // +kubebuilder:validation:XValidation:message="External addresses must be unique",rule="self.all(a, self.exists_one(b, b.address == a.address))" + // +kubebuilder:validation:XValidation:message="Only one external address may be held per address class",rule="self.all(a, self.exists_one(b, b.class == a.class))" + ExternalAddresses []NetworkInterfaceExternalAddress `json:"externalAddresses,omitempty"` + + // reclaimPolicy decides what becomes of this interface, and its addresses, + // when the claim holding it is deleted. It comes from the claim, and a claim + // asking for a different policy cannot bind this interface. + // + // +kubebuilder:validation:Optional + // +kubebuilder:default="Delete" + ReclaimPolicy NetworkInterfaceReclaimPolicy `json:"reclaimPolicy,omitempty"` +} + +// NetworkInterfaceStatus defines the observed state of NetworkInterface: which +// claim holds it, what realizes it on the data plane, and whether programming +// has succeeded. +type NetworkInterfaceStatus struct { + // phase reports whether a claim holds the interface. Bound means the claim in + // spec.claimRef holds it. Available means it is retained and holding its + // addresses with no claim bound. + // + // +kubebuilder:validation:Optional + Phase NetworkInterfacePhase `json:"phase,omitempty"` + + // networkContextRef is the network's presence in this location, resolved or + // created while fulfilling the claim. It is a breadcrumb for operators + // tracing where a network landed, and nothing needs it to configure a NIC. + // + // +kubebuilder:validation:Optional + NetworkContextRef *LocalNetworkContextRef `json:"networkContextRef,omitempty"` + + // attachmentRef is the data-plane resource realizing this interface. The + // provider sets it once an attachment exists. + // + // +kubebuilder:validation:Optional + AttachmentRef *NetworkInterfaceAttachmentRef `json:"attachmentRef,omitempty"` + + // vpc is the base62 identifier of the VPC backing this network in this + // location, matching the identifier the fabric keys on. The provider records + // it when the attachment is programmed. + // + // +kubebuilder:validation:Optional + VPC string `json:"vpc,omitempty"` + + // conditions report the current state of the interface. Allocated means every + // address is held. Programmed means the data plane carries them. + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// NetworkInterface is an interface on a network, together with the addresses it +// holds. It is the unit that owns addresses: as long as the interface exists, +// its addresses stay allocated to it. +// +// You do not create a NetworkInterface. Ask for one with a +// NetworkInterfaceClaim, and the operator creates the interface, allocates its +// addresses, and binds the two. A provider then reads the interface to +// configure a NIC, and reports what it programmed in status. +// +// An interface outlives the instance using it. Whether it outlives the claim +// that asked for it depends on spec.reclaimPolicy. +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Network",type=string,JSONPath=".spec.network.name" +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Claim",type=string,JSONPath=".spec.claimRef.name" +// +kubebuilder:printcolumn:name="Allocated",type=string,JSONPath=`.status.conditions[?(@.type=="Allocated")].status` +// +kubebuilder:printcolumn:name="Programmed",type=string,JSONPath=`.status.conditions[?(@.type=="Programmed")].status` +type NetworkInterface struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +kubebuilder:validation:Required + Spec NetworkInterfaceSpec `json:"spec,omitempty"` + + // +kubebuilder:default={conditions:{{type:"Allocated",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} + Status NetworkInterfaceStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetworkInterfaceList contains a list of NetworkInterface. +type NetworkInterfaceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkInterface `json:"items"` +} diff --git a/api/v1alpha/networkinterfaceclaim_types.go b/api/v1alpha/networkinterfaceclaim_types.go new file mode 100644 index 00000000..b30fa394 --- /dev/null +++ b/api/v1alpha/networkinterfaceclaim_types.go @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // NetworkInterfaceClaimBound reports that an interface is bound to the claim + // and named in status.networkInterfaceRef. + NetworkInterfaceClaimBound = "Bound" + + // NetworkInterfaceClaimAllocated reports that every requested address family, + // and every requested class, holds an address. + NetworkInterfaceClaimAllocated = "Allocated" + + // NetworkInterfaceClaimProgrammed reports that the data plane carries the + // claimed addresses. + NetworkInterfaceClaimProgrammed = "Programmed" + + // NetworkInterfaceClaimReady reports that the claim is bound, allocated, and + // programmed. A workload that needs the network should wait on this one + // condition rather than on the three it summarizes. + NetworkInterfaceClaimReady = "Ready" +) + +// NetworkInterfaceAddressRequest asks for one address beyond the ones the +// interface holds inside its network, such as a public IPv4 address in front of +// a private one. +type NetworkInterfaceAddressRequest struct { + // class is the IPAM class to allocate from, such as public-ipv4. + // + // A class names a kind of address, and the platform decides which pool and + // prefix length serve it. A class never names a pool, a prefix length, or a + // CIDR, so a class cannot be used to ask for a particular address. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Class string `json:"class"` +} + +// NetworkInterfaceClaimSpec defines the desired state of NetworkInterfaceClaim. +// Every field states what the interface must be able to do, never which +// interface or which address to use. +// +// Most of the spec is immutable, because the addresses are allocated against +// it. To change one of those fields, delete the claim and create a new one, +// accepting that the workload gets new addresses unless the interface is +// retained. +// +// +kubebuilder:validation:XValidation:message="networkInterfaceName is immutable and cannot be set, changed, or cleared after creation",rule="has(self.networkInterfaceName) == has(oldSelf.networkInterfaceName) && (!has(self.networkInterfaceName) || self.networkInterfaceName == oldSelf.networkInterfaceName)" +// +kubebuilder:validation:XValidation:message="addresses is immutable and cannot be set, changed, or cleared after creation",rule="has(self.addresses) == has(oldSelf.addresses) && (!has(self.addresses) || self.addresses == oldSelf.addresses)" +type NetworkInterfaceClaimSpec struct { + // network is the network the interface attaches to. The network must already + // exist in the same namespace as the claim. + // + // Immutable. An interface that changed network would hold addresses from a + // space it no longer belongs to, so move a workload by recreating the claim + // against the other network. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:message="network is immutable and cannot be changed after creation",rule="self == oldSelf" + Network LocalNetworkRef `json:"network"` + + // interfaceName is the device name the interface presents to the guest + // operating system, such as eth0 or eth1. Set it when a workload has more + // than one interface and the guest configuration names them. + // + // Immutable, because the guest is configured against it. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=15 + // +kubebuilder:default="eth0" + // +kubebuilder:validation:XValidation:message="interfaceName is immutable and cannot be changed after creation",rule="self == oldSelf" + InterfaceName string `json:"interfaceName,omitempty"` + + // ipFamilies are the address families the interface must carry, in priority + // order. List [IPv6, IPv4] for a dual-stack interface. The first family + // listed holds the interface's primary address, which is the one reported in + // single-address fields such as an instance's network IP. + // + // Every family listed must be satisfiable or the claim does not bind. Asking + // for a family the network does not carry fails the claim outright rather + // than leaving it pending, and no partially addressed interface is ever + // published. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=2 + // +kubebuilder:default={IPv6} + // +kubebuilder:validation:XValidation:message="Each address family may be requested at most once",rule="self.all(f, self.exists_one(g, g == f))" + // +kubebuilder:validation:XValidation:message="ipFamilies is immutable and cannot be changed after creation",rule="self == oldSelf" + IPFamilies []IPFamily `json:"ipFamilies,omitempty"` + + // addresses request extra addresses by class, beyond the ones the interface + // holds inside its network. Each appears in status.externalAddresses as a + // bare address, mapped onto the interface address of the same family. + // + // Omit this field for ordinary private addressing, which is the common case. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=4 + // +kubebuilder:validation:XValidation:message="Each address class may be requested at most once",rule="self.all(a, self.exists_one(b, b.class == a.class))" + Addresses []NetworkInterfaceAddressRequest `json:"addresses,omitempty"` + + // reclaimPolicy decides what becomes of the bound interface, and its + // addresses, when this claim is deleted. + // + // Delete deletes the interface and returns its addresses to IPAM. A workload + // recreated later comes back on different addresses. + // + // Retain keeps the interface, unbound and still holding its addresses, so a + // later claim of this name binds it again and the workload returns to the + // same addresses. Choose Retain when an address is published in DNS, allowed + // through a firewall, or otherwise depended on from outside. + // + // A retained address is reserved, and billable, for as long as the interface + // exists. Deleting the interface does not return it to the pool today, so + // choose Retain for addresses worth holding rather than as a default. + // + // Both policies keep the addresses while the claim exists, including across + // instance replacement. They differ only on scale-down and deletion. + // + // Immutable. An address keeps the policy it was allocated under, and a claim + // asking for a policy the interface was not allocated under cannot bind it. + // + // +kubebuilder:validation:Optional + // +kubebuilder:default="Delete" + // +kubebuilder:validation:XValidation:message="reclaimPolicy is immutable and cannot be changed after creation",rule="self == oldSelf" + ReclaimPolicy NetworkInterfaceReclaimPolicy `json:"reclaimPolicy,omitempty"` + + // networkInterfaceName binds one specific interface by name, instead of the + // interface named after this claim. The named interface must already carry + // every family and class this claim asks for, under the same reclaim policy, + // and must not be held by another claim. + // + // Leave it empty, which is the normal case. The claim then binds the + // interface of its own name, retained by an earlier claim, or creates one. + // + // Immutable, including from empty to set. Rebinding a workload to a different + // interface means a new claim. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + NetworkInterfaceName string `json:"networkInterfaceName,omitempty"` +} + +// NetworkInterfaceClaimStatus defines the observed state of +// NetworkInterfaceClaim. It repeats the bound interface's addresses so a +// consumer reads one object rather than following the reference. +type NetworkInterfaceClaimStatus struct { + // networkInterfaceRef is the interface bound to this claim, in the same + // namespace. Read it to reach fields the claim does not repeat, such as the + // MTU and the data-plane attachment. + // + // +kubebuilder:validation:Optional + NetworkInterfaceRef *LocalNetworkInterfaceRef `json:"networkInterfaceRef,omitempty"` + + // addresses are the addresses the bound interface holds inside its network, + // each with its prefix length and, once the location has a subnet, its + // gateway. They are copied from the interface, which remains the source of + // truth. + // + // +kubebuilder:validation:Optional + Addresses []NetworkInterfaceAddress `json:"addresses,omitempty"` + + // externalAddresses are the addresses the bound interface is reachable at from + // outside the network, one per class the claim requested. Each is a bare + // address with no prefix length. They are copied from the interface. + // + // +kubebuilder:validation:Optional + ExternalAddresses []NetworkInterfaceExternalAddress `json:"externalAddresses,omitempty"` + + // conditions report the current state of the claim. Wait on Ready, which is + // true once the claim is bound, its addresses are allocated, and the data + // plane carries them. + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// NetworkInterfaceClaim asks for an interface on a network. It is the resource +// a user creates. The operator finds or creates a NetworkInterface that +// satisfies it, allocates the addresses, and reports them in status. +// +// A claim describes what the interface must be able to do, never which +// interface or address to use. One claim holds at most one interface, and one +// interface is held by at most one claim. +// +// A claim's name is what makes addresses stable. It names the slot in a +// workload rather than the instance filling it, so an instance replaced by +// another that asks for the same claim name comes back on the same interface +// and the same addresses. What happens when the claim itself is deleted is +// spec.reclaimPolicy. +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Network",type=string,JSONPath=".spec.network.name" +// +kubebuilder:printcolumn:name="Interface",type=string,JSONPath=".status.networkInterfaceRef.name" +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +type NetworkInterfaceClaim struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +kubebuilder:validation:Required + Spec NetworkInterfaceClaimSpec `json:"spec,omitempty"` + + // +kubebuilder:default={conditions:{{type:"Bound",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Allocated",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} + Status NetworkInterfaceClaimStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetworkInterfaceClaimList contains a list of NetworkInterfaceClaim. +type NetworkInterfaceClaimList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkInterfaceClaim `json:"items"` +} + +// NetworkInterfaceClaimTemplate describes an interface a workload needs, from +// which one NetworkInterfaceClaim is created per slot. It is embedded in a +// workload API the way a StatefulSet embeds volumeClaimTemplates, and serves +// the same purpose: the workload declares the interface once, and every slot +// gets a claim of its own that outlives the instance filling it. +// +// Each claim is named from the template and the slot, and both parts stay the +// same for the life of that slot, so a replacement instance derives the same +// name, finds the claim already there, and returns to the addresses it was +// already holding. +type NetworkInterfaceClaimTemplate struct { + // metadata is the standard object metadata for the claims this template + // produces. + // + // The name distinguishes one interface of a slot from another and becomes + // part of every claim's name. Labels and annotations are copied onto every + // claim produced. No other field is honoured. + // + // +kubebuilder:validation:Optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // spec is the claim created for each slot. It is copied onto every claim the + // template produces, so every slot gets the same network, families, classes, + // and reclaim policy. + // + // +kubebuilder:validation:Required + Spec NetworkInterfaceClaimSpec `json:"spec"` +} diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index aceb7595..406fb91b 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -607,6 +607,21 @@ func (in *LocalNetworkContextRef) DeepCopy() *LocalNetworkContextRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalNetworkInterfaceRef) DeepCopyInto(out *LocalNetworkInterfaceRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalNetworkInterfaceRef. +func (in *LocalNetworkInterfaceRef) DeepCopy() *LocalNetworkInterfaceRef { + if in == nil { + return nil + } + out := new(LocalNetworkInterfaceRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalNetworkRef) DeepCopyInto(out *LocalNetworkRef) { *out = *in @@ -1203,6 +1218,342 @@ func (in *NetworkIPAM) DeepCopy() *NetworkIPAM { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterface) DeepCopyInto(out *NetworkInterface) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterface. +func (in *NetworkInterface) DeepCopy() *NetworkInterface { + if in == nil { + return nil + } + out := new(NetworkInterface) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterface) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceAddress) DeepCopyInto(out *NetworkInterfaceAddress) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceAddress. +func (in *NetworkInterfaceAddress) DeepCopy() *NetworkInterfaceAddress { + if in == nil { + return nil + } + out := new(NetworkInterfaceAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceAddressRequest) DeepCopyInto(out *NetworkInterfaceAddressRequest) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceAddressRequest. +func (in *NetworkInterfaceAddressRequest) DeepCopy() *NetworkInterfaceAddressRequest { + if in == nil { + return nil + } + out := new(NetworkInterfaceAddressRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceAttachmentRef) DeepCopyInto(out *NetworkInterfaceAttachmentRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceAttachmentRef. +func (in *NetworkInterfaceAttachmentRef) DeepCopy() *NetworkInterfaceAttachmentRef { + if in == nil { + return nil + } + out := new(NetworkInterfaceAttachmentRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaim) DeepCopyInto(out *NetworkInterfaceClaim) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaim. +func (in *NetworkInterfaceClaim) DeepCopy() *NetworkInterfaceClaim { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterfaceClaim) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimList) DeepCopyInto(out *NetworkInterfaceClaimList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkInterfaceClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimList. +func (in *NetworkInterfaceClaimList) DeepCopy() *NetworkInterfaceClaimList { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterfaceClaimList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimRef) DeepCopyInto(out *NetworkInterfaceClaimRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimRef. +func (in *NetworkInterfaceClaimRef) DeepCopy() *NetworkInterfaceClaimRef { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimSpec) DeepCopyInto(out *NetworkInterfaceClaimSpec) { + *out = *in + out.Network = in.Network + if in.IPFamilies != nil { + in, out := &in.IPFamilies, &out.IPFamilies + *out = make([]IPFamily, len(*in)) + copy(*out, *in) + } + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]NetworkInterfaceAddressRequest, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimSpec. +func (in *NetworkInterfaceClaimSpec) DeepCopy() *NetworkInterfaceClaimSpec { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimStatus) DeepCopyInto(out *NetworkInterfaceClaimStatus) { + *out = *in + if in.NetworkInterfaceRef != nil { + in, out := &in.NetworkInterfaceRef, &out.NetworkInterfaceRef + *out = new(LocalNetworkInterfaceRef) + **out = **in + } + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]NetworkInterfaceAddress, len(*in)) + copy(*out, *in) + } + if in.ExternalAddresses != nil { + in, out := &in.ExternalAddresses, &out.ExternalAddresses + *out = make([]NetworkInterfaceExternalAddress, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimStatus. +func (in *NetworkInterfaceClaimStatus) DeepCopy() *NetworkInterfaceClaimStatus { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimTemplate) DeepCopyInto(out *NetworkInterfaceClaimTemplate) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimTemplate. +func (in *NetworkInterfaceClaimTemplate) DeepCopy() *NetworkInterfaceClaimTemplate { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceExternalAddress) DeepCopyInto(out *NetworkInterfaceExternalAddress) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceExternalAddress. +func (in *NetworkInterfaceExternalAddress) DeepCopy() *NetworkInterfaceExternalAddress { + if in == nil { + return nil + } + out := new(NetworkInterfaceExternalAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceList) DeepCopyInto(out *NetworkInterfaceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkInterface, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceList. +func (in *NetworkInterfaceList) DeepCopy() *NetworkInterfaceList { + if in == nil { + return nil + } + out := new(NetworkInterfaceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterfaceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceSpec) DeepCopyInto(out *NetworkInterfaceSpec) { + *out = *in + out.Network = in.Network + if in.ClaimRef != nil { + in, out := &in.ClaimRef, &out.ClaimRef + *out = new(NetworkInterfaceClaimRef) + **out = **in + } + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]NetworkInterfaceAddress, len(*in)) + copy(*out, *in) + } + if in.ExternalAddresses != nil { + in, out := &in.ExternalAddresses, &out.ExternalAddresses + *out = make([]NetworkInterfaceExternalAddress, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceSpec. +func (in *NetworkInterfaceSpec) DeepCopy() *NetworkInterfaceSpec { + if in == nil { + return nil + } + out := new(NetworkInterfaceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceStatus) DeepCopyInto(out *NetworkInterfaceStatus) { + *out = *in + if in.NetworkContextRef != nil { + in, out := &in.NetworkContextRef, &out.NetworkContextRef + *out = new(LocalNetworkContextRef) + **out = **in + } + if in.AttachmentRef != nil { + in, out := &in.AttachmentRef, &out.AttachmentRef + *out = new(NetworkInterfaceAttachmentRef) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceStatus. +func (in *NetworkInterfaceStatus) DeepCopy() *NetworkInterfaceStatus { + if in == nil { + return nil + } + out := new(NetworkInterfaceStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkList) DeepCopyInto(out *NetworkList) { *out = *in diff --git a/config/crd/bases/networking.datumapis.com_networkinterfaceclaims.yaml b/config/crd/bases/networking.datumapis.com_networkinterfaceclaims.yaml new file mode 100644 index 00000000..0b587210 --- /dev/null +++ b/config/crd/bases/networking.datumapis.com_networkinterfaceclaims.yaml @@ -0,0 +1,437 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: networkinterfaceclaims.networking.datumapis.com +spec: + group: networking.datumapis.com + names: + kind: NetworkInterfaceClaim + listKind: NetworkInterfaceClaimList + plural: networkinterfaceclaims + singular: networkinterfaceclaim + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.network.name + name: Network + type: string + - jsonPath: .status.networkInterfaceRef.name + name: Interface + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1alpha + schema: + openAPIV3Schema: + description: |- + NetworkInterfaceClaim asks for an interface on a network. It is the resource + a user creates. The operator finds or creates a NetworkInterface that + satisfies it, allocates the addresses, and reports them in status. + + A claim describes what the interface must be able to do, never which + interface or address to use. One claim holds at most one interface, and one + interface is held by at most one claim. + + A claim's name is what makes addresses stable. It names the slot in a + workload rather than the instance filling it, so an instance replaced by + another that asks for the same claim name comes back on the same interface + and the same addresses. What happens when the claim itself is deleted is + spec.reclaimPolicy. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NetworkInterfaceClaimSpec defines the desired state of NetworkInterfaceClaim. + Every field states what the interface must be able to do, never which + interface or which address to use. + + Most of the spec is immutable, because the addresses are allocated against + it. To change one of those fields, delete the claim and create a new one, + accepting that the workload gets new addresses unless the interface is + retained. + properties: + addresses: + description: |- + addresses request extra addresses by class, beyond the ones the interface + holds inside its network. Each appears in status.externalAddresses as a + bare address, mapped onto the interface address of the same family. + + Omit this field for ordinary private addressing, which is the common case. + items: + description: |- + NetworkInterfaceAddressRequest asks for one address beyond the ones the + interface holds inside its network, such as a public IPv4 address in front of + a private one. + properties: + class: + description: |- + class is the IPAM class to allocate from, such as public-ipv4. + + A class names a kind of address, and the platform decides which pool and + prefix length serve it. A class never names a pool, a prefix length, or a + CIDR, so a class cannot be used to ask for a particular address. + maxLength: 63 + minLength: 1 + type: string + required: + - class + type: object + maxItems: 4 + minItems: 1 + type: array + x-kubernetes-validations: + - message: Each address class may be requested at most once + rule: self.all(a, self.exists_one(b, b.class == a.class)) + interfaceName: + default: eth0 + description: |- + interfaceName is the device name the interface presents to the guest + operating system, such as eth0 or eth1. Set it when a workload has more + than one interface and the guest configuration names them. + + Immutable, because the guest is configured against it. + maxLength: 15 + minLength: 1 + type: string + x-kubernetes-validations: + - message: interfaceName is immutable and cannot be changed after + creation + rule: self == oldSelf + ipFamilies: + default: + - IPv6 + description: |- + ipFamilies are the address families the interface must carry, in priority + order. List [IPv6, IPv4] for a dual-stack interface. The first family + listed holds the interface's primary address, which is the one reported in + single-address fields such as an instance's network IP. + + Every family listed must be satisfiable or the claim does not bind. Asking + for a family the network does not carry fails the claim outright rather + than leaving it pending, and no partially addressed interface is ever + published. + items: + enum: + - IPv4 + - IPv6 + type: string + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: Each address family may be requested at most once + rule: self.all(f, self.exists_one(g, g == f)) + - message: ipFamilies is immutable and cannot be changed after creation + rule: self == oldSelf + network: + description: |- + network is the network the interface attaches to. The network must already + exist in the same namespace as the claim. + + Immutable. An interface that changed network would hold addresses from a + space it no longer belongs to, so move a workload by recreating the claim + against the other network. + properties: + name: + description: The network name + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: network is immutable and cannot be changed after creation + rule: self == oldSelf + networkInterfaceName: + description: |- + networkInterfaceName binds one specific interface by name, instead of the + interface named after this claim. The named interface must already carry + every family and class this claim asks for, under the same reclaim policy, + and must not be held by another claim. + + Leave it empty, which is the normal case. The claim then binds the + interface of its own name, retained by an earlier claim, or creates one. + + Immutable, including from empty to set. Rebinding a workload to a different + interface means a new claim. + maxLength: 253 + minLength: 1 + type: string + reclaimPolicy: + default: Delete + description: |- + reclaimPolicy decides what becomes of the bound interface, and its + addresses, when this claim is deleted. + + Delete deletes the interface and returns its addresses to IPAM. A workload + recreated later comes back on different addresses. + + Retain keeps the interface, unbound and still holding its addresses, so a + later claim of this name binds it again and the workload returns to the + same addresses. Choose Retain when an address is published in DNS, allowed + through a firewall, or otherwise depended on from outside. + + A retained address is reserved, and billable, for as long as the interface + exists. Deleting the interface does not return it to the pool today, so + choose Retain for addresses worth holding rather than as a default. + + Both policies keep the addresses while the claim exists, including across + instance replacement. They differ only on scale-down and deletion. + + Immutable. An address keeps the policy it was allocated under, and a claim + asking for a policy the interface was not allocated under cannot bind it. + enum: + - Delete + - Retain + type: string + x-kubernetes-validations: + - message: reclaimPolicy is immutable and cannot be changed after + creation + rule: self == oldSelf + required: + - network + type: object + x-kubernetes-validations: + - message: networkInterfaceName is immutable and cannot be set, changed, + or cleared after creation + rule: has(self.networkInterfaceName) == has(oldSelf.networkInterfaceName) + && (!has(self.networkInterfaceName) || self.networkInterfaceName == + oldSelf.networkInterfaceName) + - message: addresses is immutable and cannot be set, changed, or cleared + after creation + rule: has(self.addresses) == has(oldSelf.addresses) && (!has(self.addresses) + || self.addresses == oldSelf.addresses) + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Bound + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Allocated + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Ready + description: |- + NetworkInterfaceClaimStatus defines the observed state of + NetworkInterfaceClaim. It repeats the bound interface's addresses so a + consumer reads one object rather than following the reference. + properties: + addresses: + description: |- + addresses are the addresses the bound interface holds inside its network, + each with its prefix length and, once the location has a subnet, its + gateway. They are copied from the interface, which remains the source of + truth. + items: + description: |- + NetworkInterfaceAddress is an address the interface holds inside its network. + These are the addresses configured on the NIC itself, and they always carry a + prefix length. + properties: + address: + description: |- + address is the address the interface holds, in CIDR notation, such as + 10.128.0.2/32 or 2001:db8:a001::1/128. + + For IPv6 this may be a block delegated to the interface rather than a + single address, such as 2001:db8:a001::/96. The interface owns the whole + block and assigns within it. + maxLength: 45 + minLength: 1 + type: string + class: + description: |- + class is the IPAM class this address was allocated from, such as + private-ipv6. It is empty for the addresses a claim requests by family + rather than by class. + maxLength: 63 + type: string + family: + description: family is the address family of this entry. + enum: + - IPv4 + - IPv6 + type: string + gateway: + description: |- + gateway is the next hop the interface routes through for this family, such + as 10.128.0.1. It is resolved from the subnet backing the network in this + location, so nothing has to read the subnet to configure the NIC. It is + empty until that subnet exists. + maxLength: 45 + type: string + primary: + description: |- + primary marks the address projected into single-address fields, such as an + instance's reported network IP. + + Exactly one address is primary for the interface as a whole, not one per + family. It is the address of the first family the claim listed in + spec.ipFamilies. + type: boolean + required: + - address + - family + type: object + type: array + conditions: + description: |- + conditions report the current state of the claim. Wait on Ready, which is + true once the claim is bound, its addresses are allocated, and the data + plane carries them. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + externalAddresses: + description: |- + externalAddresses are the addresses the bound interface is reachable at from + outside the network, one per class the claim requested. Each is a bare + address with no prefix length. They are copied from the interface. + items: + description: |- + NetworkInterfaceExternalAddress is an address reachable from outside the + network, mapped onto an address the interface holds inside it. A public IPv4 + address in front of a private address is the usual case. + + Unlike an interface address, an external address is a bare address with no + prefix length, such as 203.0.113.10, because nothing configures it on the + NIC. The data plane maps it onto the interface address of the same family. + properties: + address: + description: |- + address is the externally reachable address, such as 203.0.113.10. It + carries no prefix length. + maxLength: 45 + minLength: 1 + type: string + class: + description: |- + class is the IPAM class this address was allocated from, such as + public-ipv4. It matches the class the claim requested in spec.addresses. + maxLength: 63 + minLength: 1 + type: string + family: + description: family is the address family of this entry. + enum: + - IPv4 + - IPv6 + type: string + required: + - address + - class + - family + type: object + type: array + networkInterfaceRef: + description: |- + networkInterfaceRef is the interface bound to this claim, in the same + namespace. Read it to reach fields the claim does not repeat, such as the + MTU and the data-plane attachment. + properties: + name: + description: name is the network interface name. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml new file mode 100644 index 00000000..2fd787b7 --- /dev/null +++ b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml @@ -0,0 +1,385 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: networkinterfaces.networking.datumapis.com +spec: + group: networking.datumapis.com + names: + kind: NetworkInterface + listKind: NetworkInterfaceList + plural: networkinterfaces + singular: networkinterface + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.network.name + name: Network + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .spec.claimRef.name + name: Claim + type: string + - jsonPath: .status.conditions[?(@.type=="Allocated")].status + name: Allocated + type: string + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha + schema: + openAPIV3Schema: + description: |- + NetworkInterface is an interface on a network, together with the addresses it + holds. It is the unit that owns addresses: as long as the interface exists, + its addresses stay allocated to it. + + You do not create a NetworkInterface. Ask for one with a + NetworkInterfaceClaim, and the operator creates the interface, allocates its + addresses, and binds the two. A provider then reads the interface to + configure a NIC, and reports what it programmed in status. + + An interface outlives the instance using it. Whether it outlives the claim + that asked for it depends on spec.reclaimPolicy. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NetworkInterfaceSpec defines the desired state of NetworkInterface. It is + written by the operator when a claim is fulfilled, and it carries everything + a provider needs to configure a NIC without reading any other resource. + properties: + addresses: + description: |- + addresses are the addresses the interface holds inside its network, at most + one per address family, exactly one of them primary. Each carries a prefix + length and, once the location has a subnet, the gateway to route through. + items: + description: |- + NetworkInterfaceAddress is an address the interface holds inside its network. + These are the addresses configured on the NIC itself, and they always carry a + prefix length. + properties: + address: + description: |- + address is the address the interface holds, in CIDR notation, such as + 10.128.0.2/32 or 2001:db8:a001::1/128. + + For IPv6 this may be a block delegated to the interface rather than a + single address, such as 2001:db8:a001::/96. The interface owns the whole + block and assigns within it. + maxLength: 45 + minLength: 1 + type: string + class: + description: |- + class is the IPAM class this address was allocated from, such as + private-ipv6. It is empty for the addresses a claim requests by family + rather than by class. + maxLength: 63 + type: string + family: + description: family is the address family of this entry. + enum: + - IPv4 + - IPv6 + type: string + gateway: + description: |- + gateway is the next hop the interface routes through for this family, such + as 10.128.0.1. It is resolved from the subnet backing the network in this + location, so nothing has to read the subnet to configure the NIC. It is + empty until that subnet exists. + maxLength: 45 + type: string + primary: + description: |- + primary marks the address projected into single-address fields, such as an + instance's reported network IP. + + Exactly one address is primary for the interface as a whole, not one per + family. It is the address of the first family the claim listed in + spec.ipFamilies. + type: boolean + required: + - address + - family + type: object + maxItems: 4 + type: array + x-kubernetes-validations: + - message: Exactly one address must be primary + rule: size(self) == 0 || self.filter(a, has(a.primary) && a.primary).size() + == 1 + - message: Only one address may be held per address family + rule: self.all(a, self.exists_one(b, b.family == a.family)) + claimRef: + description: |- + claimRef is the claim currently holding this interface. It is empty while a + retained interface waits, unbound, for a claim of its name to return. + properties: + name: + description: |- + name is the name of the NetworkInterfaceClaim, in the same namespace as the + interface. A claim name stays with the workload slot it serves, so a + replacement instance binds this same interface and its addresses. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + externalAddresses: + description: |- + externalAddresses are the addresses the interface is reachable at from + outside the network, each mapped onto the interface address of the same + family. They come from the classes the claim requested, and they are absent + for a workload that only needs private addressing. + items: + description: |- + NetworkInterfaceExternalAddress is an address reachable from outside the + network, mapped onto an address the interface holds inside it. A public IPv4 + address in front of a private address is the usual case. + + Unlike an interface address, an external address is a bare address with no + prefix length, such as 203.0.113.10, because nothing configures it on the + NIC. The data plane maps it onto the interface address of the same family. + properties: + address: + description: |- + address is the externally reachable address, such as 203.0.113.10. It + carries no prefix length. + maxLength: 45 + minLength: 1 + type: string + class: + description: |- + class is the IPAM class this address was allocated from, such as + public-ipv4. It matches the class the claim requested in spec.addresses. + maxLength: 63 + minLength: 1 + type: string + family: + description: family is the address family of this entry. + enum: + - IPv4 + - IPv6 + type: string + required: + - address + - class + - family + type: object + maxItems: 4 + type: array + x-kubernetes-validations: + - message: External addresses must be unique + rule: self.all(a, self.exists_one(b, b.address == a.address)) + - message: Only one external address may be held per address class + rule: self.all(a, self.exists_one(b, b.class == a.class)) + interfaceName: + default: eth0 + description: |- + interfaceName is the device name the interface presents to the guest + operating system, such as eth0 or eth1. It comes from the claim. + maxLength: 15 + minLength: 1 + type: string + mtu: + description: |- + mtu is the MTU, in bytes, the interface must be configured with. It is + resolved from the network, so a provider never has to read the network to + configure the NIC. + format: int32 + maximum: 8856 + minimum: 1300 + type: integer + network: + description: |- + network is the network this interface belongs to, in the same namespace as + the interface. It comes from the claim and does not change. + properties: + name: + description: The network name + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: network is immutable and cannot be changed after creation + rule: self == oldSelf + reclaimPolicy: + default: Delete + description: |- + reclaimPolicy decides what becomes of this interface, and its addresses, + when the claim holding it is deleted. It comes from the claim, and a claim + asking for a different policy cannot bind this interface. + enum: + - Delete + - Retain + type: string + required: + - network + type: object + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Allocated + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + NetworkInterfaceStatus defines the observed state of NetworkInterface: which + claim holds it, what realizes it on the data plane, and whether programming + has succeeded. + properties: + attachmentRef: + description: |- + attachmentRef is the data-plane resource realizing this interface. The + provider sets it once an attachment exists. + properties: + apiGroup: + description: |- + apiGroup is the API group of the referent, such as + compute.datumapis.com. + maxLength: 253 + minLength: 1 + type: string + kind: + description: kind is the kind of the referent. + maxLength: 63 + minLength: 1 + type: string + name: + description: name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - apiGroup + - kind + - name + type: object + conditions: + description: |- + conditions report the current state of the interface. Allocated means every + address is held. Programmed means the data plane carries them. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + networkContextRef: + description: |- + networkContextRef is the network's presence in this location, resolved or + created while fulfilling the claim. It is a breadcrumb for operators + tracing where a network landed, and nothing needs it to configure a NIC. + properties: + name: + description: The network context name + type: string + required: + - name + type: object + phase: + description: |- + phase reports whether a claim holds the interface. Bound means the claim in + spec.claimRef holds it. Available means it is retained and holding its + addresses with no claim bound. + enum: + - Available + - Bound + type: string + vpc: + description: |- + vpc is the base62 identifier of the VPC backing this network in this + location, matching the identifier the fabric keys on. The provider records + it when the attachment is programmed. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 1dad8cb9..3b8099b0 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -16,6 +16,8 @@ resources: - bases/networking.datumapis.com_connectors.yaml - bases/networking.datumapis.com_connectoradvertisements.yaml - bases/networking.datumapis.com_connectorclasses.yaml +- bases/networking.datumapis.com_networkinterfaces.yaml +- bases/networking.datumapis.com_networkinterfaceclaims.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/docs/api/networkinterfaceclaims.md b/docs/api/networkinterfaceclaims.md new file mode 100644 index 00000000..c5f1805c --- /dev/null +++ b/docs/api/networkinterfaceclaims.md @@ -0,0 +1,580 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha](#networkingdatumapiscomv1alpha) + +# networking.datumapis.com/v1alpha + +Resource Types: + +- [NetworkInterfaceClaim](#networkinterfaceclaim) + + + + +## NetworkInterfaceClaim +[↩ Parent](#networkingdatumapiscomv1alpha ) + + + + + + +NetworkInterfaceClaim asks for an interface on a network. It is the resource +a user creates. The operator finds or creates a NetworkInterface that +satisfies it, allocates the addresses, and reports them in status. + +A claim describes what the interface must be able to do, never which +interface or address to use. One claim holds at most one interface, and one +interface is held by at most one claim. + +A claim's name is what makes addresses stable. It names the slot in a +workload rather than the instance filling it, so an instance replaced by +another that asks for the same claim name comes back on the same interface +and the same addresses. What happens when the claim itself is deleted is +spec.reclaimPolicy. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    apiVersionstringnetworking.datumapis.com/v1alphatrue
    kindstringNetworkInterfaceClaimtrue
    metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
    specobject + NetworkInterfaceClaimSpec defines the desired state of NetworkInterfaceClaim. +Every field states what the interface must be able to do, never which +interface or which address to use. + +Most of the spec is immutable, because the addresses are allocated against +it. To change one of those fields, delete the claim and create a new one, +accepting that the workload gets new addresses unless the interface is +retained.
    +
    + Validations:
  • has(self.networkInterfaceName) == has(oldSelf.networkInterfaceName) && (!has(self.networkInterfaceName) || self.networkInterfaceName == oldSelf.networkInterfaceName): networkInterfaceName is immutable and cannot be set, changed, or cleared after creation
  • has(self.addresses) == has(oldSelf.addresses) && (!has(self.addresses) || self.addresses == oldSelf.addresses): addresses is immutable and cannot be set, changed, or cleared after creation
    • +     		true
    statusobject + NetworkInterfaceClaimStatus defines the observed state of +NetworkInterfaceClaim. It repeats the bound interface's addresses so a +consumer reads one object rather than following the reference.
    +
    + Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Bound] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Allocated] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Programmed] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Ready]]]
    +     		false
    + + +### NetworkInterfaceClaim.spec +[↩ Parent](#networkinterfaceclaim) + + + +NetworkInterfaceClaimSpec defines the desired state of NetworkInterfaceClaim. +Every field states what the interface must be able to do, never which +interface or which address to use. + +Most of the spec is immutable, because the addresses are allocated against +it. To change one of those fields, delete the claim and create a new one, +accepting that the workload gets new addresses unless the interface is +retained. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    networkobject + network is the network the interface attaches to. The network must already +exist in the same namespace as the claim. + +Immutable. An interface that changed network would hold addresses from a +space it no longer belongs to, so move a workload by recreating the claim +against the other network.
    +
    + Validations:
  • self == oldSelf: network is immutable and cannot be changed after creation
    • +     		true
    addresses[]object + addresses request extra addresses by class, beyond the ones the interface +holds inside its network. Each appears in status.externalAddresses as a +bare address, mapped onto the interface address of the same family. + +Omit this field for ordinary private addressing, which is the common case.
    +
    + Validations:
  • self.all(a, self.exists_one(b, b.class == a.class)): Each address class may be requested at most once
    • +     		false
    interfaceNamestring + interfaceName is the device name the interface presents to the guest +operating system, such as eth0 or eth1. Set it when a workload has more +than one interface and the guest configuration names them. + +Immutable, because the guest is configured against it.
    +
    + Validations:
  • self == oldSelf: interfaceName is immutable and cannot be changed after creation
    • + Default: eth0
    +     		false
    ipFamilies[]enum + ipFamilies are the address families the interface must carry, in priority +order. List [IPv6, IPv4] for a dual-stack interface. The first family +listed holds the interface's primary address, which is the one reported in +single-address fields such as an instance's network IP. + +Every family listed must be satisfiable or the claim does not bind. Asking +for a family the network does not carry fails the claim outright rather +than leaving it pending, and no partially addressed interface is ever +published.
    +
    + Validations:
  • self.all(f, self.exists_one(g, g == f)): Each address family may be requested at most once
  • self == oldSelf: ipFamilies is immutable and cannot be changed after creation
    • + Enum: IPv4, IPv6
    + Default: [IPv6]
    +     		false
    networkInterfaceNamestring + networkInterfaceName binds one specific interface by name, instead of the +interface named after this claim. The named interface must already carry +every family and class this claim asks for, under the same reclaim policy, +and must not be held by another claim. + +Leave it empty, which is the normal case. The claim then binds the +interface of its own name, retained by an earlier claim, or creates one. + +Immutable, including from empty to set. Rebinding a workload to a different +interface means a new claim.
    +     		false
    reclaimPolicyenum + reclaimPolicy decides what becomes of the bound interface, and its +addresses, when this claim is deleted. + +Delete deletes the interface and returns its addresses to IPAM. A workload +recreated later comes back on different addresses. + +Retain keeps the interface, unbound and still holding its addresses, so a +later claim of this name binds it again and the workload returns to the +same addresses. Choose Retain when an address is published in DNS, allowed +through a firewall, or otherwise depended on from outside. + +A retained address is reserved, and billable, for as long as the interface +exists. Deleting the interface does not return it to the pool today, so +choose Retain for addresses worth holding rather than as a default. + +Both policies keep the addresses while the claim exists, including across +instance replacement. They differ only on scale-down and deletion. + +Immutable. An address keeps the policy it was allocated under, and a claim +asking for a policy the interface was not allocated under cannot bind it.
    +
    + Validations:
  • self == oldSelf: reclaimPolicy is immutable and cannot be changed after creation
    • + Enum: Delete, Retain
    + Default: Delete
    +     		false
    + + +### NetworkInterfaceClaim.spec.network +[↩ Parent](#networkinterfaceclaimspec) + + + +network is the network the interface attaches to. The network must already +exist in the same namespace as the claim. + +Immutable. An interface that changed network would hold addresses from a +space it no longer belongs to, so move a workload by recreating the claim +against the other network. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + The network name
    +     		true
    + + +### NetworkInterfaceClaim.spec.addresses[index] +[↩ Parent](#networkinterfaceclaimspec) + + + +NetworkInterfaceAddressRequest asks for one address beyond the ones the +interface holds inside its network, such as a public IPv4 address in front of +a private one. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    classstring + class is the IPAM class to allocate from, such as public-ipv4. + +A class names a kind of address, and the platform decides which pool and +prefix length serve it. A class never names a pool, a prefix length, or a +CIDR, so a class cannot be used to ask for a particular address.
    +     		true
    + + +### NetworkInterfaceClaim.status +[↩ Parent](#networkinterfaceclaim) + + + +NetworkInterfaceClaimStatus defines the observed state of +NetworkInterfaceClaim. It repeats the bound interface's addresses so a +consumer reads one object rather than following the reference. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addresses[]object + addresses are the addresses the bound interface holds inside its network, +each with its prefix length and, once the location has a subnet, its +gateway. They are copied from the interface, which remains the source of +truth.
    +     		false
    conditions[]object + conditions report the current state of the claim. Wait on Ready, which is +true once the claim is bound, its addresses are allocated, and the data +plane carries them.
    +     		false
    externalAddresses[]object + externalAddresses are the addresses the bound interface is reachable at from +outside the network, one per class the claim requested. Each is a bare +address with no prefix length. They are copied from the interface.
    +     		false
    networkInterfaceRefobject + networkInterfaceRef is the interface bound to this claim, in the same +namespace. Read it to reach fields the claim does not repeat, such as the +MTU and the data-plane attachment.
    +     		false
    + + +### NetworkInterfaceClaim.status.addresses[index] +[↩ Parent](#networkinterfaceclaimstatus) + + + +NetworkInterfaceAddress is an address the interface holds inside its network. +These are the addresses configured on the NIC itself, and they always carry a +prefix length. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + address is the address the interface holds, in CIDR notation, such as +10.128.0.2/32 or 2001:db8:a001::1/128. + +For IPv6 this may be a block delegated to the interface rather than a +single address, such as 2001:db8:a001::/96. The interface owns the whole +block and assigns within it.
    +     		true
    familyenum + family is the address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    classstring + class is the IPAM class this address was allocated from, such as +private-ipv6. It is empty for the addresses a claim requests by family +rather than by class.
    +     		false
    gatewaystring + gateway is the next hop the interface routes through for this family, such +as 10.128.0.1. It is resolved from the subnet backing the network in this +location, so nothing has to read the subnet to configure the NIC. It is +empty until that subnet exists.
    +     		false
    primaryboolean + primary marks the address projected into single-address fields, such as an +instance's reported network IP. + +Exactly one address is primary for the interface as a whole, not one per +family. It is the address of the first family the claim listed in +spec.ipFamilies.
    +     		false
    + + +### NetworkInterfaceClaim.status.conditions[index] +[↩ Parent](#networkinterfaceclaimstatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### NetworkInterfaceClaim.status.externalAddresses[index] +[↩ Parent](#networkinterfaceclaimstatus) + + + +NetworkInterfaceExternalAddress is an address reachable from outside the +network, mapped onto an address the interface holds inside it. A public IPv4 +address in front of a private address is the usual case. + +Unlike an interface address, an external address is a bare address with no +prefix length, such as 203.0.113.10, because nothing configures it on the +NIC. The data plane maps it onto the interface address of the same family. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + address is the externally reachable address, such as 203.0.113.10. It +carries no prefix length.
    +     		true
    classstring + class is the IPAM class this address was allocated from, such as +public-ipv4. It matches the class the claim requested in spec.addresses.
    +     		true
    familyenum + family is the address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    + + +### NetworkInterfaceClaim.status.networkInterfaceRef +[↩ Parent](#networkinterfaceclaimstatus) + + + +networkInterfaceRef is the interface bound to this claim, in the same +namespace. Read it to reach fields the claim does not repeat, such as the +MTU and the data-plane attachment. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + name is the network interface name.
    +     		true
    diff --git a/docs/api/networkinterfaces.md b/docs/api/networkinterfaces.md new file mode 100644 index 00000000..e67f22bf --- /dev/null +++ b/docs/api/networkinterfaces.md @@ -0,0 +1,580 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha](#networkingdatumapiscomv1alpha) + +# networking.datumapis.com/v1alpha + +Resource Types: + +- [NetworkInterface](#networkinterface) + + + + +## NetworkInterface +[↩ Parent](#networkingdatumapiscomv1alpha ) + + + + + + +NetworkInterface is an interface on a network, together with the addresses it +holds. It is the unit that owns addresses: as long as the interface exists, +its addresses stay allocated to it. + +You do not create a NetworkInterface. Ask for one with a +NetworkInterfaceClaim, and the operator creates the interface, allocates its +addresses, and binds the two. A provider then reads the interface to +configure a NIC, and reports what it programmed in status. + +An interface outlives the instance using it. Whether it outlives the claim +that asked for it depends on spec.reclaimPolicy. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    apiVersionstringnetworking.datumapis.com/v1alphatrue
    kindstringNetworkInterfacetrue
    metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
    specobject + NetworkInterfaceSpec defines the desired state of NetworkInterface. It is +written by the operator when a claim is fulfilled, and it carries everything +a provider needs to configure a NIC without reading any other resource.
    +     		true
    statusobject + NetworkInterfaceStatus defines the observed state of NetworkInterface: which +claim holds it, what realizes it on the data plane, and whether programming +has succeeded.
    +
    + Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Allocated] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Programmed]]]
    +     		false
    + + +### NetworkInterface.spec +[↩ Parent](#networkinterface) + + + +NetworkInterfaceSpec defines the desired state of NetworkInterface. It is +written by the operator when a claim is fulfilled, and it carries everything +a provider needs to configure a NIC without reading any other resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    networkobject + network is the network this interface belongs to, in the same namespace as +the interface. It comes from the claim and does not change.
    +
    + Validations:
  • self == oldSelf: network is immutable and cannot be changed after creation
    • +     		true
    addresses[]object + addresses are the addresses the interface holds inside its network, at most +one per address family, exactly one of them primary. Each carries a prefix +length and, once the location has a subnet, the gateway to route through.
    +
    + Validations:
  • size(self) == 0 || self.filter(a, has(a.primary) && a.primary).size() == 1: Exactly one address must be primary
  • self.all(a, self.exists_one(b, b.family == a.family)): Only one address may be held per address family
    • +     		false
    claimRefobject + claimRef is the claim currently holding this interface. It is empty while a +retained interface waits, unbound, for a claim of its name to return.
    +     		false
    externalAddresses[]object + externalAddresses are the addresses the interface is reachable at from +outside the network, each mapped onto the interface address of the same +family. They come from the classes the claim requested, and they are absent +for a workload that only needs private addressing.
    +
    + Validations:
  • self.all(a, self.exists_one(b, b.address == a.address)): External addresses must be unique
  • self.all(a, self.exists_one(b, b.class == a.class)): Only one external address may be held per address class
    • +     		false
    interfaceNamestring + interfaceName is the device name the interface presents to the guest +operating system, such as eth0 or eth1. It comes from the claim.
    +
    + Default: eth0
    +     		false
    mtuinteger + mtu is the MTU, in bytes, the interface must be configured with. It is +resolved from the network, so a provider never has to read the network to +configure the NIC.
    +
    + Format: int32
    + Minimum: 1300
    + Maximum: 8856
    +     		false
    reclaimPolicyenum + reclaimPolicy decides what becomes of this interface, and its addresses, +when the claim holding it is deleted. It comes from the claim, and a claim +asking for a different policy cannot bind this interface.
    +
    + Enum: Delete, Retain
    + Default: Delete
    +     		false
    + + +### NetworkInterface.spec.network +[↩ Parent](#networkinterfacespec) + + + +network is the network this interface belongs to, in the same namespace as +the interface. It comes from the claim and does not change. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + The network name
    +     		true
    + + +### NetworkInterface.spec.addresses[index] +[↩ Parent](#networkinterfacespec) + + + +NetworkInterfaceAddress is an address the interface holds inside its network. +These are the addresses configured on the NIC itself, and they always carry a +prefix length. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + address is the address the interface holds, in CIDR notation, such as +10.128.0.2/32 or 2001:db8:a001::1/128. + +For IPv6 this may be a block delegated to the interface rather than a +single address, such as 2001:db8:a001::/96. The interface owns the whole +block and assigns within it.
    +     		true
    familyenum + family is the address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    classstring + class is the IPAM class this address was allocated from, such as +private-ipv6. It is empty for the addresses a claim requests by family +rather than by class.
    +     		false
    gatewaystring + gateway is the next hop the interface routes through for this family, such +as 10.128.0.1. It is resolved from the subnet backing the network in this +location, so nothing has to read the subnet to configure the NIC. It is +empty until that subnet exists.
    +     		false
    primaryboolean + primary marks the address projected into single-address fields, such as an +instance's reported network IP. + +Exactly one address is primary for the interface as a whole, not one per +family. It is the address of the first family the claim listed in +spec.ipFamilies.
    +     		false
    + + +### NetworkInterface.spec.claimRef +[↩ Parent](#networkinterfacespec) + + + +claimRef is the claim currently holding this interface. It is empty while a +retained interface waits, unbound, for a claim of its name to return. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + name is the name of the NetworkInterfaceClaim, in the same namespace as the +interface. A claim name stays with the workload slot it serves, so a +replacement instance binds this same interface and its addresses.
    +     		true
    + + +### NetworkInterface.spec.externalAddresses[index] +[↩ Parent](#networkinterfacespec) + + + +NetworkInterfaceExternalAddress is an address reachable from outside the +network, mapped onto an address the interface holds inside it. A public IPv4 +address in front of a private address is the usual case. + +Unlike an interface address, an external address is a bare address with no +prefix length, such as 203.0.113.10, because nothing configures it on the +NIC. The data plane maps it onto the interface address of the same family. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + address is the externally reachable address, such as 203.0.113.10. It +carries no prefix length.
    +     		true
    classstring + class is the IPAM class this address was allocated from, such as +public-ipv4. It matches the class the claim requested in spec.addresses.
    +     		true
    familyenum + family is the address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    + + +### NetworkInterface.status +[↩ Parent](#networkinterface) + + + +NetworkInterfaceStatus defines the observed state of NetworkInterface: which +claim holds it, what realizes it on the data plane, and whether programming +has succeeded. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    attachmentRefobject + attachmentRef is the data-plane resource realizing this interface. The +provider sets it once an attachment exists.
    +     		false
    conditions[]object + conditions report the current state of the interface. Allocated means every +address is held. Programmed means the data plane carries them.
    +     		false
    networkContextRefobject + networkContextRef is the network's presence in this location, resolved or +created while fulfilling the claim. It is a breadcrumb for operators +tracing where a network landed, and nothing needs it to configure a NIC.
    +     		false
    phaseenum + phase reports whether a claim holds the interface. Bound means the claim in +spec.claimRef holds it. Available means it is retained and holding its +addresses with no claim bound.
    +
    + Enum: Available, Bound
    +     		false
    vpcstring + vpc is the base62 identifier of the VPC backing this network in this +location, matching the identifier the fabric keys on. The provider records +it when the attachment is programmed.
    +     		false
    + + +### NetworkInterface.status.attachmentRef +[↩ Parent](#networkinterfacestatus) + + + +attachmentRef is the data-plane resource realizing this interface. The +provider sets it once an attachment exists. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    apiGroupstring + apiGroup is the API group of the referent, such as +compute.datumapis.com.
    +     		true
    kindstring + kind is the kind of the referent.
    +     		true
    namestring + name is the name of the referent.
    +     		true
    + + +### NetworkInterface.status.conditions[index] +[↩ Parent](#networkinterfacestatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### NetworkInterface.status.networkContextRef +[↩ Parent](#networkinterfacestatus) + + + +networkContextRef is the network's presence in this location, resolved or +created while fulfilling the claim. It is a breadcrumb for operators +tracing where a network landed, and nothing needs it to configure a NIC. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + The network context name
    +     		true
    diff --git a/docs/enhancements/network-interfaces.md b/docs/enhancements/network-interfaces.md new file mode 100644 index 00000000..91a7fc83 --- /dev/null +++ b/docs/enhancements/network-interfaces.md @@ -0,0 +1,756 @@ +--- +status: provisional +stage: alpha +latest-milestone: "v0.x" +--- + +# A network interface a workload can be handed + +- [Summary](#summary) +- [Motivation](#motivation) + - [Goals](#goals) + - [Non-Goals](#non-goals) +- [Proposal](#proposal) + - [What it feels like](#what-it-feels-like) + - [Notes/Constraints/Caveats](#notesconstraintscaveats) +- [Design Details](#design-details) + - [NetworkInterfaceClaim](#networkinterfaceclaim) + - [NetworkInterface](#networkinterface) + - [Binding](#binding) + - [Fulfilling a claim](#fulfilling-a-claim) + - [Reaching the data plane](#reaching-the-data-plane) + - [What compute writes](#what-compute-writes) + - [A workload in two locations](#a-workload-in-two-locations) +- [What this depends on](#what-this-depends-on) +- [Drawbacks](#drawbacks) +- [Alternatives](#alternatives) +- [Open Questions](#open-questions) +- [References](#references) + +## Summary + +Compute asks for a network interface the way a pod asks for storage: it writes a +**`NetworkInterfaceClaim`** naming a network and what the interface needs, and NSO binds it +to a **`NetworkInterface`** carrying everything required to configure a NIC — addresses, +gateway, MTU, and the data-plane identity behind them. + +Two properties follow from that shape. **Compute stops reaching into networking +internals**: it no longer creates `NetworkBinding` and `SubnetClaim` objects, and no longer +needs to know that a `NetworkContext` exists. And **an interface becomes a thing that +outlives the instance using it**, which is what makes a per-instance address, a retained +address, and an address a consumer can actually see all possible at once. + +This document is written for the compute service, for infrastructure providers that +configure NICs, and for NSO, which fulfills claims. It defines the shape of the two +resources and the contract between the three. + +## Motivation + +The interface between compute and networking today is not an interface. It is compute +reaching through NSO's internals, and three problems come out of that. + +**An address is shared where it should be per-instance.** `WorkloadDeploymentReconciler` +creates one `SubnetClaim` per deployment and every instance in that deployment draws from +it. The code says so in a `TODO`. Instances in a deployment cannot have distinct addresses +because nothing per-instance exists to hold one. + +**Nobody can see their address.** `Instance.status.networkInterfaces[].assignments.networkIP` +is declared, printed as a column, and never written. A consumer running +`kubectl get instance` sees an empty field where the answer should be. + +**Every provider re-derives the same facts.** A provider configuring a NIC needs the +address, the gateway, the MTU, and which network it belongs to. Today it walks +`NetworkBinding` → `NetworkContext` → `SubnetClaim` → `Subnet` to assemble them, which +means NSO cannot change those objects without breaking every provider, and a new provider +must learn all four before it can bring up one interface. + +All three are the same missing noun. There is no object that means *this instance's +interface on this network in this location*, so the facts about it live scattered across +the objects that happen to produce them. + +[Compute PR #210](https://github.com/datum-cloud/compute/pull/210) settles where an address +comes from. This document settles what holds it. + +### Goals + +- Give compute one resource to create and one resource to read, with no knowledge of how + NSO satisfies it. +- Give every instance its own interface, and every interface its own addresses. +- Give infrastructure providers a single resource carrying everything needed to configure + a NIC. +- Let an interface — and therefore its addresses — survive the instance that used it, so a + replacement instance comes back to the same address. +- Report allocation and programming separately, so an instance is never told its network + is ready before it can carry traffic. +- Let NSO change how it allocates, and what it allocates from, without a compute release. + +### Non-Goals + +- **Choosing addresses.** Which address an interface gets, out of what space, under what + policy, is [compute PR #210](https://github.com/datum-cloud/compute/pull/210). This + document consumes that answer and never re-decides it. +- **Programming the data plane.** A `NetworkInterface` states what an interface must be; + `VPCAttachment` and the agents behind it make it so. +- **Retiring `NetworkBinding`, `NetworkContext`, `SubnetClaim`, or `Subnet`.** They remain + NSO's internals and keep doing what they do. What changes is that nothing outside NSO + writes them. +- **Interface-level network policy.** `InstanceNetworkInterface.networkPolicy` stays where + it is, on the compute side, and is not part of this contract. +- **Multiple interfaces per instance beyond the first.** The model is a list and carries + more than one without changes; whether compute exposes that at launch is a compute + product decision, not an API constraint here. +- **Interfaces for anything other than an instance.** Load balancers and gateways also + attach to networks, and the same pair of resources should serve them. Nothing here + assumes an instance, but nothing here is designed against a second consumer either. +- **Hot-attach and hot-detach.** Adding or removing an interface on a running instance is + additive to this model and out of scope. + +## Proposal + +Compute creates a claim. NSO binds an interface to it. Providers read the interface. +Nobody reads anything else. + +### What it feels like + +A consumer writes the same workload compute PR #210 describes — a network, a list of +families, and optionally a class of extra address: + +```yaml +apiVersion: compute.datumapis.com/v1alpha +kind: Workload +metadata: + name: hello-sandbox +spec: + template: + spec: + runtime: + sandbox: + containers: + - name: app + image: ghcr.io/datum-cloud/hello-unikraft:latest + networkInterfaces: + - name: eth0 + network: + name: default + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: public-unicast-ipv4 + placements: + - name: default + locations: + - us-central-1 + scaleSettings: + minReplicas: 2 +``` + +Compute turns that into one claim per instance, per interface, in the cell the deployment +landed at: + +```yaml +apiVersion: networking.datumapis.com/v1alpha +kind: NetworkInterfaceClaim +metadata: + # Derived from the slot and the interface. Stable across every instance that + # ever fills this slot. + name: hello-sandbox-default-us-central-1-0-eth0 + ownerReferences: + - kind: Instance + name: hello-sandbox-default-us-central-1-0 +spec: + network: + name: default + interfaceName: eth0 + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: public-unicast-ipv4 +``` + +NSO binds it, and the claim reports the answer: + +```console +$ kubectl get networkinterfaceclaim hello-sandbox-default-us-central-1-0-eth0 -o yaml +status: + networkInterfaceRef: + name: nic-4f2a9c1e + addresses: + - family: IPv6 address: fd20:a1b:2c3d:1:0:1::/96 primary: true + - family: IPv4 address: 10.128.0.2/32 + externalAddresses: + - family: IPv4 address: 198.51.100.11 class: public-unicast-ipv4 + conditions: + - type: Bound status: "True" + - type: Allocated status: "True" + - type: Programmed status: "True" + - type: Ready status: "True" +``` + +The addresses appear on the claim as well as on the interface, deliberately. Compute +watches one object per interface and never needs a second read to answer "what address did +this instance get" — the same reason a `PersistentVolumeClaim` reports its own capacity. + +A provider bringing up the NIC reads the interface, and reads nothing else: + +```console +$ kubectl get networkinterface nic-4f2a9c1e -o yaml +spec: + network: + name: default + claimRef: + name: hello-sandbox-default-us-central-1-0-eth0 + uid: 8c1d… + interfaceName: eth0 + mtu: 1460 + reclaimPolicy: Retain + addresses: + - family: IPv6 address: fd20:a1b:2c3d:1:0:1::/96 gateway: fd20:a1b:2c3d:1::1 primary: true + - family: IPv4 address: 10.128.0.2/32 gateway: 10.128.0.1 + externalAddresses: + - family: IPv4 address: 198.51.100.11 class: public-unicast-ipv4 +status: + phase: Bound + networkContextRef: + name: default-us-central-1 + attachmentRef: + apiGroup: cloud.datumapis.com + kind: VPCAttachment + name: nic-4f2a9c1e + vpc: 3kF9qP2x + conditions: + - type: Allocated status: "True" + - type: Programmed status: "True" +``` + +Everything a NIC needs is in `spec`. Nothing in `spec` requires a second lookup to +interpret. The `NetworkContext` a provider used to have to find appears in `status` as a +breadcrumb for operators — nothing reads it to do its job. + +### Notes/Constraints/Caveats + +- **The claim is the durable identity, not the instance.** Its name derives from the slot, + so a replacement instance finds a claim that already holds its addresses. +- **A claim binds exactly one interface, and an interface binds exactly one claim.** There + is no fan-out and no re-matching. +- **Consumers never write either resource.** Compute writes claims on their behalf; NSO + writes interfaces. +- **`Allocated` and `Programmed` are separate, and `Ready` requires both.** Allocation is + synchronous, programming is not, and an instance released on allocation alone comes up + before its packets can move. +- **Both resources live in the cell the deployment landed at.** Location is implicit in + where the claim exists; nobody writes it down. +- **The addresses on a claim's status are a copy, and the interface is the source of + truth.** They are written in the same reconcile that sets `Bound`. +- **A retained interface returns to `Available`, not to a dead end.** Compute PR #210 is + explicit that the storage `Released` state — where an operator must clear a stale + reference by hand — must not be copied, and it is not. + +## Design Details + +### NetworkInterfaceClaim + +A claim states what an interface must be able to do. Every field is intent; none of it +describes a result. + +```yaml +apiVersion: networking.datumapis.com/v1alpha +kind: NetworkInterfaceClaim +spec: + # The network this interface attaches to. A LocalNetworkRef — a claim and its + # network are always in the same namespace. + # Required. Immutable: an interface that changed network after allocation + # holds addresses from a space it no longer belongs to. + network: + name: default + + # The name the interface presents to the guest. Defaults to eth0. Immutable. + interfaceName: eth0 + + # The address families this interface must carry, in priority order. The first + # is the interface's primary address. Defaults to [IPv6] — the platform is + # IPv6-first. + # + # All requested families must be satisfiable, or the claim does not bind. A + # partially-addressed interface is not published. A family the network does + # not carry is a validation failure, not a pending condition. + ipFamilies: + - IPv6 + - IPv4 + + # Additional addresses beyond the network-internal ones, each named by class. + # A class is an IPAM concept: the consumer names a kind of address, never a + # pool, a prefix length, or a CIDR. Omitted entirely for ordinary private + # addressing, which is the common case. + addresses: + - class: public-unicast-ipv4 + + # What becomes of the bound interface when this claim is deleted. + # Delete — the interface is deleted and its addresses released. + # Retain — the interface survives, unbound, still holding its addresses, + # waiting for a claim of this name to come back. + # Defaults to Delete. The IP class may set a different default; this overrides + # it. + reclaimPolicy: Retain + + # Optional. Binds a specific existing interface by name, the part + # PersistentVolumeClaim.volumeName plays for storage. Left empty — the normal + # case — NSO chooses or creates one. + networkInterfaceName: "" +``` + +Status carries the binding, a copy of the result, and the conditions: + +```yaml +status: + # Set once, when the claim binds. Never recomputed. + networkInterfaceRef: + name: nic-4f2a9c1e + + # Copied from the bound interface so a consumer reads one object. + addresses: [...] + externalAddresses: [...] + + conditions: + # An interface has been bound to this claim. + - type: Bound + # Every requested family holds an address. + - type: Allocated + # The data plane can carry those addresses. + - type: Programmed + # Bound, Allocated, and Programmed are all true. + - type: Ready +``` + +`Allocated` and `Programmed` are surfaced on the claim rather than left on the interface +because they are the two facts compute gates an instance on, and compute should not have to +hold a second watch to learn them. `SubnetClaim` already defaults exactly this trio of +conditions, so the pattern is NSO's own. + +### NetworkInterface + +An interface is a result. It is written by NSO, read by providers, and never authored by +hand outside of an operator repairing something. + +```yaml +apiVersion: networking.datumapis.com/v1alpha +kind: NetworkInterface +spec: + # The network this interface belongs to. + network: + name: default + + # The claim currently holding this interface. Empty when the interface is + # retained and unbound. The uid is recorded so a claim deleted and recreated + # under the same name is recognised as a different claim. + claimRef: + name: hello-sandbox-default-us-central-1-0-eth0 + uid: 8c1d… + + interfaceName: eth0 + + # Resolved from Network.spec.mtu, so a provider never reads the network. + mtu: 1460 + + # The addresses inside the network. One entry per family, exactly one primary. + # For IPv6 this is the endpoint's whole /96 block, not a single address; the + # interface owns the block and assigns within it. + addresses: + - family: IPv6 + address: fd20:a1b:2c3d:1:0:1::/96 + gateway: fd20:a1b:2c3d:1::1 + primary: true + class: tenant-endpoint-ipv6 + - family: IPv4 + address: 10.128.0.2/32 + gateway: 10.128.0.1 + class: tenant-endpoint-ipv4 + + # Addresses reachable from outside the network, each mapped one-to-one onto + # the interface's address of the same family. + externalAddresses: + - family: IPv4 + address: 198.51.100.11 + class: public-unicast-ipv4 + + reclaimPolicy: Retain + +status: + # Available — allocated, holding addresses, bound to nothing. + # Bound — held by the claim in spec.claimRef. + phase: Bound + + # The network's presence in this location, which NSO resolved or created while + # fulfilling the claim. Recorded for operators; nothing depends on it. + networkContextRef: + name: default-us-central-1 + + # The data-plane realization, once one exists. + attachmentRef: + apiGroup: cloud.datumapis.com + kind: VPCAttachment + name: nic-4f2a9c1e + + # Base62 VPC identifier, matching VPC.status.vpc. The fabric keys on this. + vpc: 3kF9qP2x + + conditions: + - type: Allocated + - type: Programmed +``` + +**There is no `Released` phase.** A retained interface whose claim is deleted goes straight +back to `Available` with its addresses intact, and the next claim of the same name binds it. +Storage's `Released` state requires an operator to clear a stale reference before the volume +can be used again; an address held in a finite public range cannot wait on that. The `uid` +in `claimRef` is what makes this safe — it is the record that distinguishes the claim that +held the interface from the one asking for it now. + +**Addresses live in `spec`, not `status`.** They are the desired configuration of a NIC, +and a provider that has to read `status` to know what to configure has no way to tell a +requested address from an observed one. `status` holds only what NSO observed: where the +network is, what realizes the interface, and whether programming succeeded. + +### Binding + +Binding follows storage, because storage's model is the one that survives a holder being +replaced. + +**A claim binds once, at creation.** NSO either finds a retained interface whose name the +claim asks for, or allocates a new one. From then on the claim records the interface and +the interface records the claim, and nothing recomputes the pairing. There is no matching +pass, no window where an address is loose, and no way for a re-match to go wrong. + +**The claim's name is the durable identity.** Compute derives it from the slot — workload, +placement, location, ordinal — and the interface name, all of which are stable across every +instance that ever fills that slot. A replacement instance creates a claim, finds the name +already exists, and adopts it. + +**A claim ends when its slot does.** Deleting the workload deletes its claims through +ownership; scaling down deletes the claims of the slots it removes. `reclaimPolicy` then +decides what happens to the interface: + +| Event | `Delete` | `Retain` | +|---|---|---| +| Instance replaced or rescheduled | claim survives — same interface, same addresses | same | +| Instance redeployed with a new template | claim survives — same interface, same addresses | same | +| Scale down then back up | new interface, new addresses | same interface, same addresses | +| Workload deleted then recreated | new interface, new addresses | same interface, same addresses | + +The first two rows are the same in both columns and that is the point: **an interface +survives a redeploy on its own. Surviving a scale-down takes `Retain`.** + +A retained interface still holds its addresses, still counts against its holder's budget, +and carries a lease — otherwise a public address sits out of service indefinitely with +nothing pressuring anyone to release it. Expiry, its duration, and operator force-release +are IPAM's to define; this resource only records the policy that triggers them. + +### Fulfilling a claim + +What NSO does with a claim is the part compute stops doing. + +**Resolve the network's presence in this location.** The claim names a network and exists +in a cell, which is enough to find or create the `NetworkContext` for that pair. Compute +creates no `NetworkBinding` — that object is now NSO's business, created on the claim's +behalf and reported back only as a breadcrumb in `status.networkContextRef`. + +**Allocate an address per requested family.** Each becomes an `IPClaim` of the appropriate +class against the platform allocator, with the network and location supplied as scope, per +[compute PR #210](https://github.com/datum-cloud/compute/pull/210). Each entry in the +claim's `addresses` list becomes one more `IPClaim` of the class it names. Every one must +succeed before anything is published. + +**Create the interface and bind it.** The addresses, the gateways read from the location's +subnet, and the MTU read from the network land in `spec`. `Allocated` goes true, the claim +goes `Bound`, and compute can see an address. + +**Wait for the data plane.** `Programmed` follows separately, when the attachment realizing +the interface reports ready. + +The ordering matters for the failure case: an exhausted pool, a location with no space, or +a family the network does not carry all fail before an interface exists, so a claim that +cannot be satisfied reports why rather than binding to something incomplete. The condition +message should name what ran out and at which level, which is the property PR #210 asks +the allocator to preserve. + +### Reaching the data plane + +A `NetworkInterface` says what an interface must be. `VPCAttachment`, in the +[cloud](https://github.com/datum-cloud/cloud) API group, is where it becomes real, and the +split is deliberate: an interface is allocated as soon as a claim exists, before an instance +has been scheduled to any node, while an attachment cannot exist until a node, a container, +and a veth pair do. + +``` +NetworkInterfaceClaim compute's intent created per instance, per interface + │ binds + ▼ +NetworkInterface NSO's answer addresses, gateway, MTU + │ realized by + ▼ +VPCAttachment the node's reality node, containerID, VRF, veth, pod subnet + │ attaches to + ▼ +VPC the data plane base62 identity the fabric keys on +``` + +The agent on the node creates the `VPCAttachment` from the interface, copying +`spec.addresses` into `spec.interface.addresses` and naming the VPC backing this network in +this location. It reports back the facts only a node knows — the container ID, the host and +VRF device names, the pod subnet — and NSO sets `Programmed` on the interface when the +attachment reports ready, copying the VPC identifier onto `status.vpc`. + +Nothing in `VPCAttachment` changes to support this. It already requires the addresses to +have been decided elsewhere; this names the elsewhere. + +`Programmed` going false — an attachment lost, a node drained — does not release the +interface. The addresses stay allocated because the claim still exists, and the instance's +`Ready` condition reflects the loss without renumbering anything. + +### What compute writes + +Compute's changes are additive on the consumer-facing side and a removal on the internal +side. `InstanceNetworkInterface.network` keeps its existing `NetworkRef` type — the +consumer-facing reference does not move. + +**On `InstanceNetworkInterface`**, four fields join it: + +| Field | Meaning | Default | +|---|---|---| +| `name` | the interface name in the guest, and the claim-name suffix | `eth0` | +| `ipFamilies` | families to carry, in priority order | `[IPv6]` | +| `reclaimPolicy` | whether addresses survive a scale-down | `Delete` | +| `addresses[].class` | extra addresses by class | none | + +`ipFamilies` and `reclaimPolicy` are the two fields +[compute #112](https://github.com/datum-cloud/compute/issues/112) already calls for. +`addresses[].class` is compute PR #210's. `name` is new here, and exists because a claim +name has to be derived from something stable that a consumer chose. + +**On `InstanceNetworkInterfaceStatus`**, the single-address shape grows into the list the +interface actually holds: + +```yaml +status: + networkInterfaces: + - name: eth0 + addresses: + - family: IPv6 address: fd20:a1b:2c3d:1:0:1::/96 primary: true + - family: IPv4 address: 10.128.0.2/32 + external: + - family: IPv4 address: 198.51.100.11 + assignments: + # Retained: the primary address of each family, so existing print + # columns and clients keep working. + networkIP: fd20:a1b:2c3d:1:0:1:: + externalIP: 198.51.100.11 + conditions: + - type: Allocated status: "True" + - type: Programmed status: "True" +``` + +`assignments` keeps its shape and finally gets written. It becomes a projection of the +primary address rather than a field of its own, which means the print column on `Instance` +starts showing a value without anyone changing the column. + +**In `WorkloadDeploymentReconciler`**, the per-deployment `NetworkBinding` and `SubnetClaim` +are replaced by one `NetworkInterfaceClaim` per instance per interface. That is the fix for +the shared-allocation problem: the object that holds an address is now per-instance, so +addresses can be too. + +**The `network` scheduling gate becomes per-instance.** Today it is removed when a shared +allocation is ready, which releases every instance in the deployment at once. It should be +removed for an instance when that instance's own claims report `Ready` — meaning bound, +allocated, and programmed. An instance whose interface is allocated but not yet programmed +stays gated, which is the whole reason the two conditions are separate. + +Nothing changes in the federation path. Claims and interfaces are created in the POP cell +by NSO, which already runs there for exactly this reason. Addresses land on the local +`Instance`, and the existing write-back to Karmada and `InstanceProjector` mirror carry them +to the project. The consumer reads addresses in the same place they read everything else +about an instance. + +### A workload in two locations + +The same workload from compute PR #210 — one network, two locations, two replicas each, +dual-stack, a public address per instance. Every object this design causes to exist: + +**The consumer's project.** Two objects, both written by the consumer: the `Network` and +the `Workload`. Nothing about interfaces appears here at all. + +**Each POP cell.** The deployment arrives by placement; everything below it is created +locally. + +``` +us-central-1 eu-west-1 + WorkloadDeployment/…-americas WorkloadDeployment/…-europe + Instance/…-americas-us-central-1-{0,1} Instance/…-europe-eu-west-1-{0,1} + NetworkInterfaceClaim/…-{0,1}-eth0 NetworkInterfaceClaim/…-{0,1}-eth0 + NetworkInterface ×2 NetworkInterface ×2 + NetworkContext/default-us-central-1 NetworkContext/default-eu-west-1 + VPCAttachment ×2 VPCAttachment ×2 +``` + +Four claims, four interfaces, four attachments, and one `NetworkContext` per location that +the first claim in that location caused to exist. Each interface holds three addresses — an +IPv6 endpoint block, an IPv4 address, and a public IPv4 address — for the twelve +allocations compute PR #210 accounts for. + +**What each layer knows.** The claim knows the network, the interface name, the families, +and the policy. The interface adds the addresses, the gateways, and the MTU. The attachment +adds the node, the container, and the devices. No layer repeats the one below it, and no +consumer of a layer needs the one above it. + +**Scaling to one replica in `us-central-1`** deletes one instance, and with it one claim. +With `reclaimPolicy: Retain`, the interface stays, holding `fd20:a1b:2c3d:1:0:2::/96`, +`10.128.0.3/32`, and `198.51.100.12`, in phase `Available`. Scaling back up recreates a +claim with the identical name, which binds that interface, and the new instance comes back +on the address the old one had. With `Delete`, all three addresses go back to their pools +and the new instance gets whatever is next. + +Removing the `eu-west-1` placement releases the addresses its instances held, but not that +location's `NetworkContext` or subnet. Those belong to the network, and other workloads on +it draw from them. + +## What this depends on + +An interface resource is necessary and not sufficient. Each of the following is assumed and +none of it is provided here. + +- **[Compute PR #210](https://github.com/datum-cloud/compute/pull/210) must land first.** + This design consumes classes, `IPClaim`, retention, and the central allocator wholesale. + Without it there is no answer to what address an interface gets, and `spec.addresses` has + nothing to hold. +- **A network's default families and an interface's must agree.** `NetworkSpec.ipFamilies` + defaults to `[IPv4]` while a claim here defaults to `[IPv6]`, so the default workload on + the default network requests a family its network does not carry. PR #210 raises the same + mismatch as an open question; this design is where it surfaces as a hard failure, and it + needs settling before either ships. +- **A network needs one routing identity across every location it reaches**, unique + platform-wide, or the two halves of a multi-location workload are unrelated networks + sharing a name. That it is not the per-location forwarding-instance identifier needs to + stay true. +- **A moved instance needs its old route withdrawn before the new one is trusted.** + Retention makes this worse, not better: the address stays valid across the move, so both + advertisements look legitimate and traffic splits. +- **Subnets and gateways need programming, not just allocation.** + `spec.addresses[].gateway` is only useful if something answers at that address. Without + it, oversized packets are dropped silently — handshakes succeed and large transfers hang. + This is exactly why `Programmed` is a separate condition rather than folded into + `Allocated`. +- **Consuming a class must be a privilege.** `spec.addresses[].class` on a claim is written + by compute on a consumer's behalf. If the class name is the only authorization boundary, + the check on it must fail closed, and it must check the consumer's project rather than + the platform identity that made the call. +- **Providers must migrate.** The GCP and Unikraft providers read `NetworkBinding`, + `NetworkContext`, `SubnetClaim`, and `Subnet` today. They keep working until they are + moved to `NetworkInterface`, and compute's creation of the old objects cannot be removed + until every one of them has moved. + +## Drawbacks + +- **Two resources where consumers see none.** Every interface now costs a claim, an + interface, and an attachment where a deployment used to cost one shared subnet claim. At + four replicas that is twelve objects instead of one. The cost is per-instance object + count in the POP cell, and it buys the per-instance address that motivates the work. +- **A retained interface holds capacity nobody else can use.** That is the price of an + address that survives a scale-down, and on a finite public range it is the price that + matters. The lease is the mitigation, not the elimination. +- **The binding is invisible until it exists.** A claim that cannot bind — no space in the + location, a family the network does not carry — reports a condition and holds its instance + gated. That is correct and it is also a new way for an instance to be stuck, which needs + the failure to name what could not be satisfied rather than reporting "not ready". +- **Splitting `Allocated` from `Programmed` makes readiness slower to reach**, on purpose. + Instances that used to come up on allocation now wait for the data plane. Some workloads + will notice. + +## Alternatives + +- **Put the addresses on `Instance.status` and skip both resources.** Rejected: it is close + to the status quo, it gives providers nothing stable to watch, and an address on an + instance's status cannot outlive the instance, which forecloses retention entirely. +- **One resource instead of two — a `NetworkInterface` compute creates directly.** + Rejected: it merges intent with result, so a provider cannot tell a requested address + from an allocated one, and it removes the object that survives the instance. The claim + exists precisely so something outlives its holder. +- **Reuse `VPCAttachment` as the compute-facing resource.** Rejected: an attachment + requires a node, a container ID, and device names, none of which exist when the address + must be allocated. It is the realization of an interface, not the request for one. +- **Keep `NetworkBinding` and `SubnetClaim` and just add a per-instance `SubnetClaim`.** + Rejected: it fixes the sharing problem and none of the others. Compute stays coupled to + NSO internals, providers still walk four objects, and there is still nothing that survives + an instance. +- **Bind by re-matching rather than by never unbinding.** Rejected for the same reason + compute PR #210 rejects it: releasing an interface on instance deletion and re-matching it + later opens a window where the address is loose and needs a durable identity distinct from + the holder. The claim name is already that identity. +- **A `Released` phase, as storage has.** Rejected: it requires an operator to clear a stale + reference before the interface is usable, which for a public address means capacity held + hostage to a manual step. The `uid` on `claimRef` provides the same safety without the + dead end. +- **Let the claim carry the location explicitly**, as `SubnetClaim` does. Rejected: the + claim exists in the cell that serves the location, so the field would be a value the + writer copies from where it is already standing, and a value that can disagree with + reality. + +## Open Questions + +**What size block does an endpoint actually get?** The +[tenant addressing plan](https://github.com/datum-cloud/enhancements/blob/main/architecture/design/network/addressing/tenant.md) +specifies a `/96` per endpoint. `VPCAttachment.status.podSubnet` documents a `/80`. Both +cannot be right, and the answer changes what `spec.addresses[].address` holds for IPv6 and +how much space an interface can hand to containers without a control-plane round trip. + +**Where does a load balancer's interface come from?** Nothing in this design is +instance-specific, and a load balancer needs the same object with the same fields. Whether +it uses the same claim kind, or the shape simply gets copied, should be settled before a +second consumer arrives and settles it by accident. + +**Does the claim need to express bandwidth or queue policy?** Interface-level rate limits, +queue disciplines, and offload capabilities are properties of a NIC a consumer might +reasonably ask for. Adding them later is additive; deciding now whether they belong on the +claim or on the instance type keeps them from landing in both. + +**Should an interface be attachable to more than one network?** The model says no — one +interface, one network, and multi-network reachability comes from multiple interfaces. That +is the simpler answer and it should be confirmed against the connector and interconnect +work rather than assumed. + +**Does `NetworkInterface` need to name the VPC in `spec`?** Today it appears only in +`status`, discovered when the attachment is programmed. If a provider ever needs the VPC +identity before an attachment exists, it moves — and that would make the interface depend on +a location's data plane being resolved at allocation time, which it currently does not. + +## References + +**What this builds on** + +- [IP classes for workload address allocation](https://github.com/datum-cloud/compute/pull/210) + — where an address comes from, and the retention model this document reuses without + restating. +- [Tenant addressing](https://github.com/datum-cloud/enhancements/blob/main/architecture/design/network/addressing/tenant.md) + — the per-network IPv6 `/48`, the per-location `/64`, and the per-endpoint block an + interface holds. +- [Federated Deployment Scheduling](https://github.com/datum-cloud/compute/blob/main/docs/enhancements/federated-deployment-scheduling.md) + — which control plane each object lives in, and the write-back path addresses travel to + reach a consumer. + +**The work this serves** + +- [network-services-operator#164](https://github.com/datum-cloud/network-services-operator/issues/164) + — the issue calling for these two resources, and the decoupling they provide. +- [compute#112](https://github.com/datum-cloud/compute/issues/112) + — the compute-side issue: per-instance allocation, addresses that reach + `Instance.status`, and the `ipFamilies` and `reclaimPolicy` fields. + +**The types involved** + +- [`Network`, `NetworkContext`, `SubnetClaim`](../../api/v1alpha) — the network an interface + belongs to, its presence in a location, and the internals compute stops writing. +- [`VPC` and `VPCAttachment`](https://github.com/datum-cloud/cloud/tree/main/api/v1alpha1) + — the data plane an interface is realized on. +- [Compute API types](https://github.com/datum-cloud/compute/tree/main/api/v1alpha) + — `Instance`, `InstanceNetworkInterface`, and the status fields this fills in. From cdba60f02828f5e76d63cf891d551ee39ab1ba9a Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 12:24:26 -0500 Subject: [PATCH 03/13] feat: fulfill network interface claims from IPAM Binds a claim by allocating real addresses from the IPAM service, then publishing an interface carrying them. Allocation is synchronous, so a bound claim reports its addresses in the same reconcile. The project an allocation belongs to comes from the namespace the claim lives in, asserted per request by impersonation over one shared upstream connection. A namespace naming no project allocates nothing and says which label is missing, because allocating into the wrong project is the worst outcome available here. Key changes: - One IPClaim per requested family and per named class, each named deterministically from the claim so a replacement instance finds the addresses that already exist. - Allocation is all or nothing, and a partial failure releases what it took. - Delete removes the interface and its addresses; Retain returns the interface to Available still holding them, and a later claim of the same name rebinds it. - Adoption is refused when an interface belongs to another network, or does not satisfy the claim, rather than publishing another network's addresses. - Finalizers on both objects release addresses whichever is deleted first, and a watch rebuilds an interface deleted under a live claim. - Conditions report what is true: Allocated turns false when an address has no allocation behind it or a different one, and Ready is computed from the three conditions it summarizes. - Failures name their cause, and a transient one is retried rather than leaving a healthy claim reporting failure forever. --- config/e2e/config.yaml | 8 + config/rbac/role.yaml | 13 + go.mod | 58 +- go.sum | 106 +- internal/cmd/manager/manager.go | 135 +- internal/config/config.go | 92 ++ internal/config/zz_generated.deepcopy.go | 49 + internal/config/zz_generated.defaults.go | 11 + internal/controller/ipam_errors.go | 57 + internal/controller/ipam_project_client.go | 147 ++ internal/controller/metrics.go | 11 + .../controller/networkinterface_controller.go | 132 ++ .../networkinterfaceclaim_controller.go | 1257 ++++++++++++++++ .../networkinterfaceclaim_controller_test.go | 1301 +++++++++++++++++ 14 files changed, 3257 insertions(+), 120 deletions(-) create mode 100644 internal/controller/ipam_errors.go create mode 100644 internal/controller/ipam_project_client.go create mode 100644 internal/controller/networkinterface_controller.go create mode 100644 internal/controller/networkinterfaceclaim_controller.go create mode 100644 internal/controller/networkinterfaceclaim_controller_test.go diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index 812afa1a..b11fb7a1 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -16,3 +16,11 @@ gateway: gateway.networking.datumapis.com/certificate-issuer: [] downstreamResourceManagement: kubeconfigPath: /etc/downstream-cluster/kubeconfig +ipam: + kubeconfigPath: /etc/ipam-cluster/kubeconfig + impersonateUsername: nso-ipam-agent +networkInterface: + enabled: true + location: + name: us-central-1 + namespace: default diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index fd1e8b98..cf59916f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -125,6 +125,13 @@ rules: - get - list - watch +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - patch - apiGroups: - externaldns.k8s.io resources: @@ -229,6 +236,8 @@ rules: - httpproxies - networkbindings - networkcontexts + - networkinterfaceclaims + - networkinterfaces - networkpolicies - networks - subnetclaims @@ -251,6 +260,8 @@ rules: - httpproxies/finalizers - networkbindings/finalizers - networkcontexts/finalizers + - networkinterfaceclaims/finalizers + - networkinterfaces/finalizers - networkpolicies/finalizers - networks/finalizers - subnetclaims/finalizers @@ -267,6 +278,8 @@ rules: - httpproxies/status - networkbindings/status - networkcontexts/status + - networkinterfaceclaims/status + - networkinterfaces/status - networkpolicies/status - networks/status - subnetclaims/status diff --git a/go.mod b/go.mod index 2c461b2e..4f21ba48 100644 --- a/go.mod +++ b/go.mod @@ -25,21 +25,21 @@ require ( github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 go.miloapis.com/dns-operator v0.5.1 - go.miloapis.com/milo v0.28.1 + go.miloapis.com/milo v0.28.4-0.20260629130346-79689376fe11 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 go.opentelemetry.io/otel/sdk v1.44.0 go.opentelemetry.io/otel/trace v1.44.0 - golang.org/x/net v0.55.0 - golang.org/x/sync v0.20.0 + golang.org/x/net v0.56.0 + golang.org/x/sync v0.22.0 google.golang.org/grpc v1.81.1 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af k8s.io/api v0.36.1 k8s.io/apiextensions-apiserver v0.36.1 k8s.io/apimachinery v0.36.1 k8s.io/client-go v0.36.1 - k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 + k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/gateway-api v1.5.1 sigs.k8s.io/gateway-api/conformance v1.5.1 @@ -47,6 +47,11 @@ require ( sigs.k8s.io/yaml v1.6.0 ) +require ( + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect +) + require ( cel.dev/expr v0.25.2 // indirect dario.cat/mergo v1.0.2 // indirect @@ -120,7 +125,7 @@ require ( github.com/docker/go-events v0.0.0-20250808211157-605354379745 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dominikbraun/graph v0.23.0 // indirect - github.com/ebitengine/purego v0.10.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260409050421-3f47accd6e14 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect @@ -128,7 +133,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fatih/color v1.19.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -146,7 +151,7 @@ require ( github.com/go-openapi/loads v0.23.3 // indirect github.com/go-openapi/spec v0.22.5 // indirect github.com/go-openapi/strfmt v0.26.3 // indirect - github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-openapi/swag v0.25.4 // indirect github.com/go-openapi/swag/conv v0.26.0 // indirect github.com/go-openapi/swag/fileutils v0.26.0 // indirect github.com/go-openapi/swag/jsonname v0.26.0 // indirect @@ -167,7 +172,7 @@ require ( github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-containerregistry v0.21.6 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect @@ -183,7 +188,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/kortschak/goroutine v1.1.3 // indirect @@ -193,8 +197,7 @@ require ( github.com/lib/pq v1.11.2 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e // indirect - github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect - github.com/mailru/easyjson v0.9.1 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -206,11 +209,11 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect - github.com/moby/moby/api v1.54.2 // indirect - github.com/moby/moby/client v0.4.1 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/moby/api v1.55.0 // indirect + github.com/moby/moby/client v0.5.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/spdystream v0.5.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect @@ -247,7 +250,7 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/segmentio/ksuid v1.0.4 // indirect - github.com/shirou/gopsutil/v4 v4.26.4 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect @@ -257,8 +260,8 @@ require ( github.com/spf13/viper v1.21.0 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tklauser/go-sysconf v0.3.16 // indirect - github.com/tklauser/numcpus v0.11.0 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/vishvananda/netlink v1.3.1 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -270,9 +273,10 @@ require ( go.etcd.io/etcd/api/v3 v3.6.8 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect + go.miloapis.com/ipam v0.3.2-0.20260813020700-13bd2c8c077b go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect @@ -284,16 +288,16 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.52.0 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/crypto/x509roots/fallback v0.0.0-20250406160420-959f8f3db0fb // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/mod v0.36.0 // indirect + golang.org/x/mod v0.37.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/tools v0.47.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect @@ -306,7 +310,7 @@ require ( k8s.io/cli-runtime v0.36.1 // indirect k8s.io/component-base v0.36.1 // indirect k8s.io/klog/v2 v2.140.0 // indirect - k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 // indirect k8s.io/kubectl v0.36.1 // indirect k8s.io/metrics v0.36.1 // indirect k8s.io/streaming v0.36.1 // indirect @@ -321,3 +325,5 @@ require ( sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) + +replace go.miloapis.com/ipam => github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b diff --git a/go.sum b/go.sum index 644b3719..6d66fd3d 100644 --- a/go.sum +++ b/go.sum @@ -190,8 +190,8 @@ github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucV github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= -github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/gateway v1.8.1 h1:i8POtrPtel3n0iGP55MNy6U1rTW4msyhkEWjXAF09ts= @@ -214,8 +214,8 @@ github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0= github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -266,8 +266,10 @@ github.com/go-openapi/spec v0.22.5 h1:KhO7RBlKQfonUWX2WzQCoLIXVA6AcNqDGZ3a1Dutdl github.com/go-openapi/spec v0.22.5/go.mod h1:vxpOtMya5TXtENXKE5bKqv5NjocVhyhxHrlZfvKnZ74= github.com/go-openapi/strfmt v0.26.3 h1:rzmslHarJgBbf2qfGge+X3htclQfmXqBZMm0Too0HhU= github.com/go-openapi/strfmt v0.26.3/go.mod h1:a5nsUw0oRpQzZeOwx8bi6cKbzFZslpbCKt1LEot+KnQ= -github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= -github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= github.com/go-openapi/swag/conv v0.26.0 h1:5yGGsPYI1ZCva93U0AoKi/iZrNhaJEjr324YVsiD89I= github.com/go-openapi/swag/conv v0.26.0/go.mod h1:tpAmIL7X58VPnHHiSO4uE3jBeRamGsFsfdDeDtb5ECE= github.com/go-openapi/swag/fileutils v0.26.0 h1:WJoPRvsA7QRiiWluowkLJa9jaYR7FCuxmDvnCgaRRxU= @@ -282,6 +284,8 @@ github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaM github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= github.com/go-openapi/swag/mangling v0.26.0 h1:Du2YC4YLA/Y5m/YKQd7AnY5qq0wRKSFZTTt8ktFaXcQ= github.com/go-openapi/swag/mangling v0.26.0/go.mod h1:jifS7W9vbg+pw63bT+GI53otluMQL3CeemuyCHKwVx0= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= github.com/go-openapi/swag/stringutils v0.26.0 h1:qZQngLxs5s7SLijc3N2ZO+fUq2o8LjuWAASSrJuh+xg= github.com/go-openapi/swag/stringutils v0.26.0/go.mod h1:sWn5uY+QIIspwPhvgnqJsH8xqFT2ZbYcvbcFanRyhFE= github.com/go-openapi/swag/typeutils v0.26.0 h1:2kdEwdiNWy+JJdOvu5MA2IIg2SylWAFuuyQIKYybfq4= @@ -329,8 +333,8 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -386,8 +390,6 @@ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= @@ -427,10 +429,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9 github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e h1:hz4quJkaJWDo+xW+G6wTF6d6/95QvJ+o2D0+bB/tJ1U= github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e/go.mod h1:9z/y9glKmWEdV50tjlUPxFwi1goQfIrrsoZbnMyIZbY= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003/go.mod h1:0CLeXlf59Lg6C0kjLSDf47ft73Dh37CwymYRKWwAn04= -github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k= -github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= -github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= -github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -453,6 +453,8 @@ github.com/microsoft/go-mssqldb v1.10.0/go.mod h1:mnG7lGa9iYJbzJqGCXyuQCegStKMr3 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b h1:iJ0zDKVRgC4cz0/eOX8dp+wBNw8iqaGI87zwgqdBZH8= +github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b/go.mod h1:Jj7xg4lJi9psE0+4PuOg/GQOG8rG13h112xYoM994rc= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -465,16 +467,16 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= -github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= -github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY= -github.com/moby/moby/client v0.4.1/go.mod h1:z52C9O2POPOsnxZAy//WtKcQ32P+jT/NGeXu/7nfjGQ= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= @@ -589,8 +591,8 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shirou/gopsutil/v4 v4.26.4 h1:B4SXVbcwTyrocPHEmWBC4uCYr4Xcu3MK1TXqbprAOWY= -github.com/shirou/gopsutil/v4 v4.26.4/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -642,10 +644,10 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= -github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= -github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= -github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= @@ -692,8 +694,8 @@ go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= go.miloapis.com/dns-operator v0.5.1 h1:3HmturI/Opl+l4/KurN6B0/XEeVefoSvYU0ewcZ8BZ8= go.miloapis.com/dns-operator v0.5.1/go.mod h1:yQSBOx1ZhSzZgg5RyNKyNHfZgjXRKhxkcGD+jvbebQE= -go.miloapis.com/milo v0.28.1 h1:30bQS4EwadbOBsn3UC8qSBENN1koQgsQpcvqPjXfmlI= -go.miloapis.com/milo v0.28.1/go.mod h1:p9O2kk194mvoL8rhqjwb+LWB+GIyY4vQqiTowwibVWo= +go.miloapis.com/milo v0.28.4-0.20260629130346-79689376fe11 h1:gWqYe5aYpRn7jFUo2hxSsvj0KHC2xFzK2RGCcd3ClHg= +go.miloapis.com/milo v0.28.4-0.20260629130346-79689376fe11/go.mod h1:p9O2kk194mvoL8rhqjwb+LWB+GIyY4vQqiTowwibVWo= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= @@ -703,8 +705,8 @@ go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 h1:4fnRcNpc6YFtG3zsFw9a go.opentelemetry.io/contrib/exporters/autoexport v0.67.0/go.mod h1:qTvIHMFKoxW7HXg02gm6/Wofhq5p3Ib/A/NNt1EoBSQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 h1:Dn8rkudDzY6KV9dr/D/bTUuWgqDf9xe0rr4G2elrn0Y= @@ -762,8 +764,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/crypto/x509roots/fallback v0.0.0-20250406160420-959f8f3db0fb h1:Iu0p/klM0SM7atONioa/bPhLS7cjhnip99x1OIGibwg= golang.org/x/crypto/x509roots/fallback v0.0.0-20250406160420-959f8f3db0fb/go.mod h1:lxN5T34bK4Z/i6cMaU7frUU57VkDXFD4Kamfl/cp9oU= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= @@ -771,8 +773,8 @@ golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aI golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -783,8 +785,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -792,8 +794,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -815,18 +817,18 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -834,8 +836,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -898,16 +900,16 @@ k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kms v0.36.1 h1:XdvKpywoW4k7YUHDh5uYP4mahJXECswHGfCddBBYLZs= k8s.io/kms v0.36.1/go.mod h1:g91diTD9h0oJCCHkTb00krlF+Qm5HTnkWLi9Q/TpRoc= -k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= -k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 h1:V+sn9a/1fEYDGwnllCmqXBk8x7obZ+hl869Q3Abumkg= +k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= k8s.io/kubectl v0.36.1 h1:96HqS9twIdHM0MlJLTwbo14b9kUKPkOzZ4tlRDLv4qI= k8s.io/kubectl v0.36.1/go.mod h1:/DGPAIewKsFWF9VFgGvkPhao2Ev4SNuE3BioZo8yPbk= k8s.io/metrics v0.36.1 h1:MQPb+G4RhrKEpt8NETPssbW8QgGUc4Jbqu1jx+kPqGk= k8s.io/metrics v0.36.1/go.mod h1:xqS8XcWLjDzo6E7DJm/GfjKpRKdN5/MtJAQFuV6nLUc= k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= -k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= -k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o= periph.io/x/host/v3 v3.8.5 h1:g4g5xE1XZtDiGl1UAJaUur1aT7uNiFLMkyMEiZ7IHII= diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 11dd37b3..397c2d19 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -50,6 +50,7 @@ import ( networkingv1alphawebhooks "go.datum.net/network-services-operator/internal/webhook/v1alpha" webhookgatewayv1alpha1 "go.datum.net/network-services-operator/internal/webhook/v1alpha1" dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -73,6 +74,7 @@ func init() { utilruntime.Must(cmacmev1.AddToScheme(scheme)) utilruntime.Must(cmv1.AddToScheme(scheme)) utilruntime.Must(dnsv1alpha1.AddToScheme(scheme)) + utilruntime.Must(ipamv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -370,6 +372,13 @@ func NewCommand(build BuildInfo) *cobra.Command { os.Exit(1) } + if serverConfig.NetworkInterface.Enabled { + if err := setupNetworkInterfaceClaimController(serverConfig, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NetworkInterfaceClaim") + os.Exit(1) + } + } + if err := (&controller.HTTPProxyReconciler{ Config: serverConfig, DownstreamCluster: downstreamCluster, @@ -492,48 +501,8 @@ func NewCommand(build BuildInfo) *cobra.Command { } } - if err := networkinggatewayv1webhooks.SetupGatewayWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Gateway") - os.Exit(1) - } - - if err := networkinggatewayv1webhooks.SetupHTTPRouteWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRoute") - os.Exit(1) - } - - if err := networkinggatewayv1webhooks.SetupBackendTLSPolicyWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRoute") - os.Exit(1) - } - - if err := networkingv1alphawebhooks.SetupHTTPProxyWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPProxy") - os.Exit(1) - } - - if err := networkingv1alphawebhooks.SetupDomainWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Domain") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupBackendTrafficPolicyWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "BackendTrafficPolicy") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupSecurityPolicyWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "SecurityPolicy") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupHTTPRouteFilterWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRouteFilter") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupBackendWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Backend") + if webhook, err := setupWebhooks(mgr, serverConfig); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", webhook) os.Exit(1) } @@ -611,6 +580,88 @@ type legacyRunnableProvider interface { Run(context.Context, mcmanager.Manager) error } +// setupWebhooks registers every admission webhook. It returns the name of the +// webhook that failed. +func setupWebhooks(mgr mcmanager.Manager, serverConfig config.NetworkServicesOperator) (string, error) { + registrations := []struct { + name string + setup func() error + }{ + {"Gateway", func() error { + return networkinggatewayv1webhooks.SetupGatewayWebhookWithManager(mgr, serverConfig) + }}, + {"HTTPRoute", func() error { + return networkinggatewayv1webhooks.SetupHTTPRouteWebhookWithManager(mgr, serverConfig) + }}, + {"BackendTLSPolicy", func() error { + return networkinggatewayv1webhooks.SetupBackendTLSPolicyWebhookWithManager(mgr) + }}, + {"HTTPProxy", func() error { + return networkingv1alphawebhooks.SetupHTTPProxyWebhookWithManager(mgr) + }}, + {"Domain", func() error { + return networkingv1alphawebhooks.SetupDomainWebhookWithManager(mgr) + }}, + {"BackendTrafficPolicy", func() error { + return webhookgatewayv1alpha1.SetupBackendTrafficPolicyWebhookWithManager(mgr, serverConfig) + }}, + {"SecurityPolicy", func() error { + return webhookgatewayv1alpha1.SetupSecurityPolicyWebhookWithManager(mgr, serverConfig) + }}, + {"HTTPRouteFilter", func() error { + return webhookgatewayv1alpha1.SetupHTTPRouteFilterWebhookWithManager(mgr, serverConfig) + }}, + {"Backend", func() error { + return webhookgatewayv1alpha1.SetupBackendWebhookWithManager(mgr) + }}, + } + + for _, registration := range registrations { + if err := registration.setup(); err != nil { + return registration.name, err + } + } + return "", nil +} + +// setupNetworkInterfaceClaimController wires the controllers to one IPAM +// connection. Impersonation names the project on each request. +func setupNetworkInterfaceClaimController( + serverConfig config.NetworkServicesOperator, + mgr mcmanager.Manager, +) error { + ipamRestConfig, err := serverConfig.IPAM.RestConfig() + if err != nil { + return fmt.Errorf("unable to load IPAM kubeconfig: %w", err) + } + + ipamScheme, err := controller.IPAMScheme() + if err != nil { + return fmt.Errorf("unable to build IPAM scheme: %w", err) + } + + ipamClients, err := controller.NewIPAMClientFactory( + ipamRestConfig, + ipamScheme, + serverConfig.IPAM.ImpersonateUsername, + ) + if err != nil { + return fmt.Errorf("unable to build IPAM client factory: %w", err) + } + + if err := (&controller.NetworkInterfaceClaimReconciler{ + Config: serverConfig, + IPAM: ipamClients, + }).SetupWithManager(mgr); err != nil { + return err + } + + return (&controller.NetworkInterfaceReconciler{ + Config: serverConfig, + IPAM: ipamClients, + }).SetupWithManager(mgr) +} + func initializeClusterDiscovery( serverConfig config.NetworkServicesOperator, deploymentCluster cluster.Cluster, diff --git a/internal/config/config.go b/internal/config/config.go index 575e7f8f..0c6c852d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -86,6 +86,95 @@ type NetworkServicesOperator struct { // ProjectClient configures the Kubernetes client connection used for both // project discovery and per-project cluster connections. ProjectClient ClientConnectionConfig `json:"projectClient,omitempty"` + + // IPAM configures the connection to the IPAM aggregated API server that + // network interface addresses are claimed from. + IPAM IPAMConfig `json:"ipam,omitempty"` + + // NetworkInterface configures the controller that fulfils + // NetworkInterfaceClaims. + NetworkInterface NetworkInterfaceConfig `json:"networkInterface,omitempty"` +} + +// +k8s:deepcopy-gen=true + +// IPAMConfig describes how the operator reaches the IPAM API server. One +// connection serves every project. Each request names its project through +// impersonation. +type IPAMConfig struct { + // KubeconfigPath is the path to a kubeconfig file pointing at the cluster + // serving the IPAM API. When empty, the operator's own in-cluster config is + // used. + KubeconfigPath string `json:"kubeconfigPath,omitempty"` + + // ImpersonateUsername is the user the operator impersonates when claiming + // addresses. IPAM checks this user's permissions, not the operator's. + // + // +default="nso-ipam-agent" + ImpersonateUsername string `json:"impersonateUsername,omitempty"` + + // Client configures the Kubernetes client connection to the IPAM API + // server. + Client ClientConnectionConfig `json:"client,omitempty"` +} + +func SetDefaults_IPAMConfig(obj *IPAMConfig) { + if obj.ImpersonateUsername == "" { + obj.ImpersonateUsername = "nso-ipam-agent" + } +} + +func (c *IPAMConfig) RestConfig() (*rest.Config, error) { + var ( + cfg *rest.Config + err error + ) + if c.KubeconfigPath == "" { + cfg, err = ctrl.GetConfig() + } else { + cfg, err = clientcmd.BuildConfigFromFlags("", c.KubeconfigPath) + } + if err != nil { + return nil, err + } + + c.Client.ApplyTo(cfg) + return cfg, nil +} + +// +k8s:deepcopy-gen=true + +// NetworkInterfaceConfig configures the NetworkInterfaceClaim controller. +type NetworkInterfaceConfig struct { + // Enabled registers the NetworkInterfaceClaim controller. Leave it off on a + // control plane that serves no location. + Enabled bool `json:"enabled,omitempty"` + + // Location is the location this control plane serves. Claims carry no + // location of their own, so every claim allocates against this one. + Location LocationConfig `json:"location,omitempty"` +} + +// +k8s:deepcopy-gen=true + +type LocationConfig struct { + Name string `json:"name,omitempty"` + + Namespace string `json:"namespace,omitempty"` +} + +func (c *NetworkInterfaceConfig) validate() error { + if !c.Enabled { + return nil + } + var errs []error + if c.Location.Name == "" { + errs = append(errs, errors.New("location.name is required when enabled is true")) + } + if c.Location.Namespace == "" { + errs = append(errs, errors.New("location.namespace is required when enabled is true")) + } + return errors.Join(errs...) } // +k8s:deepcopy-gen=true @@ -1273,6 +1362,9 @@ func (c *NetworkServicesOperator) Validate() error { if err := c.Gateway.validate(); err != nil { return fmt.Errorf("gateway: %w", err) } + if err := c.NetworkInterface.validate(); err != nil { + return fmt.Errorf("networkInterface: %w", err) + } return nil } diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index 131f416d..bee685a2 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -447,6 +447,22 @@ func (in *HTTPRouteValidationOptions) DeepCopy() *HTTPRouteValidationOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAMConfig) DeepCopyInto(out *IPAMConfig) { + *out = *in + out.Client = in.Client +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAMConfig. +func (in *IPAMConfig) DeepCopy() *IPAMConfig { + if in == nil { + return nil + } + out := new(IPAMConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IrohConnectorConfig) DeepCopyInto(out *IrohConnectorConfig) { *out = *in @@ -496,6 +512,21 @@ func (in *LeaderElectionConfig) DeepCopy() *LeaderElectionConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocationConfig) DeepCopyInto(out *LocationConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocationConfig. +func (in *LocationConfig) DeepCopy() *LocationConfig { + if in == nil { + return nil + } + out := new(LocationConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) { *out = *in @@ -517,6 +548,22 @@ func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceConfig) DeepCopyInto(out *NetworkInterfaceConfig) { + *out = *in + out.Location = in.Location +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceConfig. +func (in *NetworkInterfaceConfig) DeepCopy() *NetworkInterfaceConfig { + if in == nil { + return nil + } + out := new(NetworkInterfaceConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkServicesOperator) DeepCopyInto(out *NetworkServicesOperator) { *out = *in @@ -535,6 +582,8 @@ func (in *NetworkServicesOperator) DeepCopyInto(out *NetworkServicesOperator) { out.ControlPlaneClient = in.ControlPlaneClient out.DownstreamClient = in.DownstreamClient out.ProjectClient = in.ProjectClient + out.IPAM = in.IPAM + out.NetworkInterface = in.NetworkInterface } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkServicesOperator. diff --git a/internal/config/zz_generated.defaults.go b/internal/config/zz_generated.defaults.go index 87f5179c..acc4d62d 100644 --- a/internal/config/zz_generated.defaults.go +++ b/internal/config/zz_generated.defaults.go @@ -359,4 +359,15 @@ func SetObjectDefaults_NetworkServicesOperator(in *NetworkServicesOperator) { if in.ProjectClient.Burst == 0 { in.ProjectClient.Burst = 100 } + SetDefaults_IPAMConfig(&in.IPAM) + if in.IPAM.ImpersonateUsername == "" { + in.IPAM.ImpersonateUsername = "nso-ipam-agent" + } + SetDefaults_ClientConnectionConfig(&in.IPAM.Client) + if in.IPAM.Client.QPS == 0 { + in.IPAM.Client.QPS = 50 + } + if in.IPAM.Client.Burst == 0 { + in.IPAM.Client.Burst = 100 + } } diff --git a/internal/controller/ipam_errors.go b/internal/controller/ipam_errors.go new file mode 100644 index 00000000..805db6c5 --- /dev/null +++ b/internal/controller/ipam_errors.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "errors" + "net/http" + + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +// allocationFailureReason classifies why IPAM refused an allocation, by status +// code only. IPAM's own message is passed through unread. +type allocationFailureReason string + +const ( + allocationFailureExhausted allocationFailureReason = "AddressPoolExhausted" + allocationFailureConflict allocationFailureReason = "RetainedAddressConflict" + // Usually an address class or pool that is not configured, rather than a + // bad request, and the status code does not tell the two apart. + allocationFailureRejected allocationFailureReason = "AddressAllocationRejected" + allocationFailureUnknown allocationFailureReason = "AllocationFailed" +) + +const httpStatusInsufficientStorage = 507 + +func classifyAllocationFailure(err error) allocationFailureReason { + var status apierrors.APIStatus + if !errors.As(err, &status) { + return allocationFailureUnknown + } + + switch int(status.Status().Code) { + case httpStatusInsufficientStorage: + return allocationFailureExhausted + case http.StatusConflict: + return allocationFailureConflict + case http.StatusBadRequest, http.StatusUnprocessableEntity: + return allocationFailureRejected + default: + return allocationFailureUnknown + } +} + +func allocationFailureMessage(reason allocationFailureReason, request allocationRequest, err error) string { + switch reason { + case allocationFailureExhausted: + return "No address is left for " + request.describe() + ": " + err.Error() + case allocationFailureConflict: + return "A retained allocation still holds the name for " + request.describe() + ": " + err.Error() + case allocationFailureRejected: + return "IPAM would not allocate " + request.describe() + + ". This usually means the address class, or a pool backing it, is not configured in this project: " + err.Error() + default: + return "Allocating " + request.describe() + " failed: " + err.Error() + } +} diff --git a/internal/controller/ipam_project_client.go b/internal/controller/ipam_project_client.go new file mode 100644 index 00000000..dd7589c6 --- /dev/null +++ b/internal/controller/ipam_project_client.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "sync" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + + "go.datum.net/network-services-operator/internal/downstreamclient" +) + +const ( + ipamParentAPIGroupExtraKey = "iam.miloapis.com/parent-api-group" + ipamParentTypeExtraKey = "iam.miloapis.com/parent-type" + ipamParentNameExtraKey = "iam.miloapis.com/parent-name" + + ipamParentAPIGroup = "resourcemanager.miloapis.com" + ipamParentType = "Project" +) + +// IPAMClientFactory returns a client bound to one project. Every IPAM request +// goes through one, so no request can reach IPAM without naming a project. +type IPAMClientFactory interface { + ClientForProject(project string) (client.Client, error) +} + +// NewIPAMClientFactory builds project-scoped clients from one connection. The +// clients are uncached, because a cache would watch every project served. +func NewIPAMClientFactory(base *rest.Config, scheme *runtime.Scheme, actAsUsername string) (IPAMClientFactory, error) { + if base == nil { + return nil, fmt.Errorf("a rest config is required") + } + if actAsUsername == "" { + return nil, fmt.Errorf("an impersonation username is required") + } + return &impersonatingIPAMClientFactory{ + base: base, + scheme: scheme, + actAsUsername: actAsUsername, + clients: map[string]client.Client{}, + }, nil +} + +type impersonatingIPAMClientFactory struct { + base *rest.Config + scheme *runtime.Scheme + actAsUsername string + + mu sync.Mutex + clients map[string]client.Client +} + +func (f *impersonatingIPAMClientFactory) ClientForProject(project string) (client.Client, error) { + if project == "" { + return nil, errNoProject + } + + f.mu.Lock() + defer f.mu.Unlock() + + if existing, ok := f.clients[project]; ok { + return existing, nil + } + + cfg := rest.CopyConfig(f.base) + cfg.Impersonate = rest.ImpersonationConfig{ + UserName: f.actAsUsername, + Extra: map[string][]string{ + ipamParentAPIGroupExtraKey: {ipamParentAPIGroup}, + ipamParentTypeExtraKey: {ipamParentType}, + ipamParentNameExtraKey: {project}, + }, + } + + cl, err := client.New(cfg, client.Options{Scheme: f.scheme}) + if err != nil { + return nil, fmt.Errorf("failed building IPAM client for project %q: %w", project, err) + } + + f.clients[project] = cl + return cl, nil +} + +// IPAMScheme is the scheme a project-scoped IPAM client is built with. +func IPAMScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := ipamv1alpha1.AddToScheme(scheme); err != nil { + return nil, err + } + return scheme, nil +} + +var errNoProject = fmt.Errorf("no project") + +// projectFromNamespace reads the project a namespace belongs to. A namespace +// that names no project resolves to nothing, never to a default. +func projectFromNamespace(ns *corev1.Namespace) (string, error) { + value, ok := ns.Labels[downstreamclient.UpstreamOwnerClusterNameLabel] + if !ok || value == "" { + return "", fmt.Errorf("namespace %q carries no %s label", ns.Name, downstreamclient.UpstreamOwnerClusterNameLabel) + } + + project := downstreamclient.UpstreamClusterNameFromLabel(value) + if project == "" { + return "", fmt.Errorf("namespace %q has %s=%q, which names no project", ns.Name, downstreamclient.UpstreamOwnerClusterNameLabel, value) + } + + return project, nil +} + +func projectNamespaceFromNamespace(ns *corev1.Namespace) (string, error) { + value, ok := ns.Labels[downstreamclient.UpstreamOwnerNamespaceLabel] + if !ok || value == "" { + return "", fmt.Errorf("namespace %q carries no %s label", ns.Name, downstreamclient.UpstreamOwnerNamespaceLabel) + } + return value, nil +} + +func ensureProjectNamespace(ctx context.Context, cl client.Client, name string) error { + var existing corev1.Namespace + err := cl.Get(ctx, client.ObjectKey{Name: name}, &existing) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + + namespace := &corev1.Namespace{} + namespace.Name = name + if err := cl.Create(ctx, namespace); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go index 78784ca8..71cf0c16 100644 --- a/internal/controller/metrics.go +++ b/internal/controller/metrics.go @@ -17,9 +17,20 @@ const ( metricLabelHostname = "hostname" metricLabelSecret = "secret" metricLabelReason = "reason" + metricLabelProject = "project" ) var ( + // missingAllocationsTotal counts addresses no IPClaim holds. IPAM may give + // the same address to another claim. + missingAllocationsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "nso_network_interface_missing_allocations_total", + Help: "Total addresses advertised by a NetworkInterface with no IPClaim holding them, by project.", + }, + []string{metricLabelProject}, + ) + // replicatorConflictsTotal counts resource-version conflicts observed by the // gateway-resource-replicator controller. Conflicts arise when the upstream or // downstream API server rejects an update because the local object's diff --git a/internal/controller/networkinterface_controller.go b/internal/controller/networkinterface_controller.go new file mode 100644 index 00000000..73bf047a --- /dev/null +++ b/internal/controller/networkinterface_controller.go @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "errors" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" +) + +// NetworkInterfaceReconciler releases an interface's addresses when the +// interface is deleted. A retained interface has no claim to do this for it. +type NetworkInterfaceReconciler struct { + Config config.NetworkServicesOperator + IPAM IPAMClientFactory + + claims *NetworkInterfaceClaimReconciler + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces/finalizers,verbs=update + +func (r *NetworkInterfaceReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx, "cluster", req.ClusterName) + + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + logger.Info("reconciling network interface") + defer logger.Info("reconcile complete") + + return ctrl.Result{}, r.reconcileInterface(ctx, cl.GetClient(), req.NamespacedName) +} + +func (r *NetworkInterfaceReconciler) reconcileInterface( + ctx context.Context, + cl client.Client, + key client.ObjectKey, +) error { + var iface networkingv1alpha.NetworkInterface + if err := cl.Get(ctx, key, &iface); err != nil { + return client.IgnoreNotFound(err) + } + + if iface.DeletionTimestamp.IsZero() { + if controllerutil.AddFinalizer(&iface, networkInterfaceFinalizer) { + if err := cl.Update(ctx, &iface); err != nil { + return fmt.Errorf("failed adding finalizer: %w", err) + } + } + return nil + } + + if !controllerutil.ContainsFinalizer(&iface, networkInterfaceFinalizer) { + return nil + } + + held, err := r.heldByLiveClaim(ctx, cl, &iface) + if err != nil { + return err + } + + // The claim rebuilds the interface and finds the same addresses by name. + // Releasing here would give a running workload new ones. + if !held { + if err := r.claims.releaseAddresses(ctx, cl, iface.Namespace, &iface); err != nil { + return err + } + } + + controllerutil.RemoveFinalizer(&iface, networkInterfaceFinalizer) + if err := cl.Update(ctx, &iface); err != nil { + return fmt.Errorf("failed removing finalizer: %w", err) + } + + return nil +} + +// heldByLiveClaim reports whether a claim named in claimRef still exists and is +// not being deleted. +func (r *NetworkInterfaceReconciler) heldByLiveClaim( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, +) (bool, error) { + ref := iface.Spec.ClaimRef + if ref == nil { + return false, nil + } + + var claim networkingv1alpha.NetworkInterfaceClaim + key := client.ObjectKey{Namespace: iface.Namespace, Name: ref.Name} + if err := cl.Get(ctx, key, &claim); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("failed reading claim %q: %w", ref.Name, err) + } + + // A claim being deleted holds nothing. Its own release cannot free these + // addresses once the interface is gone. + return claim.DeletionTimestamp.IsZero(), nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NetworkInterfaceReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.IPAM == nil { + return errors.New("an IPAM client factory is required") + } + + r.mgr = mgr + r.claims = &NetworkInterfaceClaimReconciler{Config: r.Config, IPAM: r.IPAM, mgr: mgr} + + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkInterface{}, mcbuilder.WithEngageWithLocalCluster(false)). + Named("networkinterface"). + Complete(r) +} diff --git a/internal/controller/networkinterfaceclaim_controller.go b/internal/controller/networkinterfaceclaim_controller.go new file mode 100644 index 00000000..c48827d1 --- /dev/null +++ b/internal/controller/networkinterfaceclaim_controller.go @@ -0,0 +1,1257 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mchandler "sigs.k8s.io/multicluster-runtime/pkg/handler" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" +) + +const ( + networkInterfaceClaimFinalizer = "networking.datumapis.com/networkinterfaceclaim-release" + networkInterfaceFinalizer = "networking.datumapis.com/networkinterface-release" + + allocationClaimAnnotation = "networking.datumapis.com/allocation-claim" + + ipamScopeRoleNetwork = "network" + ipamScopeRoleLocation = "location" + + datumNetworkingAPIGroup = "networking.datumapis.com" + + maxObjectNameLength = 253 + ipClaimNameHashLen = 12 + + // Nothing watches the network, the namespace or IPAM, so a rejected claim + // needs its own way back. + rejectedClaimRetryInterval = time.Minute +) + +// NetworkInterfaceClaimReconciler binds a NetworkInterfaceClaim to a +// NetworkInterface. +type NetworkInterfaceClaimReconciler struct { + Config config.NetworkServicesOperator + IPAM IPAMClientFactory + + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkbindings,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=subnets,verbs=get;list;watch +// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch + +func (r *NetworkInterfaceClaimReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx, "cluster", req.ClusterName) + + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + logger.Info("reconciling network interface claim") + defer logger.Info("reconcile complete") + + return r.reconcileClaim(ctx, cl.GetClient(), cl.GetEventRecorder("networkinterfaceclaim-controller"), req.NamespacedName) +} + +func (r *NetworkInterfaceClaimReconciler) reconcileClaim( + ctx context.Context, + cl client.Client, + recorder events.EventRecorder, + key client.ObjectKey, +) (ctrl.Result, error) { + var claim networkingv1alpha.NetworkInterfaceClaim + if err := cl.Get(ctx, key, &claim); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !claim.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.release(ctx, cl, &claim) + } + + if controllerutil.AddFinalizer(&claim, networkInterfaceClaimFinalizer) { + if err := cl.Update(ctx, &claim); err != nil { + return ctrl.Result{}, fmt.Errorf("failed adding finalizer: %w", err) + } + } + + return r.fulfill(ctx, cl, recorder, &claim) +} + +func (r *NetworkInterfaceClaimReconciler) fulfill( + ctx context.Context, + cl client.Client, + recorder events.EventRecorder, + claim *networkingv1alpha.NetworkInterfaceClaim, +) (ctrl.Result, error) { + routing, err := r.resolveProject(ctx, cl, claim.Namespace) + if err != nil { + var unresolvable *projectUnresolvable + if errors.As(err, &unresolvable) { + return r.reject(ctx, cl, claim, "ProjectUnresolved", unresolvable.Error()) + } + return ctrl.Result{}, err + } + + var network networkingv1alpha.Network + networkKey := client.ObjectKey{Namespace: claim.Namespace, Name: claim.Spec.Network.Name} + if err := cl.Get(ctx, networkKey, &network); err != nil { + if apierrors.IsNotFound(err) { + return r.reject(ctx, cl, claim, "NetworkNotFound", + fmt.Sprintf("Network %q was not found in namespace %q", networkKey.Name, networkKey.Namespace)) + } + return ctrl.Result{}, fmt.Errorf("failed fetching network: %w", err) + } + + for _, family := range claim.Spec.IPFamilies { + if !slices.Contains(network.Spec.IPFamilies, family) { + return r.reject(ctx, cl, claim, "AddressFamilyNotCarried", + fmt.Sprintf("Network %q in namespace %q does not carry address family %s", + network.Name, network.Namespace, family)) + } + } + + networkContextName := r.resolveNetworkContext(ctx, cl, claim, &network) + + ipamClient, err := r.IPAM.ClientForProject(routing.project) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed building IPAM client: %w", err) + } + + iface, err := r.bindInterface(ctx, cl, ipamClient, routing, claim, &network) + if err != nil { + var failure *allocationFailure + if errors.As(err, &failure) { + return r.reject(ctx, cl, claim, string(failure.reason), failure.message) + } + var refused *bindingRefused + if errors.As(err, &refused) { + return r.reject(ctx, cl, claim, refused.reason, refused.message) + } + return ctrl.Result{}, err + } + + allocations, err := r.checkAllocations(ctx, ipamClient, recorder, routing, claim, iface) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.syncInterface(ctx, cl, iface, claim, &network, networkContextName); err != nil { + return ctrl.Result{}, err + } + + if err := r.publishInterfaceStatus(ctx, cl, iface, networkContextName, allocations); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, r.publishClaimStatus(ctx, cl, claim, iface, allocations) +} + +type projectRouting struct { + project string + projectNamespace string +} + +func (r *NetworkInterfaceClaimReconciler) resolveProject( + ctx context.Context, + cl client.Client, + namespaceName string, +) (projectRouting, error) { + var namespace corev1.Namespace + if err := cl.Get(ctx, client.ObjectKey{Name: namespaceName}, &namespace); err != nil { + return projectRouting{}, fmt.Errorf("failed reading namespace %q: %w", namespaceName, err) + } + + project, err := projectFromNamespace(&namespace) + if err != nil { + return projectRouting{}, &projectUnresolvable{message: err.Error()} + } + + projectNamespace, err := projectNamespaceFromNamespace(&namespace) + if err != nil { + return projectRouting{}, &projectUnresolvable{message: err.Error()} + } + + return projectRouting{project: project, projectNamespace: projectNamespace}, nil +} + +// projectUnresolvable means the namespace does not name a project. A failure to +// read the namespace is an ordinary error, and retrying fixes it. +type projectUnresolvable struct { + message string +} + +func (e *projectUnresolvable) Error() string { return e.message } + +// resolveNetworkContext records where the network lives. An unresolved context +// does not hold up allocation. +func (r *NetworkInterfaceClaimReconciler) resolveNetworkContext( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, + network *networkingv1alpha.Network, +) string { + logger := log.FromContext(ctx) + location := r.location() + + binding := &networkingv1alpha.NetworkBinding{} + binding.Namespace = claim.Namespace + binding.Name = fmt.Sprintf("%s-%s-%s", network.Name, location.Namespace, location.Name) + + _, err := controllerutil.CreateOrUpdate(ctx, cl, binding, func() error { + binding.Spec.Network = networkingv1alpha.NetworkRef{ + Namespace: network.Namespace, + Name: network.Name, + } + binding.Spec.Location = location + return controllerutil.SetControllerReference(network, binding, cl.Scheme()) + }) + if err != nil { + logger.Error(err, "failed ensuring network binding", "binding", binding.Name) + return "" + } + + if binding.Status.NetworkContextRef == nil { + return "" + } + return binding.Status.NetworkContextRef.Name +} + +func (r *NetworkInterfaceClaimReconciler) location() networkingv1alpha.LocationReference { + return networkingv1alpha.LocationReference{ + Name: r.Config.NetworkInterface.Location.Name, + Namespace: r.Config.NetworkInterface.Location.Namespace, + } +} + +type bindingRefused struct { + reason string + message string +} + +func (e *bindingRefused) Error() string { return e.message } + +func interfaceSatisfies( + iface *networkingv1alpha.NetworkInterface, + claim *networkingv1alpha.NetworkInterfaceClaim, +) error { + // The addresses belong to the network they were allocated from. Publishing + // them under another network would hand one network's addresses to another. + if iface.Spec.Network.Name != claim.Spec.Network.Name { + return &bindingRefused{ + reason: "NetworkMismatch", + message: fmt.Sprintf( + "Network interface %q holds addresses on network %q and cannot be bound by a claim naming network %q", + iface.Name, iface.Spec.Network.Name, claim.Spec.Network.Name), + } + } + + if iface.Spec.InterfaceName != claim.Spec.InterfaceName { + return &bindingRefused{ + reason: "InterfaceNameMismatch", + message: fmt.Sprintf( + "Network interface %q presents as %q to the guest and cannot be bound by a claim asking for %q", + iface.Name, iface.Spec.InterfaceName, claim.Spec.InterfaceName), + } + } + + for _, family := range claim.Spec.IPFamilies { + if !slices.ContainsFunc(iface.Spec.Addresses, func(a networkingv1alpha.NetworkInterfaceAddress) bool { + return a.Family == family + }) { + return &bindingRefused{ + reason: "AddressFamilyMissing", + message: fmt.Sprintf( + "Network interface %q holds no %s address, which this claim requires", + iface.Name, family), + } + } + } + + for _, request := range claim.Spec.Addresses { + if !slices.ContainsFunc(iface.Spec.ExternalAddresses, func(a networkingv1alpha.NetworkInterfaceExternalAddress) bool { + return a.Class == request.Class + }) { + return &bindingRefused{ + reason: "AddressClassMissing", + message: fmt.Sprintf( + "Network interface %q holds no address of class %q, which this claim requires", + iface.Name, request.Class), + } + } + } + + // An address keeps the reclaim policy it was allocated under, so a claim + // asking for a different one cannot be honoured. + if iface.Spec.ReclaimPolicy != claim.Spec.ReclaimPolicy { + return &bindingRefused{ + reason: "ReclaimPolicyMismatch", + message: fmt.Sprintf( + "Network interface %q holds addresses allocated with reclaimPolicy %s and cannot be bound by a claim requesting %s", + iface.Name, iface.Spec.ReclaimPolicy, claim.Spec.ReclaimPolicy), + } + } + + return nil +} + +func (r *NetworkInterfaceClaimReconciler) bindInterface( + ctx context.Context, + cl client.Client, + ipamClient client.Client, + routing projectRouting, + claim *networkingv1alpha.NetworkInterfaceClaim, + network *networkingv1alpha.Network, +) (*networkingv1alpha.NetworkInterface, error) { + interfaceKey := client.ObjectKey{Namespace: claim.Namespace, Name: interfaceNameForClaim(claim)} + + var existing networkingv1alpha.NetworkInterface + err := cl.Get(ctx, interfaceKey, &existing) + if err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed fetching network interface: %w", err) + } + + if err == nil { + if ref := existing.Spec.ClaimRef; ref != nil && ref.Name != claim.Name { + return nil, &bindingRefused{ + reason: "InterfaceHeldByAnotherClaim", + message: fmt.Sprintf( + "Network interface %q is held by claim %q", existing.Name, ref.Name), + } + } + + if err := interfaceSatisfies(&existing, claim); err != nil { + return nil, err + } + + existing.Spec.ClaimRef = &networkingv1alpha.NetworkInterfaceClaimRef{Name: claim.Name} + controllerutil.AddFinalizer(&existing, networkInterfaceFinalizer) + if err := cl.Update(ctx, &existing); err != nil { + return nil, fmt.Errorf("failed binding network interface: %w", err) + } + return &existing, nil + } + + requests, err := r.allocationRequests(ctx, ipamClient, claim) + if err != nil { + return nil, err + } + + allocated, err := r.allocate(ctx, ipamClient, routing, claim, network, requests) + if err != nil { + return nil, err + } + + iface := &networkingv1alpha.NetworkInterface{} + iface.Namespace = interfaceKey.Namespace + iface.Name = interfaceKey.Name + iface.Annotations = map[string]string{allocationClaimAnnotation: claim.Name} + iface.Finalizers = []string{networkInterfaceFinalizer} + iface.Spec = networkingv1alpha.NetworkInterfaceSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: network.Name}, + ClaimRef: &networkingv1alpha.NetworkInterfaceClaimRef{Name: claim.Name}, + InterfaceName: claim.Spec.InterfaceName, + MTU: network.Spec.MTU, + ReclaimPolicy: claim.Spec.ReclaimPolicy, + } + + for _, entry := range allocated { + if entry.request.external { + iface.Spec.ExternalAddresses = append(iface.Spec.ExternalAddresses, networkingv1alpha.NetworkInterfaceExternalAddress{ + Family: entry.request.family, + Address: entry.bareAddress(), + Class: entry.request.className, + }) + continue + } + + iface.Spec.Addresses = append(iface.Spec.Addresses, networkingv1alpha.NetworkInterfaceAddress{ + Family: entry.request.family, + Address: entry.cidr, + Primary: entry.request.family == claim.Spec.IPFamilies[0], + Class: entry.request.className, + }) + } + + if err := cl.Create(ctx, iface); err != nil { + return nil, fmt.Errorf("failed creating network interface: %w", err) + } + + return iface, nil +} + +type allocationRequest struct { + discriminator string + family networkingv1alpha.IPFamily + className string + external bool +} + +func (a allocationRequest) describe() string { + if a.className != "" { + return fmt.Sprintf("an address of class %q", a.className) + } + return fmt.Sprintf("an %s address", a.family) +} + +type allocatedAddress struct { + request allocationRequest + cidr string +} + +func (a allocatedAddress) bareAddress() string { + prefix, err := netip.ParsePrefix(a.cidr) + if err != nil { + return a.cidr + } + if prefix.Bits() != prefix.Addr().BitLen() { + return a.cidr + } + return prefix.Addr().String() +} + +func (r *NetworkInterfaceClaimReconciler) allocationRequests( + ctx context.Context, + ipamClient client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, +) ([]allocationRequest, error) { + requests := make([]allocationRequest, 0, len(claim.Spec.IPFamilies)+len(claim.Spec.Addresses)) + + for _, family := range claim.Spec.IPFamilies { + requests = append(requests, allocationRequest{ + discriminator: familyDiscriminator(family), + family: family, + }) + } + + for _, address := range claim.Spec.Addresses { + var class ipamv1alpha1.IPClass + if err := ipamClient.Get(ctx, client.ObjectKey{Name: address.Class}, &class); err != nil { + return nil, &allocationFailure{ + reason: allocationFailureRejected, + message: fmt.Sprintf("Address class %q could not be read: %v", + address.Class, err), + } + } + + family := networkingv1alpha.IPFamily(class.Spec.IPFamily) + if !slices.Contains(claim.Spec.IPFamilies, family) { + return nil, &allocationFailure{ + reason: allocationFailureRejected, + message: fmt.Sprintf( + "Address class %q hands out %s addresses, which map onto an %s address this interface does not carry", + address.Class, family, family), + } + } + + requests = append(requests, allocationRequest{ + discriminator: classDiscriminator(address.Class), + family: family, + className: address.Class, + external: true, + }) + } + + return requests, nil +} + +type allocationFailure struct { + reason allocationFailureReason + message string +} + +func (e *allocationFailure) Error() string { return e.message } + +// allocate claims every requested address, or none. Names are derived from the +// claim, so a failed rollback leaves addresses the next attempt finds again. +func (r *NetworkInterfaceClaimReconciler) allocate( + ctx context.Context, + ipamClient client.Client, + routing projectRouting, + claim *networkingv1alpha.NetworkInterfaceClaim, + network *networkingv1alpha.Network, + requests []allocationRequest, +) ([]allocatedAddress, error) { + logger := log.FromContext(ctx) + + if err := ensureProjectNamespace(ctx, ipamClient, routing.projectNamespace); err != nil { + return nil, fmt.Errorf("failed ensuring project namespace %q: %w", routing.projectNamespace, err) + } + + location := r.location() + allocated := make([]allocatedAddress, 0, len(requests)) + created := make([]*ipamv1alpha1.IPClaim, 0, len(requests)) + + rollback := func() { + var leaked []string + for _, ipClaim := range created { + if err := ipamClient.Delete(ctx, ipClaim); err != nil && !apierrors.IsNotFound(err) { + leaked = append(leaked, ipClaim.Name) + logger.Error(err, "failed releasing address after a partial allocation", + "ipclaim", ipClaim.Name, "project", routing.project) + } + } + if len(leaked) > 0 { + logger.Info("addresses remain claimed after a failed rollback and will be reused on retry", + "ipclaims", strings.Join(leaked, ","), "project", routing.project) + } + } + + for _, request := range requests { + ipClaim := &ipamv1alpha1.IPClaim{} + ipClaim.Namespace = routing.projectNamespace + ipClaim.Name = ipClaimName(claim.Name, request.discriminator) + ipClaim.Spec = ipamv1alpha1.IPClaimSpec{ + ClassName: request.className, + ReclaimPolicy: ipamReclaimPolicy(claim.Spec.ReclaimPolicy), + Scope: map[string]ipamv1alpha1.ScopeRef{ + ipamScopeRoleNetwork: { + APIGroup: datumNetworkingAPIGroup, + Kind: "Network", + Name: network.Name, + }, + ipamScopeRoleLocation: { + APIGroup: datumNetworkingAPIGroup, + Kind: "Location", + Name: location.Name, + }, + }, + } + if request.className == "" { + ipClaim.Spec.IPFamily = ipamv1alpha1.IPFamily(request.family) + } + + // IPAM reports a duplicate name inconsistently, so ask whether the + // address exists instead of reading the refusal. + existing := &ipamv1alpha1.IPClaim{} + getErr := ipamClient.Get(ctx, client.ObjectKeyFromObject(ipClaim), existing) + if getErr != nil && !apierrors.IsNotFound(getErr) { + rollback() + return nil, fmt.Errorf("failed reading IPClaim %q: %w", ipClaim.Name, getErr) + } + + if getErr == nil { + ipClaim = existing + } else if createErr := ipamClient.Create(ctx, ipClaim); createErr != nil { + raced := &ipamv1alpha1.IPClaim{} + if err := ipamClient.Get(ctx, client.ObjectKeyFromObject(ipClaim), raced); err != nil { + rollback() + reason := classifyAllocationFailure(createErr) + return nil, &allocationFailure{ + reason: reason, + message: allocationFailureMessage(reason, request, createErr), + } + } + ipClaim = raced + } else { + created = append(created, ipClaim) + } + + if ipClaim.Status.AllocatedCIDR == "" { + rollback() + return nil, &allocationFailure{ + reason: allocationFailureUnknown, + message: fmt.Sprintf("IPAM reported no address for %s (phase %q)", + request.describe(), ipClaim.Status.Phase), + } + } + + allocated = append(allocated, allocatedAddress{ + request: request, + cidr: ipClaim.Status.AllocatedCIDR, + }) + } + + return allocated, nil +} + +func (r *NetworkInterfaceClaimReconciler) checkAllocations( + ctx context.Context, + ipamClient client.Client, + recorder events.EventRecorder, + routing projectRouting, + claim *networkingv1alpha.NetworkInterfaceClaim, + iface *networkingv1alpha.NetworkInterface, +) (allocationHealth, error) { + owner := allocationClaimName(iface) + health := allocationHealth{intact: true} + + var errs []error + for _, entry := range allocationEntries(iface) { + var ipClaim ipamv1alpha1.IPClaim + key := client.ObjectKey{ + Namespace: routing.projectNamespace, + Name: ipClaimName(owner, entry.discriminator), + } + if err := ipamClient.Get(ctx, key, &ipClaim); err != nil { + if apierrors.IsNotFound(err) { + health = allocationHealth{ + message: fmt.Sprintf("Address %s is held by no allocation in project %q", + entry.address, routing.project), + } + r.reportMissingAllocation(ctx, recorder, routing, claim, iface, entry, key.Name, "") + continue + } + errs = append(errs, fmt.Errorf("failed reading IPClaim %q: %w", key.Name, err)) + continue + } + + if !entry.holds(&ipClaim) { + health = allocationHealth{ + message: fmt.Sprintf("Address %s is published, but allocation %q holds %s", + entry.address, key.Name, ipClaim.Status.AllocatedCIDR), + } + r.reportMissingAllocation(ctx, recorder, routing, claim, iface, entry, key.Name, + ipClaim.Status.AllocatedCIDR) + } + } + + return health, errors.Join(errs...) +} + +// allocationClaimName is the claim whose name the addresses were allocated +// under, which is not always the claim holding the interface now. +func allocationClaimName(iface *networkingv1alpha.NetworkInterface) string { + if recorded := iface.Annotations[allocationClaimAnnotation]; recorded != "" { + return recorded + } + if ref := iface.Spec.ClaimRef; ref != nil { + return ref.Name + } + return iface.Name +} + +// reportMissingAllocation warns that an address may be handed to another claim. +// Reallocating instead would change the address of a running workload. +func (r *NetworkInterfaceClaimReconciler) reportMissingAllocation( + ctx context.Context, + recorder events.EventRecorder, + routing projectRouting, + claim *networkingv1alpha.NetworkInterfaceClaim, + iface *networkingv1alpha.NetworkInterface, + entry allocationEntry, + ipClaimName string, + heldInstead string, +) { + missingAllocationsTotal.WithLabelValues(routing.project).Inc() + + log.FromContext(ctx).Error(errNoAllocationBehindAddress, + "network interface advertises an address with no allocation behind it", + "interface", iface.Name, "address", entry.address, "heldInstead", heldInstead, + "ipclaim", ipClaimName, "project", routing.project) + + if recorder == nil { + return + } + + if heldInstead != "" { + recorder.Eventf(claim, iface, corev1.EventTypeWarning, "AddressAllocationMissing", "VerifyAllocation", + "Address %s on network interface %q is published, but allocation %q in project %q holds %s. "+ + "The published address belongs to no one and may be given to another claim.", + entry.address, iface.Name, ipClaimName, routing.project, heldInstead) + return + } + + recorder.Eventf(claim, iface, corev1.EventTypeWarning, "AddressAllocationMissing", "VerifyAllocation", + "Address %s on network interface %q has no allocation in project %q (IPClaim %q is gone). "+ + "IPAM considers the address free and may hand it to another claim.", + entry.address, iface.Name, routing.project, ipClaimName) +} + +var errNoAllocationBehindAddress = errors.New("no IPClaim holds this address") + +type allocationEntry struct { + discriminator string + address string + external bool +} + +// holds reports whether an allocation still carries the address the interface +// publishes. An IPClaim recreated under the same name may hold a different one. +func (e allocationEntry) holds(ipClaim *ipamv1alpha1.IPClaim) bool { + allocated := allocatedAddress{cidr: ipClaim.Status.AllocatedCIDR} + if e.external { + return allocated.bareAddress() == e.address + } + return ipClaim.Status.AllocatedCIDR == e.address +} + +func allocationEntries(iface *networkingv1alpha.NetworkInterface) []allocationEntry { + entries := make([]allocationEntry, 0, len(iface.Spec.Addresses)+len(iface.Spec.ExternalAddresses)) + for _, address := range iface.Spec.Addresses { + entries = append(entries, allocationEntry{ + discriminator: familyDiscriminator(address.Family), + address: address.Address, + }) + } + for _, address := range iface.Spec.ExternalAddresses { + entries = append(entries, allocationEntry{ + discriminator: classDiscriminator(address.Class), + address: address.Address, + external: true, + }) + } + return entries +} + +func allocationDiscriminators(iface *networkingv1alpha.NetworkInterface) []string { + entries := allocationEntries(iface) + discriminators := make([]string, 0, len(entries)) + for _, entry := range entries { + discriminators = append(discriminators, entry.discriminator) + } + return discriminators +} + +func (r *NetworkInterfaceClaimReconciler) publishInterfaceStatus( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, + networkContextName string, + allocations allocationHealth, +) error { + iface.Status.Phase = networkingv1alpha.NetworkInterfacePhaseBound + if networkContextName != "" { + iface.Status.NetworkContextRef = &networkingv1alpha.LocalNetworkContextRef{Name: networkContextName} + } + + apimeta.SetStatusCondition(&iface.Status.Conditions, allocatedCondition( + networkingv1alpha.NetworkInterfaceAllocated, iface.Generation, allocations)) + + if apimeta.FindStatusCondition(iface.Status.Conditions, networkingv1alpha.NetworkInterfaceProgrammed) == nil { + apimeta.SetStatusCondition(&iface.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceProgrammed, + Status: metav1.ConditionUnknown, + Reason: "Pending", + ObservedGeneration: iface.Generation, + Message: "Waiting for the data plane to report the attachment", + }) + } + + if err := cl.Status().Update(ctx, iface); err != nil { + return fmt.Errorf("failed updating network interface status: %w", err) + } + return nil +} + +func (r *NetworkInterfaceClaimReconciler) publishClaimStatus( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, + iface *networkingv1alpha.NetworkInterface, + allocations allocationHealth, +) error { + claim.Status.Addresses = append([]networkingv1alpha.NetworkInterfaceAddress(nil), iface.Spec.Addresses...) + claim.Status.NetworkInterfaceRef = &networkingv1alpha.LocalNetworkInterfaceRef{Name: iface.Name} + claim.Status.ExternalAddresses = append([]networkingv1alpha.NetworkInterfaceExternalAddress(nil), iface.Spec.ExternalAddresses...) + + apimeta.SetStatusCondition(&claim.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimBound, + Status: metav1.ConditionTrue, + Reason: "Bound", + ObservedGeneration: claim.Generation, + Message: fmt.Sprintf("Bound to network interface %q", iface.Name), + }) + apimeta.SetStatusCondition(&claim.Status.Conditions, allocatedCondition( + networkingv1alpha.NetworkInterfaceClaimAllocated, claim.Generation, allocations)) + seedProgrammed(&claim.Status.Conditions, claim.Generation) + setReady(&claim.Status.Conditions, claim.Generation) + + if err := cl.Status().Update(ctx, claim); err != nil { + return fmt.Errorf("failed updating claim status: %w", err) + } + return nil +} + +// syncGateways writes each address's gateway onto the interface once the +// location has a subnet. A provider configures a NIC from the interface alone, +// so the gateway has to live there and not only on the claim's status copy. +func (r *NetworkInterfaceClaimReconciler) syncInterface( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, + claim *networkingv1alpha.NetworkInterfaceClaim, + network *networkingv1alpha.Network, + networkContextName string, +) error { + changed := false + + // MTU follows the network, and the primary address follows the claim's + // first family. Both are derived, and an adopted interface carries whatever + // its previous claim left behind. + if iface.Spec.MTU != network.Spec.MTU { + iface.Spec.MTU = network.Spec.MTU + changed = true + } + for i := range iface.Spec.Addresses { + primary := iface.Spec.Addresses[i].Family == claim.Spec.IPFamilies[0] + if iface.Spec.Addresses[i].Primary != primary { + iface.Spec.Addresses[i].Primary = primary + changed = true + } + } + + if err := r.applyGateways(ctx, cl, iface, networkContextName, &changed); err != nil { + return err + } + + if !changed { + return nil + } + + if err := cl.Update(ctx, iface); err != nil { + return fmt.Errorf("failed updating network interface: %w", err) + } + return nil +} + +func (r *NetworkInterfaceClaimReconciler) applyGateways( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, + networkContextName string, + changed *bool, +) error { + if networkContextName == "" { + return nil + } + + var subnets networkingv1alpha.SubnetList + if err := cl.List(ctx, &subnets, client.InNamespace(iface.Namespace)); err != nil { + return fmt.Errorf("failed listing subnets: %w", err) + } + + for i := range iface.Spec.Addresses { + gateway := subnetGatewayFor(&subnets, networkContextName, iface.Spec.Addresses[i].Family) + if gateway == "" || iface.Spec.Addresses[i].Gateway == gateway { + continue + } + iface.Spec.Addresses[i].Gateway = gateway + *changed = true + } + return nil +} + +func subnetGatewayFor( + subnets *networkingv1alpha.SubnetList, + networkContextName string, + family networkingv1alpha.IPFamily, +) string { + for i := range subnets.Items { + subnet := &subnets.Items[i] + if subnet.Spec.NetworkContext.Name == networkContextName && subnet.Spec.IPFamily == family { + return subnetGateway(subnet) + } + } + return "" +} + +func subnetGateway(subnet *networkingv1alpha.Subnet) string { + start := subnet.Spec.StartAddress + if subnet.Status.StartAddress != nil { + start = *subnet.Status.StartAddress + } + + addr, err := netip.ParseAddr(start) + if err != nil { + return "" + } + return addr.Next().String() +} + +func (r *NetworkInterfaceClaimReconciler) reject( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, + reason string, + message string, +) (ctrl.Result, error) { + log.FromContext(ctx).Info("claim cannot be fulfilled", "reason", reason, "message", message) + + demoted := []string{networkingv1alpha.NetworkInterfaceClaimReady} + + // A claim that already holds an interface still holds it, and still holds + // the addresses IPAM allocated. Only Ready is false. + if claim.Status.NetworkInterfaceRef == nil { + demoted = append(demoted, + networkingv1alpha.NetworkInterfaceClaimBound, + networkingv1alpha.NetworkInterfaceClaimAllocated) + } + + for _, conditionType := range demoted { + apimeta.SetStatusCondition(&claim.Status.Conditions, metav1.Condition{ + Type: conditionType, + Status: metav1.ConditionFalse, + Reason: reason, + ObservedGeneration: claim.Generation, + Message: message, + }) + } + seedProgrammed(&claim.Status.Conditions, claim.Generation) + + if err := cl.Status().Update(ctx, claim); err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating claim status: %w", err) + } + + // Nothing watches the network, the namespace or IPAM, so a claim rejected + // for a condition that later clears has no other way back. + return ctrl.Result{RequeueAfter: rejectedClaimRetryInterval}, nil +} + +// allocationHealth reports whether every address the interface publishes is +// still backed by an allocation. +type allocationHealth struct { + intact bool + message string +} + +func allocatedCondition(conditionType string, generation int64, health allocationHealth) metav1.Condition { + if health.intact { + return metav1.Condition{ + Type: conditionType, + Status: metav1.ConditionTrue, + Reason: "Allocated", + ObservedGeneration: generation, + Message: "Every requested address is held", + } + } + return metav1.Condition{ + Type: conditionType, + Status: metav1.ConditionFalse, + Reason: "AddressAllocationMissing", + ObservedGeneration: generation, + Message: health.message, + } +} + +// setReady derives Ready from the conditions it depends on, so it becomes true +// on its own once the data plane reports the attachment. +func setReady(conditions *[]metav1.Condition, generation int64) { + unmet := "" + for _, conditionType := range []string{ + networkingv1alpha.NetworkInterfaceClaimBound, + networkingv1alpha.NetworkInterfaceClaimAllocated, + networkingv1alpha.NetworkInterfaceClaimProgrammed, + } { + if !apimeta.IsStatusConditionTrue(*conditions, conditionType) { + unmet = conditionType + break + } + } + + if unmet == "" { + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimReady, + Status: metav1.ConditionTrue, + Reason: "Ready", + ObservedGeneration: generation, + Message: "The interface is bound, addressed, and programmed", + }) + return + } + + status := metav1.ConditionFalse + if apimeta.IsStatusConditionPresentAndEqual(*conditions, unmet, metav1.ConditionUnknown) { + status = metav1.ConditionUnknown + } + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimReady, + Status: status, + Reason: "Not" + unmet, + ObservedGeneration: generation, + Message: fmt.Sprintf("Waiting for %s", unmet), + }) +} + +// seedProgrammed sets Programmed only when it is absent. The data plane owns +// this condition; overwriting it would revert whoever reported the attachment. +func seedProgrammed(conditions *[]metav1.Condition, generation int64) { + if apimeta.FindStatusCondition(*conditions, networkingv1alpha.NetworkInterfaceClaimProgrammed) != nil { + return + } + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimProgrammed, + Status: metav1.ConditionUnknown, + Reason: "Pending", + ObservedGeneration: generation, + Message: "Waiting for the data plane to report the attachment", + }) +} + +// release returns the addresses to their pools, or under Retain leaves the +// interface holding them for the next claim of this name. +func (r *NetworkInterfaceClaimReconciler) release( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, +) error { + if !controllerutil.ContainsFinalizer(claim, networkInterfaceClaimFinalizer) { + return nil + } + + var iface networkingv1alpha.NetworkInterface + interfaceKey := client.ObjectKey{Namespace: claim.Namespace, Name: interfaceNameForClaim(claim)} + err := cl.Get(ctx, interfaceKey, &iface) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed fetching network interface: %w", err) + } + + interfaceExists := err == nil + held := interfaceExists && iface.Spec.ClaimRef != nil && + iface.Spec.ClaimRef.Name == claim.Name + retain := claim.Spec.ReclaimPolicy == networkingv1alpha.NetworkInterfaceReclaimPolicyRetain + + switch { + case held && retain: + iface.Spec.ClaimRef = nil + if err := cl.Update(ctx, &iface); err != nil { + return fmt.Errorf("failed unbinding network interface: %w", err) + } + if err := markInterfaceAvailable(ctx, cl, &iface); err != nil { + return err + } + + case held: + if err := r.releaseAddresses(ctx, cl, claim.Namespace, &iface); err != nil { + return err + } + if controllerutil.RemoveFinalizer(&iface, networkInterfaceFinalizer) { + if err := cl.Update(ctx, &iface); err != nil { + return fmt.Errorf("failed clearing network interface finalizer: %w", err) + } + } + if err := cl.Delete(ctx, &iface); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed deleting network interface: %w", err) + } + + case interfaceExists && iface.Spec.ClaimRef == nil && retain: + // A previous attempt unbound the interface and failed before recording + // it. Nothing else writes phase for an unbound interface. + if err := markInterfaceAvailable(ctx, cl, &iface); err != nil { + return err + } + + case !retain: + // The interface is gone, or never named this claim. Either way the + // addresses this claim allocated are named after it. + if err := r.releaseClaimAllocations(ctx, cl, claim); err != nil { + return err + } + } + + controllerutil.RemoveFinalizer(claim, networkInterfaceClaimFinalizer) + if err := cl.Update(ctx, claim); err != nil { + return fmt.Errorf("failed removing finalizer: %w", err) + } + return nil +} + +func (r *NetworkInterfaceClaimReconciler) releaseAddresses( + ctx context.Context, + cl client.Client, + namespace string, + iface *networkingv1alpha.NetworkInterface, +) error { + return r.releaseIPClaims(ctx, cl, namespace, + allocationClaimName(iface), allocationDiscriminators(iface)) +} + +// releaseClaimAllocations releases the addresses this claim allocated. A claim +// that only adopted an existing interface allocated none and releases none. +func (r *NetworkInterfaceClaimReconciler) releaseClaimAllocations( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, +) error { + // A claim that never bound allocated nothing, so there is nothing to + // release and no reason to block its deletion on a project it never named. + if claim.Status.NetworkInterfaceRef == nil { + return nil + } + + return r.releaseIPClaims(ctx, cl, claim.Namespace, + claim.Name, claimDiscriminators(claim)) +} + +func markInterfaceAvailable( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, +) error { + if iface.Status.Phase == networkingv1alpha.NetworkInterfacePhaseAvailable { + return nil + } + iface.Status.Phase = networkingv1alpha.NetworkInterfacePhaseAvailable + if err := cl.Status().Update(ctx, iface); err != nil { + return fmt.Errorf("failed updating network interface status: %w", err) + } + return nil +} + +func (r *NetworkInterfaceClaimReconciler) releaseIPClaims( + ctx context.Context, + cl client.Client, + namespace string, + owner string, + discriminators []string, +) error { + if len(discriminators) == 0 { + return nil + } + + routing, err := r.resolveProject(ctx, cl, namespace) + if err != nil { + return fmt.Errorf("cannot release addresses claimed by %q: %w", owner, err) + } + + ipamClient, err := r.IPAM.ClientForProject(routing.project) + if err != nil { + return fmt.Errorf("failed building IPAM client: %w", err) + } + + var errs []error + for _, discriminator := range discriminators { + ipClaim := &ipamv1alpha1.IPClaim{} + ipClaim.Namespace = routing.projectNamespace + ipClaim.Name = ipClaimName(owner, discriminator) + if err := ipamClient.Delete(ctx, ipClaim); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("failed releasing %q: %w", ipClaim.Name, err)) + } + } + return errors.Join(errs...) +} + +func claimDiscriminators(claim *networkingv1alpha.NetworkInterfaceClaim) []string { + discriminators := make([]string, 0, len(claim.Spec.IPFamilies)+len(claim.Spec.Addresses)) + for _, family := range claim.Spec.IPFamilies { + discriminators = append(discriminators, familyDiscriminator(family)) + } + for _, address := range claim.Spec.Addresses { + discriminators = append(discriminators, classDiscriminator(address.Class)) + } + return discriminators +} + +func interfaceNameForClaim(claim *networkingv1alpha.NetworkInterfaceClaim) string { + if claim.Spec.NetworkInterfaceName != "" { + return claim.Spec.NetworkInterfaceName + } + return claim.Name +} + +func familyDiscriminator(family networkingv1alpha.IPFamily) string { + return "f-" + strings.ToLower(string(family)) +} + +func classDiscriminator(class string) string { + return "c-" + class +} + +// ipClaimName derives a stable name from the claim and what it asks for, so a +// replacement instance finds the addresses it already has. +func ipClaimName(claimName, discriminator string) string { + candidate := claimName + "-" + discriminator + if len(candidate) <= maxObjectNameLength && len(validation.IsDNS1123Subdomain(candidate)) == 0 { + return candidate + } + + sum := sha256.Sum256([]byte(claimName + "\x00" + discriminator)) + suffix := hex.EncodeToString(sum[:])[:ipClaimNameHashLen] + + prefix := claimName + if max := maxObjectNameLength - 1 - ipClaimNameHashLen; len(prefix) > max { + prefix = prefix[:max] + } + return strings.TrimRight(prefix, "-.") + "-" + suffix +} + +func ipamReclaimPolicy(policy networkingv1alpha.NetworkInterfaceReclaimPolicy) ipamv1alpha1.ReclaimPolicy { + if policy == networkingv1alpha.NetworkInterfaceReclaimPolicyRetain { + return ipamv1alpha1.ReclaimRetain + } + return ipamv1alpha1.ReclaimDelete +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NetworkInterfaceClaimReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.IPAM == nil { + return errors.New("an IPAM client factory is required") + } + if r.Config.NetworkInterface.Location.Name == "" || r.Config.NetworkInterface.Location.Namespace == "" { + return errors.New("a location is required to fulfil network interface claims") + } + + r.mgr = mgr + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkInterfaceClaim{}, mcbuilder.WithEngageWithLocalCluster(false)). + // A deleted interface is rebuilt by the claim that holds it. + Watches(&networkingv1alpha.NetworkInterface{}, mchandler.EnqueueRequestsFromMapFunc( + func(_ context.Context, obj client.Object) []ctrl.Request { + iface, ok := obj.(*networkingv1alpha.NetworkInterface) + if !ok || iface.Spec.ClaimRef == nil { + return nil + } + return []ctrl.Request{{NamespacedName: client.ObjectKey{ + Namespace: iface.Namespace, + Name: iface.Spec.ClaimRef.Name, + }}} + })). + // A subnet appearing is what makes the gateway resolvable, and no claim + // event follows it. + Watches(&networkingv1alpha.Subnet{}, func(clusterName multicluster.ClusterName, cl cluster.Cluster) mchandler.EventHandler { + return mchandler.ForCluster(handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, obj client.Object) []reconcile.Request { + var claims networkingv1alpha.NetworkInterfaceClaimList + if err := cl.GetClient().List(ctx, &claims, client.InNamespace(obj.GetNamespace())); err != nil { + log.FromContext(ctx).Error(err, "failed listing claims for a subnet event") + return nil + } + + requests := make([]reconcile.Request, 0, len(claims.Items)) + for i := range claims.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&claims.Items[i]), + }) + } + return requests + }), clusterName) + }). + Named("networkinterfaceclaim"). + Complete(r) +} diff --git a/internal/controller/networkinterfaceclaim_controller_test.go b/internal/controller/networkinterfaceclaim_controller_test.go new file mode 100644 index 00000000..e62f1dfe --- /dev/null +++ b/internal/controller/networkinterfaceclaim_controller_test.go @@ -0,0 +1,1301 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + eventsv1client "k8s.io/client-go/kubernetes/typed/events/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/yaml" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" + "go.datum.net/network-services-operator/internal/downstreamclient" +) + +const ( + testProject = "project-alpha" + testProjectNS = "default" + testLocationName = "us-central-1" + testLocationNS = "datum-locations" + testPublicV4Class = "datum-public-v4" +) + +// fakeIPAM stands in for the IPAM API server. Allocation is synchronous there, +// so the create response already carries the address. +type fakeIPAM struct { + mu sync.Mutex + + scheme *runtime.Scheme + clients map[string]client.Client + + classes []*ipamv1alpha1.IPClass + + // createdIn records, per project, the IPClaim names Create was called with. + createdIn map[string][]string + // deletedIn records, per project, the IPClaim names Delete was called with. + deletedIn map[string][]string + // allocationPolicy is the policy IPAM froze onto the allocation. The server + // releases on this value alone and never re-reads the IPClaim. + allocationPolicy map[string]ipamv1alpha1.ReclaimPolicy + // orphans maps an allocation to the CIDR it holds after its IPClaim was + // deleted under a frozen Retain. + orphans map[string]string + + // failOn refuses the named IPClaim with the given error. + failOn map[string]error + + nextV4 int + nextV6 int +} + +func newFakeIPAM(t *testing.T, classes ...*ipamv1alpha1.IPClass) *fakeIPAM { + t.Helper() + scheme, err := IPAMScheme() + require.NoError(t, err) + + return &fakeIPAM{ + scheme: scheme, + clients: map[string]client.Client{}, + classes: classes, + createdIn: map[string][]string{}, + deletedIn: map[string][]string{}, + allocationPolicy: map[string]ipamv1alpha1.ReclaimPolicy{}, + orphans: map[string]string{}, + failOn: map[string]error{}, + } +} + +func (f *fakeIPAM) ClientForProject(project string) (client.Client, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if existing, ok := f.clients[project]; ok { + return existing, nil + } + + builder := fakeclient.NewClientBuilder().WithScheme(f.scheme) + for _, class := range f.classes { + builder = builder.WithObjects(class.DeepCopy()) + } + + cl := builder.WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + ipClaim, ok := obj.(*ipamv1alpha1.IPClaim) + if !ok { + return c.Create(ctx, obj, opts...) + } + + f.mu.Lock() + f.createdIn[project] = append(f.createdIn[project], ipClaim.Name) + failure := f.failOn[ipClaim.Name] + if failure == nil { + failure = f.retainedConflictLocked(ipClaim) + } + if failure == nil { + f.allocateLocked(project, ipClaim) + } + f.mu.Unlock() + + if failure != nil { + return failure + } + + if err := c.Create(ctx, obj, opts...); err != nil { + // IPAM lets this collision escape as a raw 500, which no status + // code tells apart from a genuine internal error. + if apierrors.IsAlreadyExists(err) { + return apierrors.NewInternalError(fmt.Errorf( + `insert allocation: ERROR: duplicate key value violates unique constraint "ipam_cidr_allocations_claim_key_key" (SQLSTATE 23505)`)) + } + return err + } + return nil + }, + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if ipClaim, ok := obj.(*ipamv1alpha1.IPClaim); ok { + var stored ipamv1alpha1.IPClaim + _ = c.Get(ctx, client.ObjectKeyFromObject(ipClaim), &stored) + + f.mu.Lock() + f.deletedIn[project] = append(f.deletedIn[project], ipClaim.Name) + if f.allocationPolicy[ipClaim.Name] == ipamv1alpha1.ReclaimRetain { + f.orphans[allocationNameFor(ipClaim.Name)] = stored.Status.AllocatedCIDR + } + f.mu.Unlock() + } + return c.Delete(ctx, obj, opts...) + }, + }).Build() + + f.clients[project] = cl + return cl, nil +} + +func allocationNameFor(ipClaimName string) string { + return "alloc-" + ipClaimName +} + +// retainedConflictLocked models the other duplicate, which IPAM maps to a 409. +// An allocation left behind by a deleted claim blocks the name it used. +func (f *fakeIPAM) retainedConflictLocked(ipClaim *ipamv1alpha1.IPClaim) error { + allocationName := allocationNameFor(ipClaim.Name) + if _, orphaned := f.orphans[allocationName]; !orphaned { + return nil + } + return apierrors.NewConflict( + ipamv1alpha1.SchemeGroupVersion.WithResource("ipclaims").GroupResource(), + ipClaim.Name, + fmt.Errorf("an allocation under this identity already exists: IPAllocation %q, retained by an earlier claim of the same name; delete it to reuse the name", allocationName), + ) +} + +func (f *fakeIPAM) allocateLocked(project string, ipClaim *ipamv1alpha1.IPClaim) { + family := ipClaim.Spec.IPFamily + if ipClaim.Spec.ClassName != "" { + for _, class := range f.classes { + if class.Name == ipClaim.Spec.ClassName { + family = class.Spec.IPFamily + } + } + } + + // Frozen once, here. Later edits to the IPClaim never reach it. + f.allocationPolicy[ipClaim.Name] = ipClaim.Spec.ReclaimPolicy + + // The real server writes only allocatedCIDR, host prefixes included, and + // never status.address. + ipClaim.Status.Phase = ipamv1alpha1.ClaimBound + if family == ipamv1alpha1.IPv6 { + f.nextV6++ + ipClaim.Status.AllocatedCIDR = fmt.Sprintf("2001:db8:a000:%d::/96", f.nextV6) + } else { + f.nextV4++ + ipClaim.Status.AllocatedCIDR = fmt.Sprintf("10.128.0.%d/32", f.nextV4) + } + ipClaim.Status.PoolRef = &ipamv1alpha1.LocalRef{Name: "pool-" + project} +} + +// created and deleted are keyed by project, so a test can assert which project +// an allocation reached. +func (f *fakeIPAM) created() map[string][]string { + f.mu.Lock() + defer f.mu.Unlock() + return maps.Clone(f.createdIn) +} + +func (f *fakeIPAM) deleted() map[string][]string { + f.mu.Lock() + defer f.mu.Unlock() + return maps.Clone(f.deletedIn) +} + +func (f *fakeIPAM) createdAnywhere() int { + f.mu.Lock() + defer f.mu.Unlock() + total := 0 + for _, names := range f.createdIn { + total += len(names) + } + return total +} + +// orphanedAllocations reports allocations no IPClaim references. +func (f *fakeIPAM) orphanedAllocations() map[string]string { + f.mu.Lock() + defer f.mu.Unlock() + return maps.Clone(f.orphans) +} + +func (f *fakeIPAM) refuse(name string, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.failOn[name] = err +} + +func publicV4Class() *ipamv1alpha1.IPClass { + class := &ipamv1alpha1.IPClass{} + class.Name = testPublicV4Class + class.Spec.IPFamily = ipamv1alpha1.IPv4 + return class +} + +func startNetworkInterfaceEnv(t *testing.T) (client.Client, *rest.Config) { + t.Helper() + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("KUBEBUILDER_ASSETS unset; run via `make test` to exercise envtest") + } + + testScheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(testScheme)) + require.NoError(t, networkingv1alpha.AddToScheme(testScheme)) + + env := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + cfg, err := env.Start() + require.NoError(t, err) + t.Cleanup(func() { _ = env.Stop() }) + + cl, err := client.New(cfg, client.Options{Scheme: testScheme}) + require.NoError(t, err) + return cl, cfg +} + +type scenario struct { + restConfig *rest.Config + events *events.FakeRecorder + t *testing.T + ctx context.Context + client client.Client + ipam *fakeIPAM + reconciler *NetworkInterfaceClaimReconciler + namespace string +} + +// newScenario builds one namespace of objects. Set labelled to false for a +// namespace that names no project. +func newScenario(t *testing.T, labelled bool, networkFamilies []networkingv1alpha.IPFamily, classes ...*ipamv1alpha1.IPClass) *scenario { + t.Helper() + cl, restConfig := startNetworkInterfaceEnv(t) + ctx := context.Background() + + namespaceName := "ns-" + sanitizeName(strings.ToLower(t.Name())) + + namespace := &corev1.Namespace{} + namespace.Name = namespaceName + namespace.Labels = map[string]string{ + downstreamclient.UpstreamOwnerNamespaceLabel: testProjectNS, + } + if labelled { + namespace.Labels[downstreamclient.UpstreamOwnerClusterNameLabel] = "cluster-" + testProject + } + require.NoError(t, cl.Create(ctx, namespace)) + + network := &networkingv1alpha.Network{} + network.Namespace = namespaceName + network.Name = "default" + network.Spec = networkingv1alpha.NetworkSpec{ + IPAM: networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto}, + IPFamilies: networkFamilies, + MTU: 1460, + } + require.NoError(t, cl.Create(ctx, network)) + + ipam := newFakeIPAM(t, classes...) + + operatorConfig := config.NetworkServicesOperator{} + operatorConfig.NetworkInterface.Enabled = true + operatorConfig.NetworkInterface.Location = config.LocationConfig{ + Name: testLocationName, + Namespace: testLocationNS, + } + + return &scenario{ + restConfig: restConfig, + events: events.NewFakeRecorder(64), + t: t, + ctx: ctx, + client: cl, + ipam: ipam, + reconciler: &NetworkInterfaceClaimReconciler{Config: operatorConfig, IPAM: ipam}, + namespace: namespaceName, + } +} + +// transientNamespaceClient fails namespace reads the way a flaky API server +// does, without the namespace itself changing. +type transientNamespaceClient struct { + client.Client +} + +func (c *transientNamespaceClient) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...client.GetOption, +) error { + if _, ok := obj.(*corev1.Namespace); ok { + return apierrors.NewServiceUnavailable("the server is currently unable to handle the request") + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func sanitizeName(in string) string { + var b strings.Builder + for _, r := range in { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + return strings.Trim(b.String(), "-") +} + +func (s *scenario) createClaim(name string, spec networkingv1alpha.NetworkInterfaceClaimSpec) *networkingv1alpha.NetworkInterfaceClaim { + s.t.Helper() + claim := &networkingv1alpha.NetworkInterfaceClaim{} + claim.Namespace = s.namespace + claim.Name = name + claim.Spec = spec + claim.Spec.Network = networkingv1alpha.LocalNetworkRef{Name: "default"} + require.NoError(s.t, s.client.Create(s.ctx, claim)) + return claim +} + +func (s *scenario) reconcile(claim *networkingv1alpha.NetworkInterfaceClaim) { + s.t.Helper() + _, err := s.reconciler.reconcileClaim(s.ctx, s.client, s.events, client.ObjectKeyFromObject(claim)) + require.NoError(s.t, err) +} + +func (s *scenario) reconcileInterface(name string) { + s.t.Helper() + interfaces := &NetworkInterfaceReconciler{Config: s.reconciler.Config, IPAM: s.ipam, claims: s.reconciler} + require.NoError(s.t, interfaces.reconcileInterface(s.ctx, s.client, + client.ObjectKey{Namespace: s.namespace, Name: name})) +} + +// programNetworkContext marks the binding's context ready, which is what makes +// the interface record a context and the gateway resolvable. +func (s *scenario) programNetworkContext() string { + s.t.Helper() + + bindingName := fmt.Sprintf("default-%s-%s", testLocationNS, testLocationName) + var binding networkingv1alpha.NetworkBinding + require.NoError(s.t, s.client.Get(s.ctx, + client.ObjectKey{Namespace: s.namespace, Name: bindingName}, &binding)) + + contextName := bindingName + binding.Status.NetworkContextRef = &networkingv1alpha.NetworkContextRef{ + Namespace: s.namespace, + Name: contextName, + } + require.NoError(s.t, s.client.Status().Update(s.ctx, &binding)) + return contextName +} + +func (s *scenario) createSubnet( + name, contextName string, + family networkingv1alpha.IPFamily, + startAddress string, + prefixLength int32, +) { + s.t.Helper() + + subnet := &networkingv1alpha.Subnet{} + subnet.Namespace = s.namespace + subnet.Name = name + subnet.Spec = networkingv1alpha.SubnetSpec{ + SubnetClass: "private", + NetworkContext: networkingv1alpha.LocalNetworkContextRef{Name: contextName}, + Location: networkingv1alpha.LocationReference{ + Name: testLocationName, + Namespace: testLocationNS, + }, + IPFamily: family, + StartAddress: startAddress, + PrefixLength: prefixLength, + } + require.NoError(s.t, s.client.Create(s.ctx, subnet)) +} + +func (s *scenario) getClaim(name string) *networkingv1alpha.NetworkInterfaceClaim { + s.t.Helper() + var claim networkingv1alpha.NetworkInterfaceClaim + require.NoError(s.t, s.client.Get(s.ctx, client.ObjectKey{Namespace: s.namespace, Name: name}, &claim)) + return &claim +} + +func (s *scenario) getInterface(name string) (*networkingv1alpha.NetworkInterface, error) { + var iface networkingv1alpha.NetworkInterface + err := s.client.Get(s.ctx, client.ObjectKey{Namespace: s.namespace, Name: name}, &iface) + return &iface, err +} + +func (s *scenario) deleteClaim(claim *networkingv1alpha.NetworkInterfaceClaim) { + s.t.Helper() + require.NoError(s.t, s.client.Delete(s.ctx, claim)) + s.reconcile(claim) +} + +func (s *scenario) ipClaim(name string) *ipamv1alpha1.IPClaim { + s.t.Helper() + cl, err := s.ipam.ClientForProject(testProject) + require.NoError(s.t, err) + + var ipClaim ipamv1alpha1.IPClaim + require.NoError(s.t, cl.Get(s.ctx, client.ObjectKey{Namespace: testProjectNS, Name: name}, &ipClaim)) + return &ipClaim +} + +func conditionOf(claim *networkingv1alpha.NetworkInterfaceClaim, conditionType string) *metav1.Condition { + return apimeta.FindStatusCondition(claim.Status.Conditions, conditionType) +} + +func TestNetworkInterfaceClaimBindsDualStack(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}, + publicV4Class()) + + claim := s.createClaim("web-0-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + Addresses: []networkingv1alpha.NetworkInterfaceAddressRequest{{Class: testPublicV4Class}}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("web-0-eth0") + require.NoError(t, err) + require.Equal(t, int32(1460), iface.Spec.MTU) + require.Equal(t, "eth0", iface.Spec.InterfaceName) + require.Equal(t, networkingv1alpha.NetworkInterfacePhaseBound, iface.Status.Phase) + require.NotNil(t, iface.Spec.ClaimRef) + require.Equal(t, claim.Name, iface.Spec.ClaimRef.Name) + + require.Len(t, iface.Spec.Addresses, 2) + require.Equal(t, networkingv1alpha.IPv6Protocol, iface.Spec.Addresses[0].Family) + require.True(t, iface.Spec.Addresses[0].Primary, "the first family listed holds the primary address") + require.Equal(t, "2001:db8:a000:1::/96", iface.Spec.Addresses[0].Address) + require.Equal(t, networkingv1alpha.IPv4Protocol, iface.Spec.Addresses[1].Family) + require.False(t, iface.Spec.Addresses[1].Primary) + require.Equal(t, "10.128.0.1/32", iface.Spec.Addresses[1].Address, + "an address inside the network keeps its prefix, host prefixes included") + + require.Len(t, iface.Spec.ExternalAddresses, 1) + require.Equal(t, testPublicV4Class, iface.Spec.ExternalAddresses[0].Class) + require.Equal(t, "10.128.0.2", iface.Spec.ExternalAddresses[0].Address, + "an externally reachable address is bare, with no prefix") + + require.ElementsMatch(t, []string{ + "web-0-eth0-f-ipv6", + "web-0-eth0-f-ipv4", + "web-0-eth0-c-" + testPublicV4Class, + }, s.ipam.created()[testProject]) + + bound := s.getClaim("web-0-eth0") + require.Equal(t, "web-0-eth0", bound.Status.NetworkInterfaceRef.Name) + require.Equal(t, iface.Spec.Addresses, bound.Status.Addresses) + require.Equal(t, iface.Spec.ExternalAddresses, bound.Status.ExternalAddresses) + + require.Equal(t, metav1.ConditionTrue, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimBound).Status) + require.Equal(t, metav1.ConditionTrue, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimAllocated).Status) + require.Equal(t, metav1.ConditionUnknown, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimProgrammed).Status, + "programming is out of scope and must stay unknown") + require.NotEqual(t, metav1.ConditionTrue, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimReady).Status, + "Ready requires Programmed, which nothing reports yet") +} + +func TestNetworkInterfaceClaimFailsClosedWithoutProject(t *testing.T) { + s := newScenario(t, false, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("orphan-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + require.Zero(t, s.ipam.createdAnywhere(), + "a namespace naming no project must not reach IPAM at all") + + _, err := s.getInterface("orphan-eth0") + require.True(t, apierrors.IsNotFound(err), "nothing may be published without a project") + + rejected := s.getClaim("orphan-eth0") + for _, conditionType := range []string{ + networkingv1alpha.NetworkInterfaceClaimBound, + networkingv1alpha.NetworkInterfaceClaimAllocated, + networkingv1alpha.NetworkInterfaceClaimReady, + } { + condition := conditionOf(rejected, conditionType) + require.Equal(t, metav1.ConditionFalse, condition.Status, conditionType) + require.Equal(t, "ProjectUnresolved", condition.Reason, conditionType) + require.Contains(t, condition.Message, downstreamclient.UpstreamOwnerClusterNameLabel, + "the failure must name the label that is missing") + } + + // A claim that allocated nothing must still be deletable. + s.deleteClaim(rejected) + var gone networkingv1alpha.NetworkInterfaceClaim + err = s.client.Get(s.ctx, client.ObjectKeyFromObject(rejected), &gone) + require.True(t, apierrors.IsNotFound(err)) +} + +func TestNetworkInterfaceClaimRejectsFamilyTheNetworkDoesNotCarry(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}) + + claim := s.createClaim("v6-on-v4-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + require.Zero(t, s.ipam.createdAnywhere(), + "IPAM binds the class's family and would never report this, so NSO must catch it first") + + rejected := s.getClaim("v6-on-v4-eth0") + condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, "AddressFamilyNotCarried", condition.Reason) + require.Contains(t, condition.Message, "default") + require.Contains(t, condition.Message, "IPv6") +} + +func TestNetworkInterfaceClaimRollsBackPartialAllocation(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}) + + s.ipam.refuse("half-eth0-f-ipv4", apierrors.NewGenericServerResponse( + httpStatusInsufficientStorage, "POST", + schema.GroupResource{Group: "ipam.miloapis.com", Resource: "ipclaims"}, + "", "IPPool exhausted", 0, false)) + + claim := s.createClaim("half-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + _, err := s.getInterface("half-eth0") + require.True(t, apierrors.IsNotFound(err), "a partly addressed interface is never published") + + require.Equal(t, []string{"half-eth0-f-ipv6"}, s.ipam.deleted()[testProject], + "the address that did allocate is released rather than leaked") + + rejected := s.getClaim("half-eth0") + condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, string(allocationFailureExhausted), condition.Reason) + require.Contains(t, condition.Message, "IPPool exhausted", + "IPAM's own message is carried through verbatim") +} + +func TestNetworkInterfaceRetainRebindsSameAddresses(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + spec := networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + } + + claim := s.createClaim("slot-0-eth0", spec) + s.reconcile(claim) + + first, err := s.getInterface("slot-0-eth0") + require.NoError(t, err) + originalAddresses := first.Spec.Addresses + originalClaimUID := claim.UID + + s.deleteClaim(s.getClaim("slot-0-eth0")) + + retained, err := s.getInterface("slot-0-eth0") + require.NoError(t, err, "Retain keeps the interface") + require.Nil(t, retained.Spec.ClaimRef) + require.Equal(t, networkingv1alpha.NetworkInterfacePhaseAvailable, retained.Status.Phase) + require.Equal(t, originalAddresses, retained.Spec.Addresses, "a retained interface keeps its addresses") + require.Empty(t, s.ipam.deleted()[testProject], "Retain releases no addresses") + require.Empty(t, s.ipam.orphanedAllocations(), + "retention comes from keeping the IPClaim alive, so nothing is ever orphaned") + + allocationsBefore := len(s.ipam.created()[testProject]) + + replacement := s.createClaim("slot-0-eth0", spec) + require.NotEqual(t, originalClaimUID, replacement.UID, + "the replacement is a genuinely different object, not the same claim resurrected") + s.reconcile(replacement) + + rebound, err := s.getInterface("slot-0-eth0") + require.NoError(t, err) + require.Equal(t, originalAddresses, rebound.Spec.Addresses, + "the replacement comes back to the addresses the predecessor held") + require.Equal(t, "slot-0-eth0", rebound.Spec.ClaimRef.Name, + "the interface records the claim now holding it") + require.Equal(t, networkingv1alpha.NetworkInterfacePhaseBound, rebound.Status.Phase) + require.Len(t, s.ipam.created()[testProject], allocationsBefore, + "rebinding allocates nothing new") + + bound := s.getClaim("slot-0-eth0") + require.Equal(t, originalAddresses, bound.Status.Addresses) +} + +func TestNetworkInterfaceDeleteReleasesAddresses(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("ephemeral-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + _, err := s.getInterface("ephemeral-eth0") + require.NoError(t, err) + + s.deleteClaim(s.getClaim("ephemeral-eth0")) + + _, err = s.getInterface("ephemeral-eth0") + require.True(t, apierrors.IsNotFound(err), "Delete removes the interface") + require.Equal(t, []string{"ephemeral-eth0-f-ipv6"}, s.ipam.deleted()[testProject]) + require.Empty(t, s.ipam.orphanedAllocations(), + "an address claimed under Delete goes back to its pool") +} + +// IPAM reports a duplicate name as a 500. Treating that as a hard failure +// rejects the claim forever, because every retry asks for the same name. +func TestAllocationResumesAfterTheInterfaceFailsToLand(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}) + + spec := networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + } + + claim := s.createClaim("resumed", spec) + s.reconcile(claim) + + original, err := s.getInterface("resumed") + require.NoError(t, err) + originalAddresses := original.Spec.Addresses + + // Stand in for the interface write never landing. + controllerutil.RemoveFinalizer(original, networkInterfaceFinalizer) + require.NoError(t, s.client.Update(s.ctx, original)) + require.NoError(t, s.client.Delete(s.ctx, original)) + + allocationsBefore := len(s.ipam.created()[testProject]) + + s.reconcile(s.getClaim("resumed")) + + recovered, err := s.getInterface("resumed") + require.NoError(t, err, "the next reconcile must rebuild the interface, not reject the claim") + require.Equal(t, originalAddresses, recovered.Spec.Addresses, + "it comes back on the addresses already allocated under those names") + + require.Len(t, s.ipam.created()[testProject], allocationsBefore, + "the addresses are found by name, so nothing is re-allocated") + + bound := s.getClaim("resumed") + require.Equal(t, metav1.ConditionTrue, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimAllocated).Status, + "a 409 from a name we already own is not an allocation failure") +} + +// A retained interface sits Available and bound to nothing, which is when an +// operator is most likely to delete it. Its addresses must still be released. +func TestDeletingAnInterfaceReleasesItsAddresses(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("stranded", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(claim) + s.deleteClaim(s.getClaim("stranded")) + + available, err := s.getInterface("stranded") + require.NoError(t, err) + require.Equal(t, networkingv1alpha.NetworkInterfacePhaseAvailable, available.Status.Phase) + require.Contains(t, available.Finalizers, networkInterfaceFinalizer, + "a retained interface carries its own finalizer, having no claim to carry one for it") + + require.NoError(t, s.client.Delete(s.ctx, available)) + s.reconcileInterface("stranded") + + _, err = s.getInterface("stranded") + require.True(t, apierrors.IsNotFound(err), "the finalizer must not hold the delete open once released") + require.Equal(t, []string{"stranded-f-ipv6"}, s.ipam.deleted()[testProject], + "the addresses go back to the pool rather than outliving every object naming them") +} + +// A live claim rebuilds the interface, so releasing its addresses here would +// renumber a running workload. +func TestDeletingABoundInterfaceKeepsItsAddresses(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("still-held", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + bound, err := s.getInterface("still-held") + require.NoError(t, err) + originalAddresses := bound.Spec.Addresses + + require.NoError(t, s.client.Delete(s.ctx, bound)) + s.reconcileInterface("still-held") + + require.Empty(t, s.ipam.deleted()[testProject], + "a claim still holds this interface, so its addresses stay allocated") + + s.reconcile(s.getClaim("still-held")) + + rebuilt, err := s.getInterface("still-held") + require.NoError(t, err, "the claim rebuilds the interface it still holds") + require.Equal(t, originalAddresses, rebuilt.Spec.Addresses, + "and it comes back on the same addresses rather than renumbering") +} + +// A fake recorder cannot be forbidden, so no other test catches an event the +// controller has no grant to write. The generated role is the only signal. +func TestManagerRoleGrantsEventCreation(t *testing.T) { + manifest, err := os.ReadFile(filepath.Join("..", "..", "config", "rbac", "role.yaml")) + require.NoError(t, err) + + var role rbacv1.ClusterRole + require.NoError(t, yaml.Unmarshal(manifest, &role)) + + granted := false + for _, rule := range role.Rules { + if slices.Contains(rule.APIGroups, "events.k8s.io") && + slices.Contains(rule.Resources, "events") && + slices.Contains(rule.Verbs, "create") { + granted = true + break + } + } + + require.True(t, granted, + "the controller records events through the events.k8s.io API; without that group granted they are "+ + "rejected and the warning never reaches an operator, whatever the core-group grant says") +} + +// events.k8s.io/v1 requires an action and a resolvable regarding reference, +// and a fake recorder checks neither. This posts through a real broadcaster so +// the event has to survive validation. Authorization is covered separately by +// TestManagerRoleGrantsEventCreation, because envtest does not enforce RBAC. +func TestMissingAllocationEventReachesTheAPIServer(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("delivered", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("delivered") + require.NoError(t, err) + + eventClient, err := eventsv1client.NewForConfig(s.restConfig) + require.NoError(t, err) + + broadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: eventClient}) + require.NoError(t, broadcaster.StartRecordingToSinkWithContext(s.ctx)) + t.Cleanup(broadcaster.Shutdown) + + recorder := broadcaster.NewRecorder(s.client.Scheme(), "networkinterfaceclaim-controller") + + s.reconciler.reportMissingAllocation(s.ctx, recorder, + projectRouting{project: testProject, projectNamespace: testProjectNS}, + s.getClaim("delivered"), iface, + allocationEntry{discriminator: "f-ipv6", address: iface.Spec.Addresses[0].Address}, + "delivered-f-ipv6", "") + + var delivered *eventsv1.Event + require.Eventually(t, func() bool { + list, err := eventClient.Events(s.namespace).List(s.ctx, metav1.ListOptions{}) + if err != nil { + return false + } + for i := range list.Items { + if list.Items[i].Reason == "AddressAllocationMissing" { + delivered = &list.Items[i] + return true + } + } + return false + }, 30*time.Second, 250*time.Millisecond, + "the event never landed; the API server rejected it or the recorder never posted it") + + require.Equal(t, corev1.EventTypeWarning, delivered.Type) + require.NotEmpty(t, delivered.Action, "events.k8s.io/v1 requires an action") + require.Equal(t, "NetworkInterfaceClaim", delivered.Regarding.Kind) + require.Equal(t, "delivered", delivered.Regarding.Name) + require.NotNil(t, delivered.Related, "the interface is carried as the related object") + require.Equal(t, "NetworkInterface", delivered.Related.Kind) + require.Contains(t, delivered.Note, iface.Spec.Addresses[0].Address) + require.Contains(t, delivered.Note, "delivered-f-ipv6") + require.Contains(t, delivered.Note, testProject) +} + +// Addresses belong to the network they were allocated from. Publishing them +// under another network hands one network's addresses to another. +func TestAdoptionRefusesAnInterfaceOnAnotherNetwork(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + other := &networkingv1alpha.Network{} + other.Namespace = s.namespace + other.Name = "other" + other.Spec = networkingv1alpha.NetworkSpec{ + IPAM: networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto}, + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + MTU: 1460, + } + require.NoError(t, s.client.Create(s.ctx, other)) + + spec := networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + } + + first := s.createClaim("crossnet", spec) + s.reconcile(first) + s.deleteClaim(s.getClaim("crossnet")) + + retained, err := s.getInterface("crossnet") + require.NoError(t, err) + heldAddress := retained.Spec.Addresses[0].Address + + onOtherNetwork := spec + onOtherNetwork.Network = networkingv1alpha.LocalNetworkRef{Name: "other"} + claim := &networkingv1alpha.NetworkInterfaceClaim{} + claim.Namespace = s.namespace + claim.Name = "crossnet" + claim.Spec = onOtherNetwork + require.NoError(t, s.client.Create(s.ctx, claim)) + s.reconcile(claim) + + rejected := s.getClaim("crossnet") + condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimReady) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, "NetworkMismatch", condition.Reason) + require.Contains(t, condition.Message, "other") + + require.Empty(t, rejected.Status.Addresses, + "a claim on another network must not publish addresses allocated for this one") + + unchanged, err := s.getInterface("crossnet") + require.NoError(t, err) + require.Equal(t, heldAddress, unchanged.Spec.Addresses[0].Address) + require.Nil(t, unchanged.Spec.ClaimRef, "the interface stays unbound") +} + +// The data plane owns Programmed. Overwriting it puts Ready permanently out of +// reach, and Ready is what consumers gate on. +func TestProgrammedIsSeededThenLeftAlone(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("gated", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + seeded := s.getClaim("gated") + require.Equal(t, metav1.ConditionUnknown, + conditionOf(seeded, networkingv1alpha.NetworkInterfaceClaimProgrammed).Status) + require.NotEqual(t, metav1.ConditionTrue, + conditionOf(seeded, networkingv1alpha.NetworkInterfaceClaimReady).Status) + + // Stand in for the data plane reporting the attachment. + apimeta.SetStatusCondition(&seeded.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimProgrammed, + Status: metav1.ConditionTrue, + Reason: "Programmed", + Message: "The attachment is ready", + }) + require.NoError(t, s.client.Status().Update(s.ctx, seeded)) + + s.reconcile(s.getClaim("gated")) + + settled := s.getClaim("gated") + require.Equal(t, metav1.ConditionTrue, + conditionOf(settled, networkingv1alpha.NetworkInterfaceClaimProgrammed).Status, + "NSO must not revert the condition the data plane owns") + require.Equal(t, metav1.ConditionTrue, + conditionOf(settled, networkingv1alpha.NetworkInterfaceClaimReady).Status, + "Ready follows from the other three rather than being hardcoded") +} + +// A blip reading the namespace must not demote a healthy claim and then leave +// nothing to bring it back. +func TestTransientProjectFailureDoesNotWedgeTheClaim(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("blip", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + failing := &transientNamespaceClient{Client: s.client} + _, err := s.reconciler.reconcileClaim(s.ctx, failing, s.events, + client.ObjectKey{Namespace: s.namespace, Name: "blip"}) + require.Error(t, err, "a failed read must be retried, not turned into a rejection") + + bound := s.getClaim("blip") + require.Equal(t, metav1.ConditionTrue, + conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimBound).Status, + "a read failure says nothing about whether the claim is bound") +} + +// A rejection must carry its own way back, because nothing watches the network. +func TestRejectionRequeues(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := &networkingv1alpha.NetworkInterfaceClaim{} + claim.Namespace = s.namespace + claim.Name = "nonetwork" + claim.Spec = networkingv1alpha.NetworkInterfaceClaimSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "absent"}, + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + } + require.NoError(t, s.client.Create(s.ctx, claim)) + + result, err := s.reconciler.reconcileClaim(s.ctx, s.client, s.events, + client.ObjectKeyFromObject(claim)) + require.NoError(t, err) + require.NotZero(t, result.RequeueAfter, + "nothing else will bring this claim back once the network appears") +} + +// A provider configures a NIC from the interface alone, so the gateway has to +// reach NetworkInterface.spec. The subnet usually appears after the interface, +// so it has to be filled in later rather than only at creation. +func TestGatewayReachesTheInterfaceWhenTheSubnetAppears(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}) + + claim := s.createClaim("routed", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("routed") + require.NoError(t, err) + require.Empty(t, iface.Spec.Addresses[0].Gateway, + "no subnet yet, so no gateway, which is a legitimate state") + + contextName := s.programNetworkContext() + s.createSubnet("v4", contextName, networkingv1alpha.IPv4Protocol, "10.128.0.0", 24) + + s.reconcile(s.getClaim("routed")) + + routed, err := s.getInterface("routed") + require.NoError(t, err) + require.Equal(t, "10.128.0.1", routed.Spec.Addresses[0].Gateway, + "the gateway must be on the interface, which is all a provider reads") + + bound := s.getClaim("routed") + require.Equal(t, "10.128.0.1", bound.Status.Addresses[0].Gateway, + "the claim's copy comes from the interface rather than being resolved twice") + + // Reconciling again must not keep rewriting the same value. + before := routed.ResourceVersion + s.reconcile(s.getClaim("routed")) + settled, err := s.getInterface("routed") + require.NoError(t, err) + require.Equal(t, before, settled.ResourceVersion, + "an unchanged gateway must not write to the API server") +} + +// IPAM may give a missing address to another claim, so an operator has to see +// it. Reallocating instead would renumber a running workload. +func TestMissingAllocationIsReportedOnTheClaim(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("unbacked", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("unbacked") + require.NoError(t, err) + advertised := iface.Spec.Addresses[0].Address + + ipamClient, err := s.ipam.ClientForProject(testProject) + require.NoError(t, err) + require.NoError(t, ipamClient.Delete(s.ctx, s.ipClaim("unbacked-f-ipv6"))) + + before := testutil.ToFloat64(missingAllocationsTotal.WithLabelValues(testProject)) + s.reconcile(s.getClaim("unbacked")) + + require.Equal(t, before+1, testutil.ToFloat64(missingAllocationsTotal.WithLabelValues(testProject)), + "the metric is what makes this alertable") + + var warning string + select { + case warning = <-s.events.Events: + default: + t.Fatal("no event was recorded for an address nothing holds") + } + + require.Contains(t, warning, corev1.EventTypeWarning) + require.Contains(t, warning, "AddressAllocationMissing") + require.Contains(t, warning, advertised, "the event names the address at risk") + require.Contains(t, warning, "unbacked-f-ipv6", "and the allocation that went missing") + require.Contains(t, warning, testProject) + + // Allocated must not claim every address is held once one is not. + reported := s.getClaim("unbacked") + allocated := conditionOf(reported, networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, allocated.Status) + require.Equal(t, "AddressAllocationMissing", allocated.Reason) + require.Contains(t, allocated.Message, advertised) + + require.NotEqual(t, metav1.ConditionTrue, + conditionOf(reported, networkingv1alpha.NetworkInterfaceClaimReady).Status) + + still, err := s.getInterface("unbacked") + require.NoError(t, err) + require.Equal(t, advertised, still.Spec.Addresses[0].Address, + "the address is never silently renumbered") +} + +// Release reads addresses off the interface. Deleting the interface first +// leaves nothing to read, so it must fall back to what the claim asked for. +func TestDeletingAClaimWhoseInterfaceIsGoneReleasesItsAddresses(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}, + publicV4Class()) + + claim := s.createClaim("outlived", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + Addresses: []networkingv1alpha.NetworkInterfaceAddressRequest{{Class: testPublicV4Class}}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("outlived") + require.NoError(t, err) + controllerutil.RemoveFinalizer(iface, networkInterfaceFinalizer) + require.NoError(t, s.client.Update(s.ctx, iface)) + require.NoError(t, s.client.Delete(s.ctx, iface)) + + s.deleteClaim(s.getClaim("outlived")) + + require.ElementsMatch(t, []string{ + "outlived-f-ipv6", + "outlived-f-ipv4", + "outlived-c-" + testPublicV4Class, + }, s.ipam.deleted()[testProject], + "every address the claim minted goes back, interface or no interface") +} + +// An interface outlives the claim that allocated its addresses. A lookup keyed +// on the current holder finds nothing, which reads as nothing to release. +func TestAllocationsFollowTheMintingClaimNotTheHolder(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + spec := func() networkingv1alpha.NetworkInterfaceClaimSpec { + return networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + NetworkInterfaceName: "shared", + } + } + + minter := s.createClaim("minter", spec()) + s.reconcile(minter) + require.Equal(t, []string{"minter-f-ipv6"}, s.ipam.created()[testProject]) + + s.deleteClaim(s.getClaim("minter")) + + adopter := s.createClaim("adopter", spec()) + s.reconcile(adopter) + + iface, err := s.getInterface("shared") + require.NoError(t, err) + require.Equal(t, "minter", iface.Annotations[allocationClaimAnnotation], + "the interface keeps naming the claim its addresses were minted under") + require.Equal(t, "adopter", iface.Spec.ClaimRef.Name) + require.Len(t, s.ipam.created()[testProject], 1, "adoption allocates nothing new") + + s.deleteClaim(s.getClaim("adopter")) + require.NoError(t, s.client.Delete(s.ctx, iface)) + s.reconcileInterface("shared") + + require.Equal(t, []string{"minter-f-ipv6"}, s.ipam.deleted()[testProject], + "release must find the address under the name it was minted with") +} + +func TestAdoptedInterfaceMustSatisfyTheClaim(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}) + + single := s.createClaim("grower", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(single) + s.deleteClaim(s.getClaim("grower")) + + dualStack := s.createClaim("grower", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(dualStack) + + rejected := s.getClaim("grower") + condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, condition.Status, + "a retained interface holding one family cannot satisfy a dual-stack claim") + require.Contains(t, condition.Message, "IPv4") +} + +// IPAM freezes the reclaim policy onto the allocation. A claim asking for a +// different one cannot be honoured, so binding it would strand the address. +func TestAdoptionRefusesADifferentReclaimPolicy(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + retained := s.createClaim("switcher", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(retained) + s.deleteClaim(s.getClaim("switcher")) + + replacement := s.createClaim("switcher", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(replacement) + + condition := conditionOf(s.getClaim("switcher"), networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Contains(t, condition.Message, "Retain") + require.Empty(t, s.ipam.orphanedAllocations(), + "refusing at bind time is what keeps the allocation from being stranded later") +} + +// An immutable reclaim policy is what stops an address being stranded. +func TestReclaimPolicyIsImmutable(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("frozen", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(claim) + + switched := s.getClaim("frozen") + switched.Spec.ReclaimPolicy = networkingv1alpha.NetworkInterfaceReclaimPolicyDelete + err := s.client.Update(s.ctx, switched) + require.Error(t, err, "the API must reject the switch that would strand the allocation") + require.Contains(t, err.Error(), "immutable") +} + +func TestExternalAddressesDropTheHostPrefix(t *testing.T) { + for _, tc := range []struct { + allocated string + want string + }{ + {"198.51.100.11/32", "198.51.100.11"}, + {"2001:db8::1/128", "2001:db8::1"}, + {"10.128.0.0/24", "10.128.0.0/24"}, + {"2001:db8:a000:1::/96", "2001:db8:a000:1::/96"}, + {"not-an-address", "not-an-address"}, + } { + require.Equal(t, tc.want, allocatedAddress{cidr: tc.allocated}.bareAddress(), tc.allocated) + } +} + +func TestIPClaimNamesSurviveInstanceReplacement(t *testing.T) { + const claimName = "workload-default-us-central-1-0-eth0" + + require.Equal(t, "workload-default-us-central-1-0-eth0-f-ipv6", + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol))) + + require.Equal(t, + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol)), + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol)), + "the name depends on the claim and the request, and on nothing that changes") + + require.NotEqual(t, + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv4Protocol)), + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol))) + + require.NotEqual(t, + ipClaimName(claimName, classDiscriminator("f-ipv6")), + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol)), + "a class named like a family must not collide with the family") + + longName := strings.Repeat("a", 253) + for _, discriminator := range []string{ + familyDiscriminator(networkingv1alpha.IPv4Protocol), + familyDiscriminator(networkingv1alpha.IPv6Protocol), + classDiscriminator(strings.Repeat("c", 63)), + } { + name := ipClaimName(longName, discriminator) + require.LessOrEqual(t, len(name), 253, "names must stay valid at the maximum claim length") + require.Empty(t, validation.IsDNS1123Subdomain(name)) + require.Equal(t, name, ipClaimName(longName, discriminator), "the hashed form is deterministic too") + } + + require.NotEqual(t, + ipClaimName(longName, familyDiscriminator(networkingv1alpha.IPv4Protocol)), + ipClaimName(longName, familyDiscriminator(networkingv1alpha.IPv6Protocol)), + "truncation must not merge two requests into one name") +} From f15964584062456927717feb903e9d8167640fcd Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 12:24:37 -0500 Subject: [PATCH 04/13] test: run IPAM for real in the e2e environment The integration is not proven until a claim gets a real address back from a real IPAM service. These suites assert against IPAM directly rather than against the operator's own status, so a controller that invented addresses would fail them. Key changes: - Deploy IPAM from its published manifest bundle, pinned by digest and verified to name the commit go.mod pins, with its database on the CloudNativePG operator. - Issue the serving certificate per pod from an intermediate CA through the cert-manager CSI driver, so aggregation verifies a real chain rather than skipping TLS. - Seed classes and pools for two projects with identical class names and disjoint ranges, so a routing bug shows up as the wrong address space rather than as a passing test. - Wire a per-project impersonation kubeconfig, the mechanism production uses, and verify the namespace labels round trip through the real decoder. - Clear cluster scoped fixtures before every run and fail loudly when the clearing cannot happen, so a run that died mid-suite cannot poison the next one. - Cover dual stack binding, project isolation, a namespace naming no project, a named class, both reclaim policies, interface deletion in both orders, a family the network does not carry, and an allocation that vanished. - Every assertion of an absence carries a positive control, so a suite cannot pass by failing to reach IPAM at all. --- .github/workflows/test-e2e.yml | 21 +- Taskfile.test-infra.yml | 307 +++++++++++++++++- config/dependencies/README.md | 49 +++ .../ipam/overlay/cluster-issuer.yaml | 6 + .../ipam/overlay/kustomization.yaml | 18 + .../ipam/overlay/postgres-cluster.yaml | 55 ++++ config/dependencies/ipam/overlay/rbac.yaml | 77 +++++ config/dependencies/ipam/overlay/secret.yaml | 29 ++ .../ipam/patches/apiservice-patch.yaml | 16 + .../ipam/patches/deployment-patch.yaml | 24 ++ .../networkpolicy-apiserver-egress-patch.yaml | 26 ++ .../ipam/patches/single-replica-patch.yaml | 14 + .../dependencies/ipam/root-kustomization.yaml | 70 ++++ .../postgres-operator/kustomization.yaml | 43 +++ .../postgres-operator/namespace.yaml | 4 + config/e2e/kustomization.yaml | 14 + hack/gen-ipam-impersonation-kubeconfig.sh | 99 ++++++ hack/ipam-clear-fixtures.sh | 114 +++++++ hack/ipam-fetch-bundle.sh | 96 ++++++ hack/module-dir.sh | 52 +++ test/e2e/fixtures/ipam/RANGES.md | 83 +++++ test/e2e/fixtures/ipam/README.md | 25 ++ test/e2e/fixtures/ipam/namespaces.yaml | 58 ++++ .../fixtures/ipam/project-alpha/classes.yaml | 89 +++++ .../fixtures/ipam/project-alpha/pools.yaml | 52 +++ .../fixtures/ipam/project-beta/classes.yaml | 89 +++++ .../e2e/fixtures/ipam/project-beta/pools.yaml | 52 +++ test/e2e/fixtures/ipam/rbac.yaml | 28 ++ .../chainsaw-test.yaml | 220 +++++++++++++ .../chainsaw-test.yaml | 146 +++++++++ .../chainsaw-test.yaml | 86 +++++ .../chainsaw-test.yaml | 134 ++++++++ .../chainsaw-test.yaml | 191 +++++++++++ .../chainsaw-test.yaml | 132 ++++++++ .../chainsaw-test.yaml | 276 ++++++++++++++++ .../chainsaw-test.yaml | 204 ++++++++++++ .../chainsaw-test.yaml | 104 ++++++ .../chainsaw-test.yaml | 289 +++++++++++++++++ .../chainsaw-test.yaml | 137 ++++++++ 39 files changed, 3522 insertions(+), 7 deletions(-) create mode 100644 config/dependencies/README.md create mode 100644 config/dependencies/ipam/overlay/cluster-issuer.yaml create mode 100644 config/dependencies/ipam/overlay/kustomization.yaml create mode 100644 config/dependencies/ipam/overlay/postgres-cluster.yaml create mode 100644 config/dependencies/ipam/overlay/rbac.yaml create mode 100644 config/dependencies/ipam/overlay/secret.yaml create mode 100644 config/dependencies/ipam/patches/apiservice-patch.yaml create mode 100644 config/dependencies/ipam/patches/deployment-patch.yaml create mode 100644 config/dependencies/ipam/patches/networkpolicy-apiserver-egress-patch.yaml create mode 100644 config/dependencies/ipam/patches/single-replica-patch.yaml create mode 100644 config/dependencies/ipam/root-kustomization.yaml create mode 100644 config/dependencies/postgres-operator/kustomization.yaml create mode 100644 config/dependencies/postgres-operator/namespace.yaml create mode 100755 hack/gen-ipam-impersonation-kubeconfig.sh create mode 100755 hack/ipam-clear-fixtures.sh create mode 100755 hack/ipam-fetch-bundle.sh create mode 100755 hack/module-dir.sh create mode 100644 test/e2e/fixtures/ipam/RANGES.md create mode 100644 test/e2e/fixtures/ipam/README.md create mode 100644 test/e2e/fixtures/ipam/namespaces.yaml create mode 100644 test/e2e/fixtures/ipam/project-alpha/classes.yaml create mode 100644 test/e2e/fixtures/ipam/project-alpha/pools.yaml create mode 100644 test/e2e/fixtures/ipam/project-beta/classes.yaml create mode 100644 test/e2e/fixtures/ipam/project-beta/pools.yaml create mode 100644 test/e2e/fixtures/ipam/rbac.yaml create mode 100644 test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-project-isolation/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml create mode 100644 test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 420fe171..cc9bf2cd 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -37,8 +37,11 @@ jobs: # was removed in helm 3.13, so pin to the last release that accepts it. version: v3.12.3 + # The same installer the federated workflow uses. An explicit `make` list + # here meant the two workflows disagreed about which binaries exist, and + # a tool added to the test-infra tools task reached only one of them. - name: Install pinned tools - run: make kind kustomize cmctl chainsaw + run: task test-infra:tools - name: Bring up the prod-fidelity env run: task test-infra:up @@ -62,6 +65,22 @@ jobs: KUBECONFIG="${UKC}" kubectl -n cert-manager get pods -o wide KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get certificate,secret echo "::endgroup::" + echo "::group::IPAM (upstream project control plane)" + KUBECONFIG="${UKC}" kubectl get apiservice v1alpha1.ipam.miloapis.com -o yaml + KUBECONFIG="${UKC}" kubectl -n ipam-system get pods,certificate,configmap -o wide + KUBECONFIG="${UKC}" kubectl -n ipam-system logs -l app=ipam-apiserver --tail=200 --all-containers + KUBECONFIG="${UKC}" kubectl -n ipam-system get cluster.postgresql.cnpg.io -o wide + KUBECONFIG="${UKC}" kubectl -n ipam-system describe cluster.postgresql.cnpg.io ipam-db + KUBECONFIG="${UKC}" kubectl -n ipam-system logs -l cnpg.io/cluster=ipam-db --tail=100 --all-containers + echo "::endgroup::" + echo "::group::CloudNativePG operator" + KUBECONFIG="${UKC}" kubectl -n cnpg-system get pods -o wide + KUBECONFIG="${UKC}" kubectl -n cnpg-system logs deploy/cnpg-cloudnative-pg --tail=200 + echo "::endgroup::" + echo "::group::IPAM fixtures" + KUBECONFIG=${TMPDIR}/.ipam-tenant-impersonation.yaml kubectl --context tenant-project-alpha get ipclasses,ippools -o wide + KUBECONFIG=${TMPDIR}/.ipam-tenant-impersonation.yaml kubectl --context tenant-project-beta get ipclasses,ippools -o wide + echo "::endgroup::" KC=${TMPDIR}/.kind-nso-downstream.yaml bin/kind-v0.32.0 get kubeconfig --name nso-downstream > "${KC}" export KUBECONFIG="${KC}" diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 6236e44d..93aa5da0 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -54,10 +54,31 @@ vars: IMG: sh: echo "ghcr.io/datum-cloud/network-services-operator:$(git rev-parse --short HEAD)" + # IPAM runs on the upstream cluster as the project control plane NSO claims + # addresses from. Its manifests come from the bundle milo-os/ipam publishes on + # every push to main, and its image from ghcr.io — both at the commit go.mod + # pins, so the apiserver and the Go client NSO compiles against stay in + # lockstep. Nothing is built locally: the image is pulled from ghcr by + # digest, multi-arch since upstream 13bd2c8 fixed the arm64 build. + # + # Digest, not tag: the release workflow republishes on every push, so the tag + # can move. The tag is kept alongside for legibility. Update all three + # together, and keep IPAM_COMMIT equal to the go.mod replace directive — + # ipam-bundle fails if the artifact does not carry that commit. + IPAM_BUNDLE_REPO: ghcr.io/milo-os/ipam-kustomize + IPAM_BUNDLE_TAG: v0.0.0-13bd2c8 + IPAM_BUNDLE_DIGEST: sha256:08418654958ea6861d54fe5f9e87d9cf306b87429b29fc41a7876dc143a13bf5 + IPAM_COMMIT: 13bd2c8c077b12983fa16a8888dcd117b65a14d2 + IPAM_STAGE: '{{.TMP_DIR}}/.ipam-bundle' + CRANE: '{{.REPO}}/bin/crane' + CRANE_VERSION: v0.21.7 + IPAM_KUBECONFIG: '{{.TMP_DIR}}/.ipam-tenant-impersonation.yaml' + IPAM_PROJECTS: project-alpha project-beta + tasks: tools: - desc: "Install the pinned tool binaries this env needs into bin/ (kind v0.32.0, karmadactl, kustomize, cmctl, chainsaw). Idempotent — skips any already present. Lets CI and a clean checkout bring up the env without hand-placed binaries." + desc: "Install the pinned tool binaries this env needs into bin/ (kind v0.32.0, karmadactl, kustomize, cmctl, chainsaw, crane). Idempotent — skips any already present. Lets CI and a clean checkout bring up the env without hand-placed binaries." vars: KIND_TOOL_VERSION: v0.32.0 KARMADACTL_VERSION: v1.15.2 @@ -75,6 +96,11 @@ tasks: - '[ -x {{.REPO}}/bin/kustomize ] || GOBIN={{.REPO}}/bin go install sigs.k8s.io/kustomize/kustomize/v5@{{.KUSTOMIZE_VERSION}}' - '[ -x {{.REPO}}/bin/cmctl ] || GOBIN={{.REPO}}/bin go install github.com/cert-manager/cmctl/v2@{{.CMCTL_VERSION}}' - '[ -x {{.REPO}}/bin/chainsaw ] || GOBIN={{.REPO}}/bin go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}}' + # crane reads the published IPAM manifest bundle out of ghcr.io. The flux + # CLI can do this too, but its go.mod replace directives stop `go install` + # building it, and crane is a much smaller dependency than all of flux for + # one registry read. + - '[ -x {{.REPO}}/bin/crane ] || GOBIN={{.REPO}}/bin go install github.com/google/go-containerregistry/cmd/crane@{{.CRANE_VERSION}}' # karmadactl carries replace directives in its go.mod, so `go install` can't # build it; pull the published release binary for this OS/arch instead. - | @@ -98,9 +124,23 @@ tasks: - task: downstream-namespaces - task: cert-manager-upstream - task: cert-manager-downstream + # Fetched early: it depends on nothing in the cluster, and pulling it here + # surfaces a registry problem before the long build below. + - task: ipam-bundle - task: nso-image - task: dns-crds + # The manager mounts this secret, so it has to exist before the pod does. + - task: ipam-kubeconfig - task: prepare-upstream + # IPAM's serving certificate cannot issue until upstream cert-manager is + # running, and upstream cert-manager crashloops with "the Gateway API CRDs + # do not seem to be present" until prepare-upstream installs them. Every + # IPAM step that touches the cluster therefore has to follow it. + - task: cnpg-operator + - task: ipam-deploy + - task: ipam-wait + - task: ipam-fixtures + - task: ipam-namespaces - task: eg-downstream - task: extension-server - task: billing-usage-collector @@ -120,6 +160,10 @@ tasks: - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} || true' - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} || true' - rm -f {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + # The IPAM fixtures and their database die with the upstream cluster; only + # the impersonation kubeconfig outlives it. + - rm -f {{.IPAM_KUBECONFIG}} + - rm -rf {{.IPAM_STAGE}} - echo "✨ done." clusters: @@ -161,10 +205,12 @@ tasks: dns-crds: desc: "Install DNSZone / DNSRecordSet / DNSZoneClass CRDs from the dns-operator Go module on the upstream cluster. Required before prepare-upstream when gateway.enableDNSIntegration is true (the operator indexes and watches these types at startup)." vars: - # Escape {{.Dir}} so Task does not treat it as a Task template var; - # go list must receive the literal {{.Dir}} format string. + # Resolved through the helper, not `go list` directly: go list does not + # download, so on a cold module cache it returns an empty string and exits + # 0, and the apply below then fails on a path with a leading slash that + # says nothing about the real cause. DNS_OPERATOR_CRD_DIR: - sh: go list -m -f '{{`{{.Dir}}`}}' go.miloapis.com/dns-operator + sh: '{{.REPO}}/hack/module-dir.sh go.miloapis.com/dns-operator' cmds: - echo "📜 installing dns-operator CRDs (upstream) from {{.DNS_OPERATOR_CRD_DIR}}" - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_OPERATOR_CRD_DIR}}/config/crd/bases/dns.networking.miloapis.com_dnszoneclasses.yaml @@ -358,6 +404,233 @@ tasks: - kubectl --context {{.DOWNSTREAM_CTX}} wait gatewayclass datum-downstream-gateway-e2e --for=condition=Accepted --timeout=120s - echo "✅ core components ready." + # ---- IPAM --------------------------------------------------------------- + # + # IPAM is the upstream PROJECT control plane: NSO's manager, running on the + # same kind cluster, plays the cell that claims addresses from it. The two + # roles are separated by kubeconfig and impersonation, not by cluster + # boundary, so colocating them costs no fidelity — the cells that provision + # network interfaces run single-cluster in production and reach IPAM exactly + # this way. + + cnpg-operator: + desc: "Install the CloudNativePG operator (chart 0.29.0 / CNPG 1.30.0) on the upstream cluster and wait until it can actually admit a Cluster. IPAM's bundle ships CNPG as a Flux HelmRelease; this env has no Flux, so it takes the same chart through kustomize --enable-helm, the way cert-manager is installed here." + cmds: + - echo "🐘 installing CloudNativePG (upstream)" + - '{{.KUSTOMIZE}} build --enable-helm config/dependencies/postgres-operator | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + - echo "⏳ waiting for the CNPG CRDs to be established" + - kubectl --context {{.UPSTREAM_CTX}} wait --for=condition=Established crd/clusters.postgresql.cnpg.io --timeout=120s + - echo "⏳ waiting for the CNPG operator" + - kubectl --context {{.UPSTREAM_CTX}} -n cnpg-system rollout status deploy/cnpg-cloudnative-pg --timeout=300s + # Deployment Available is not the same as "the webhook answers": the + # operator injects its own CA into the webhook configuration after it + # starts, and a Cluster applied into that window is rejected with a TLS + # or connection error that reads exactly like a flaky apply. Block until a + # server-side dry run is genuinely admitted, the same way wait-ready does + # for the NSO webhook. + - echo "⏳ waiting for the CNPG webhook to admit" + - | + PROBE='{"apiVersion":"postgresql.cnpg.io/v1","kind":"Cluster","metadata":{"name":"admit-probe","namespace":"default"},"spec":{"instances":1,"storage":{"size":"128Mi"}}}' + for i in $(seq 1 60); do + if echo "$PROBE" | kubectl --context {{.UPSTREAM_CTX}} create --dry-run=server -f - >/dev/null 2>&1; then + echo "✅ CNPG admitting"; break + fi + if [ "$i" -eq 60 ]; then echo "❌ CNPG webhook never became ready"; exit 1; fi + sleep 2 + done + + ipam-bundle: + desc: "Fetch the published IPAM manifest bundle from ghcr.io and assemble the composed deployment in a staging directory. Pinned by digest, and verified to carry the commit go.mod pins, so a rebuild of that commit cannot silently change what is deployed. Anonymous pull — CI needs no registry credentials." + cmds: + - | + set -e + stage="{{.IPAM_STAGE}}" + if [ -f "$stage/.digest" ] && [ "$(cat "$stage/.digest")" = "{{.IPAM_BUNDLE_DIGEST}}" ]; then + echo "📦 IPAM bundle already staged ({{.IPAM_BUNDLE_TAG}})" >&2 + else + echo "📦 fetching {{.IPAM_BUNDLE_REPO}}:{{.IPAM_BUNDLE_TAG}}" >&2 + rev=$({{.REPO}}/hack/ipam-fetch-bundle.sh \ + {{.CRANE}} {{.CRANE_VERSION}} \ + {{.IPAM_BUNDLE_REPO}} {{.IPAM_BUNDLE_DIGEST}} {{.IPAM_COMMIT}} "$stage/bundle") + echo "✅ fetched $rev" >&2 + fi + # Refreshed every run: these come from the working tree, so an edit to + # the overlay or a patch must reach the staged build without re-pulling. + rm -rf "$stage/overlay" "$stage/patches" + cp -R "{{.REPO}}/config/dependencies/ipam/overlay" "$stage/overlay" + cp -R "{{.REPO}}/config/dependencies/ipam/patches" "$stage/patches" + cp "{{.REPO}}/config/dependencies/ipam/root-kustomization.yaml" "$stage/kustomization.yaml" + echo "{{.IPAM_BUNDLE_DIGEST}}" > "$stage/.digest" + + ipam-render: + desc: "Render the composed IPAM deployment to stdout. What ipam-deploy applies, without applying it — the composed root is assembled at task time, so this is the only way to review it." + deps: + - ipam-bundle + cmds: + - '{{.KUSTOMIZE}} build {{.IPAM_STAGE}}' + + ipam-deploy: + deps: + - ipam-bundle + - cnpg-operator + desc: "Apply the IPAM apiserver, its Postgres, and the tenant RBAC to the upstream cluster (config/dependencies/ipam). Applies only — ipam-wait is what gates on readiness. Must follow prepare-upstream: upstream cert-manager crashloops until the Gateway-API CRDs it probes at startup exist, and until it runs, ipam-tls is never issued." + cmds: + - echo "🔧 deploying IPAM (upstream)" + - task: ipam-certmanager-ready + # The aggregated apiserver verifies the front proxy with + # --requestheader-client-ca-file. In kind that CA lives in the host's + # extension-apiserver-authentication ConfigMap, so it has to be copied + # into ipam-system before the pod starts — without it every impersonated + # extra arrives unauthenticated and every claim reads an empty tenant. + - kubectl --context {{.UPSTREAM_CTX}} create namespace ipam-system --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + - | + kubectl --context {{.UPSTREAM_CTX}} -n kube-system get configmap extension-apiserver-authentication \ + -o jsonpath='{.data.requestheader-client-ca-file}' > {{.TMP_DIR}}/.ipam-requestheader-ca.crt + kubectl --context {{.UPSTREAM_CTX}} -n ipam-system create configmap control-plane-ca \ + --from-file=ca.crt={{.TMP_DIR}}/.ipam-requestheader-ca.crt \ + --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + - '{{.KUSTOMIZE}} build {{.IPAM_STAGE}} | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + ipam-certmanager-ready: + desc: "Make sure upstream cert-manager is actually running before anything asks it for a certificate. It exits at startup when the Gateway-API CRDs are absent, so on a fresh env it is in CrashLoopBackOff by the time prepare-upstream installs them — and the backoff can be minutes long. Restart it rather than wait the backoff out." + cmds: + - | + if kubectl --context {{.UPSTREAM_CTX}} -n cert-manager wait deploy cert-manager \ + --for=condition=Available --timeout=15s >/dev/null 2>&1; then + echo "✅ upstream cert-manager already Available" + exit 0 + fi + echo "♻️ upstream cert-manager is not Available; restarting it" + kubectl --context {{.UPSTREAM_CTX}} -n cert-manager rollout restart deploy cert-manager + kubectl --context {{.UPSTREAM_CTX}} -n cert-manager rollout status deploy cert-manager --timeout=240s + + ipam-wait: + desc: "Gate on IPAM being genuinely servable: Postgres, the apiserver rollout, then the APIService itself. The certificate gate is on the internal CA, not on a serving cert: the CSI driver issues serving certs per pod at mount time, so a CA problem surfaces as a pod that never leaves Init. The APIService gate is not redundant — a suite that starts while aggregation is still settling fails with ServiceUnavailable, which reads like a test bug and is not one." + cmds: + # The serving cert is issued per pod by the cert-manager CSI driver, from + # the CA this gate waits on. Without the CA the mount never completes and + # the pod sits in Init with nothing in its logs to explain why. + - echo "⏳ waiting for the IPAM internal CA" + - kubectl --context {{.UPSTREAM_CTX}} -n ipam-system wait certificate/ipam-ca --for=condition=Ready --timeout=180s + - echo "⏳ waiting for Postgres (CNPG cluster ipam-db)" + # The apiserver's migrate init container runs `ipam migrate up` against + # this database, so it crashloops until Postgres accepts connections. + # CNPG reports Ready only once the instance is serving, which is what the + # init container needs — a scheduled pod is not enough. This replaced a + # `wait pod -l app.kubernetes.io/name=postgresql` gate when the plain + # Deployment gave way to the operator. + - kubectl --context {{.UPSTREAM_CTX}} -n ipam-system wait cluster.postgresql.cnpg.io/ipam-db --for=condition=Ready --timeout=420s + # Ready is the operator's opinion; the init container's is the one that + # matters. Ask the database directly, as the role and database IPAM + # actually connects as, so a cluster that reports Ready while still + # finishing bootstrap fails here rather than as a crashlooping migration. + - | + for i in $(seq 1 45); do + POD=$(kubectl --context {{.UPSTREAM_CTX}} -n ipam-system get pod \ + -l cnpg.io/cluster=ipam-db -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [ -n "$POD" ] && kubectl --context {{.UPSTREAM_CTX}} -n ipam-system exec "$POD" -c postgres -- \ + pg_isready -U ipam -d ipam >/dev/null 2>&1; then + echo "✅ postgres accepting connections as ipam/ipam"; break + fi + if [ "$i" -eq 45 ]; then echo "❌ postgres never accepted a connection"; exit 1; fi + sleep 2 + done + - echo "⏳ waiting for the IPAM apiserver (readiness probe has a 60s initial delay)" + # rollout status, not `wait pod -l`: a label selector also matches pods + # left over from a previous revision that are still terminating, and the + # wait then fails on a pod that is on its way out. + - kubectl --context {{.UPSTREAM_CTX}} -n ipam-system rollout status deploy/ipam-apiserver --timeout=300s + - echo "⏳ waiting for the aggregated APIService" + - kubectl --context {{.UPSTREAM_CTX}} wait --for=condition=Available apiservice/v1alpha1.ipam.miloapis.com --timeout=180s + - echo "✅ IPAM serving ipam.miloapis.com/v1alpha1" + + ipam-kubeconfig: + desc: "Wire the NSO manager to IPAM: mint a token for its ServiceAccount and publish it as the ipam-cluster-kubeconfig secret on the upstream cluster. The manager authenticates with this and sets the project impersonation per request; the RBAC that lets it do so ships in config/dependencies/ipam/rbac.yaml." + cmds: + - echo "🔗 wiring NSO manager -> IPAM (secret ipam-cluster-kubeconfig)" + - kubectl --context {{.UPSTREAM_CTX}} create namespace network-services-operator-system --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + # The ServiceAccount is created by config/e2e, which is applied later, so + # ensure it exists before minting a token against it. Applying the same + # object twice is harmless. + - | + kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system create serviceaccount \ + network-services-operator-controller-manager --dry-run=client -o yaml \ + | kubectl --context {{.UPSTREAM_CTX}} apply -f - + # The manager runs inside the upstream cluster, so the kubeconfig must + # name the in-cluster service address rather than the host-mapped one. + - | + CA=$(kubectl --context {{.UPSTREAM_CTX}} -n kube-system get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 | tr -d '\n') + TOKEN=$(kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system create token \ + network-services-operator-controller-manager --duration=24h) + cat > {{.TMP_DIR}}/.ipam-cluster-kubeconfig.yaml <&2 + echo " Are both clusters up? Run 'task test-infra:clusters' first." >&2 + exit 1 + fi - echo "🌐 karmada init (apiserver {{.KARMADA_APISERVER_VERSION}}, advertise 127.0.0.1:32443, cert-ip incl {{.HOST_IP}}) on {{.UPSTREAM_CLUSTER}}" - mkdir -p {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} - | @@ -429,7 +712,7 @@ tasks: - '{{.CHAINSAW}} test ./test/e2e-edge/extension-server-smoke --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml' e2e: - desc: "Run chainsaw e2e scenarios against the live two-cluster env. Pass a scenario name or path after -- (e.g. `task test-infra:e2e -- waf-enforcement`); with no arg, runs every ext-server-path scenario that targets nso-upstream/nso-downstream. Use SCENARIOS=... to override the default set." + desc: "Run chainsaw e2e scenarios against the live two-cluster env. Pass a scenario name or path after -- (e.g. `task test-infra:e2e -- waf-enforcement`); with no arg, runs every ext-server-path scenario that targets nso-upstream/nso-downstream. Use SCENARIOS=... to override the default set. NOT safe to run while anything else is touching the env. The interface suites use fixed object names in the shared ipam-e2e-* namespaces (the project routing lives on those namespace labels), and this task clears and re-seeds the cluster-scoped fixtures at its head — which also republishes the tenant RBAC. So a second invocation, or a bare ipam-fixtures / ipam-fixtures-clear / ipam-tenant-rbac run, will pull the fixtures and the tenant's access out from under a suite already in flight. Run one at a time." vars: # Scenarios authored against this env's cluster names and downstream path. # Older fixtures targeting the previous cluster names are excluded here. @@ -439,6 +722,11 @@ tasks: deps: - kubeconfigs cmds: + # Reset the cluster-scoped fixtures before the first suite. A run that + # died mid-suite leaves pools and classes that chainsaw's namespace + # teardown cannot reach, and the next run would fail on "already exists" + # rather than on what it was testing. + - task: ipam-fixtures - | set -e for s in {{.SELECTED}}; do @@ -449,13 +737,17 @@ tasks: *) dir="./test/e2e-edge/$s" ;; esac echo "🧪 chainsaw: $dir" + # --parallel 1 for the same reason test-e2e sets it: suites share + # cluster-scoped IPPools and IPClasses, so running two at once has + # them carving the same address space. {{.CHAINSAW}} test "$dir" \ + --parallel 1 \ --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml done test-e2e: - desc: "Run the full control-plane e2e suite (test/e2e) against the live two-cluster env. The suite's chainsaw aliases nso-standard/nso-infra are mapped to this env's upstream/downstream kubeconfigs at invocation. Pass a scenario path after -- to run a single test." + desc: "Run the full control-plane e2e suite (test/e2e) against the live two-cluster env. The suite's chainsaw aliases nso-standard/nso-infra are mapped to this env's upstream/downstream kubeconfigs at invocation. Pass a scenario path after -- to run a single test. NOT safe to run while anything else is touching the env. The interface suites use fixed object names in the shared ipam-e2e-* namespaces (the project routing lives on those namespace labels), and this task clears and re-seeds the cluster-scoped fixtures at its head — which also republishes the tenant RBAC. So a second invocation, or a bare ipam-fixtures / ipam-fixtures-clear / ipam-tenant-rbac run, will pull the fixtures and the tenant's access out from under a suite already in flight. Run one at a time." vars: # CLI_ARGS (after --) wins so a single test can be selected; else the whole # suite directory. @@ -467,6 +759,9 @@ tasks: deps: - kubeconfigs cmds: + # See the note in `e2e`: cluster-scoped fixtures outlive a failed run, so + # the suite starts by putting them back to a known state. + - task: ipam-fixtures - echo "🧪 chainsaw suite {{.TARGET}} (nso-standard={{.UPSTREAM_CLUSTER}}, nso-infra={{.DOWNSTREAM_CLUSTER}}); report {{.REPORT_PATH}}/{{.REPORT_NAME}}.json" - | {{.CHAINSAW}} test {{.TARGET}} \ diff --git a/config/dependencies/README.md b/config/dependencies/README.md new file mode 100644 index 00000000..18d218ee --- /dev/null +++ b/config/dependencies/README.md @@ -0,0 +1,49 @@ +# Dependencies + +Deployment configuration for services NSO depends on but does not own — other +teams' components, deployed here so the test environment can exercise the real +integration instead of a stand-in. + +One named subdirectory per dependency. + +| directory | what it is | +|---|---| +| `ipam/` | The Milo IPAM aggregated apiserver (`ipam.miloapis.com`), the project control plane NSO claims addresses from. | + +## How this differs from the neighbours + +- **`config/tools/`** — things that support the environment rather than + participate in the feature under test: cert-manager, Envoy Gateway, + external-dns, kind cluster configs. A dependency here is something NSO's own + controllers talk to at runtime; a tool is scaffolding. +- **`config/e2e/`, `config/dev/`** — NSO's *own* deployment, per environment. + +## What belongs here, and what does not + +Deployment configuration only: what it takes to stand the dependency up. If a +file exists to make a test assert something, it is test data and lives in +`test/fixtures//` instead. + +The line is easy to blur, because the only consumer of this directory today is +the test environment — which makes everything in it feel like test scaffolding. +It is not. The test: + +> If the suites were deleted tomorrow, would this file still be needed to run +> the dependency? + +For `ipam/`, `overlay/` and `patches/` survive that question and stay; the +IPClass/IPPool seeds, the `ipam-e2e-*` namespaces, and the `e2e-tenant-tester` +binding do not, and live in `test/e2e/fixtures/ipam/`. + +Identities are split on the same line. `overlay/rbac.yaml` grants what the +operator needs to assert a project; the identity the suites impersonate is +bound to that same role from `test/e2e/fixtures/ipam/rbac.yaml`. + +## Convention + +A dependency is consumed at the version the Go module graph already pins, so +the manifests deployed and the client NSO compiles against cannot drift apart. +Prefer the upstream project's own published manifests over copies kept here: +`ipam/` pulls a digest-pinned OCI bundle at task time and keeps only this repo's +additions in `overlay/` and `patches/`. The name mirrors IPAM's own +`config/dependencies/postgres-operator`. diff --git a/config/dependencies/ipam/overlay/cluster-issuer.yaml b/config/dependencies/ipam/overlay/cluster-issuer.yaml new file mode 100644 index 00000000..cc23a476 --- /dev/null +++ b/config/dependencies/ipam/overlay/cluster-issuer.yaml @@ -0,0 +1,6 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-cluster-issuer +spec: + selfSigned: {} diff --git a/config/dependencies/ipam/overlay/kustomization.yaml b/config/dependencies/ipam/overlay/kustomization.yaml new file mode 100644 index 00000000..ad70bc85 --- /dev/null +++ b/config/dependencies/ipam/overlay/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Everything this env adds to the published IPAM bundle, and nothing from it. +# Kept self-contained so `task validate-kustomizations` can build it on a clean +# checkout, where the bundle has not been fetched yet. +# +# The composed deployment — this plus the bundle's base and components, with the +# patches in ../patches — is assembled at task time by test-infra:ipam-bundle +# from ../root-kustomization.yaml. + +namespace: ipam-system + +resources: + - cluster-issuer.yaml + - secret.yaml + - postgres-cluster.yaml + - rbac.yaml diff --git a/config/dependencies/ipam/overlay/postgres-cluster.yaml b/config/dependencies/ipam/overlay/postgres-cluster.yaml new file mode 100644 index 00000000..00af77af --- /dev/null +++ b/config/dependencies/ipam/overlay/postgres-cluster.yaml @@ -0,0 +1,55 @@ +# IPAM's database, declared for the CloudNativePG operator. +# +# The operator ships with IPAM's own bundle; the Cluster does not — IPAM +# installs CNPG and leaves the database to the consumer, so this is ours to +# author. It replaces the plain Deployment this env ran before. +# +# CNPG must be fully live before this applies: the CRD has to be established +# and the webhook has to be answering, or the apply fails in a way that reads +# like a flake. task test-infra:cnpg-operator gates on exactly that. +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ipam-db + namespace: ipam-system +spec: + # One instance: a single kind node cannot honour anti-affinity across + # replicas, and a failover this env can never exercise is not worth the + # startup time. + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:17.0 + + bootstrap: + initdb: + database: ipam + owner: ipam + # Credentials come from the secret above rather than being generated, so + # the DSN IPAM reads is a constant. + secret: + name: postgres-credentials + + storage: + size: 1Gi + # Unset storageClass: take the cluster default, which on kind is the + # built-in local-path provisioner. + + # Matches what the retired Deployment ran with. synchronous_commit=off is + # safe here and meaningfully faster: the suites do many small writes, and a + # lost transaction on a database that is destroyed with the cluster costs + # nothing. + postgresql: + parameters: + synchronous_commit: "off" + max_connections: "200" + shared_buffers: 192MB + + # The memory request must exceed shared_buffers, and CNPG's webhook enforces + # it. The retired Deployment asked for 128Mi while telling Postgres to reserve + # 192MB of it, which nothing validated — the operator refuses the same spec. + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: "2" + memory: 512Mi diff --git a/config/dependencies/ipam/overlay/rbac.yaml b/config/dependencies/ipam/overlay/rbac.yaml new file mode 100644 index 00000000..4bfc4212 --- /dev/null +++ b/config/dependencies/ipam/overlay/rbac.yaml @@ -0,0 +1,77 @@ +# Two RBAC layers are required to reach IPAM as a project. +# +# 1. The caller impersonates a user plus the three iam.miloapis.com extras that +# carry the project. The kube-apiserver front proxy re-emits those extras as +# X-Remote-Extra-*, which is where IPAM reads the tenant from. +# 2. IPAM authorizes through a delegated SubjectAccessReview against the host +# apiserver, so the IMPERSONATED user needs ordinary RBAC of its own. +# +# The kind admin credential is in system:masters and is authorized for +# everything already, so only the NSO manager's ServiceAccount needs the +# impersonation grant below. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nso-ipam-impersonator +rules: + - apiGroups: [""] + resources: ["users"] + verbs: ["impersonate"] + resourceNames: ["nso-ipam-agent"] + - apiGroups: ["authentication.k8s.io"] + resources: + - "userextras/iam.miloapis.com/parent-name" + - "userextras/iam.miloapis.com/parent-type" + - "userextras/iam.miloapis.com/parent-api-group" + verbs: ["impersonate"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nso-ipam-impersonator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nso-ipam-impersonator +subjects: + - kind: ServiceAccount + name: network-services-operator-controller-manager + namespace: network-services-operator-system +--- +# ipclasses is not optional: every claim resolves its class on create, so a +# subject without it gets a 403 on every allocation. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nso-ipam-tenant +rules: + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["ipam.miloapis.com"] + resources: + - "ippools" + - "ippools/status" + - "ipclasses" + - "ipclasses/status" + - "ipclaims" + - "ipclaims/status" + - "ipallocations" + - "ipallocations/status" + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nso-ipam-tenant +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nso-ipam-tenant +subjects: + # The identity the NSO manager impersonates. The test suites impersonate a + # different one, bound to this same role from test/e2e/fixtures/ipam/rbac.yaml — + # test identities do not belong in the deployment. + - kind: User + apiGroup: rbac.authorization.k8s.io + name: nso-ipam-agent diff --git a/config/dependencies/ipam/overlay/secret.yaml b/config/dependencies/ipam/overlay/secret.yaml new file mode 100644 index 00000000..6c155c4d --- /dev/null +++ b/config/dependencies/ipam/overlay/secret.yaml @@ -0,0 +1,29 @@ +# One secret serves two consumers, so the password exists in exactly one place. +# +# IPAM reads `dsn` and passes it to both the migrate init container and the +# apiserver. CNPG reads `username`/`password` to create the application role +# when it bootstraps the cluster, which is why the type is basic-auth — CNPG +# rejects anything else for a bootstrap secret. Extra keys are ignored by both. +# +# Bootstrapping from a secret we author, rather than letting CNPG generate one, +# is what keeps the DSN a static string. CNPG's generated `-app` secret has a +# random password and would force the DSN to be assembled at deploy time from +# values that change on every rebuild. +# +# The host is the read-write service CNPG creates for the cluster below: +# -rw. sslmode=disable because CNPG serves TLS with its own internal +# CA, which the client would have to be taught to trust for no benefit here. +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials + namespace: ipam-system + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/component: database + app.kubernetes.io/part-of: ipam.miloapis.com +type: kubernetes.io/basic-auth +stringData: + username: "ipam" + password: "devpassword" + dsn: "postgres://ipam:devpassword@ipam-db-rw.ipam-system.svc.cluster.local:5432/ipam?sslmode=disable" diff --git a/config/dependencies/ipam/patches/apiservice-patch.yaml b/config/dependencies/ipam/patches/apiservice-patch.yaml new file mode 100644 index 00000000..65d4b02f --- /dev/null +++ b/config/dependencies/ipam/patches/apiservice-patch.yaml @@ -0,0 +1,16 @@ +--- +# Trust the CA that signs the serving certificate, rather than skipping +# verification. +# +# cert-manager's cainjector reads the ipam-ca Certificate and writes its CA into +# spec.caBundle. That is the same CA the CSI driver issues each pod's serving +# cert from, so aggregation verifies a real chain. This replaced +# insecureSkipTLSVerify: true, which was needed only while the serving cert was +# self-signed and chained to nothing — the bundle's own component already sets +# that field to false, so this patch no longer has to touch it. +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1alpha1.ipam.miloapis.com + annotations: + cert-manager.io/inject-ca-from: ipam-system/ipam-ca diff --git a/config/dependencies/ipam/patches/deployment-patch.yaml b/config/dependencies/ipam/patches/deployment-patch.yaml new file mode 100644 index 00000000..e28cc10c --- /dev/null +++ b/config/dependencies/ipam/patches/deployment-patch.yaml @@ -0,0 +1,24 @@ +--- +# The image is pulled from ghcr by digest, so IfNotPresent: the node fetches it +# once and reuses it. Never would fail here — nothing side-loads the image any +# more, and a digest-pinned reference cannot go stale, so there is nothing for +# Always to re-check. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ipam-apiserver +spec: + template: + spec: + initContainers: + - name: migrate + imagePullPolicy: IfNotPresent + containers: + - name: apiserver + imagePullPolicy: IfNotPresent + # No Milo quota backend in the kind/test-infra cluster — the + # quota.miloapis.com CRDs don't exist, so the quota plugin's informers + # would never sync and readyz would hang. Disable quota here. + env: + - name: ENABLE_QUOTA + value: "false" diff --git a/config/dependencies/ipam/patches/networkpolicy-apiserver-egress-patch.yaml b/config/dependencies/ipam/patches/networkpolicy-apiserver-egress-patch.yaml new file mode 100644 index 00000000..47e0c332 --- /dev/null +++ b/config/dependencies/ipam/patches/networkpolicy-apiserver-egress-patch.yaml @@ -0,0 +1,26 @@ +# Let the apiserver reach the kube-apiserver by address rather than by +# namespace. +# +# The base policy allows egress to :443/:6443 selected by +# namespaceSelector (kube-system, default). That never matches in kind: a +# connection to kubernetes.default.svc is DNATed to the node's own address, +# which belongs to no pod and therefore to no namespace, so the rule selects +# nothing and the traffic is dropped. +# +# The symptom is not a connection error. The apiserver comes up, serves TLS, +# and then sits at "informer-sync failed: 2 informers not started yet: +# [*v1.PriorityLevelConfiguration *v1.FlowSchema]" forever — readiness never +# passes and the APIService stays MissingEndpoints. It presents as a flake +# because a pod that happens to start before kindnet programs the policy syncs +# its informers and stays healthy for the rest of its life. +- op: add + path: /spec/egress/- + value: + to: + - ipBlock: + cidr: 0.0.0.0/0 + ports: + - protocol: TCP + port: 6443 + - protocol: TCP + port: 443 diff --git a/config/dependencies/ipam/patches/single-replica-patch.yaml b/config/dependencies/ipam/patches/single-replica-patch.yaml new file mode 100644 index 00000000..0d749b72 --- /dev/null +++ b/config/dependencies/ipam/patches/single-replica-patch.yaml @@ -0,0 +1,14 @@ +# One replica, env-only. The IPAM base runs two so a kubelet eviction during a +# claim CREATE has a second pod to absorb the request — a real concern in +# production and one this single-node kind cluster cannot exercise anyway. +# +# It is not merely unnecessary here, it is actively broken: the second replica +# reliably hangs with "informer-sync failed: 2 informers not started yet: +# [*v1.FlowSchema *v1.PriorityLevelConfiguration]" and never becomes Ready, +# which wedges the readiness gate while the first replica serves fine. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ipam-apiserver +spec: + replicas: 1 diff --git a/config/dependencies/ipam/root-kustomization.yaml b/config/dependencies/ipam/root-kustomization.yaml new file mode 100644 index 00000000..fc5e2c08 --- /dev/null +++ b/config/dependencies/ipam/root-kustomization.yaml @@ -0,0 +1,70 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The composed IPAM deployment: the published bundle plus this env's additions. +# +# NOT named kustomization.yaml on purpose. It only builds inside the staging +# directory test-infra:ipam-bundle assembles, where `bundle/` is the extracted +# OCI artifact; in the repo it would be a kustomization that cannot build, and +# `task validate-kustomizations` discovers every kustomization.yaml there is. +# The self-contained half lives in overlay/ and is validated there. + +namespace: ipam-system + +resources: + - bundle/base + - overlay + +components: + - bundle/components/namespace + - bundle/components/api-registration + # Gives the base's CSI volume something it can actually issue from. The base + # points it at selfsigned-cluster-issuer, and a SelfSigned issuer signs every + # certificate with a freshly generated key — so each pod would get an + # unrelated cert chained to nothing and the APIService would have no stable CA + # to trust. This component bootstraps a real CA from that selfsigned issuer + # and repoints the volume at it, which is what makes per-pod issuance work. + # + # It lives here rather than in overlay/ because it comes from the fetched + # bundle, which does not exist on a clean checkout — overlay/ has to stay + # self-contained so validate-kustomizations can build it. + - bundle/components/cert-manager-ca + +# Pulled from ghcr by digest, not by the bundle's own tag: the release workflow +# republishes on every push, so a tag can move under us. The tag is recorded for +# legibility. Update it, the digest, and the bundle pins in +# Taskfile.test-infra.yml together. +# +# This is an index digest, so the node resolves its own platform. Genuinely +# multi-arch since upstream 13bd2c8: before that, both platform entries of the +# index pointed at one layer holding an x86-64 binary, which ran under emulation +# on arm64 and panicked inside pgx mid-suite. The two platforms now carry +# distinct layers and the arm64 entry is a real AArch64 binary — verified by +# reading the ELF header, not by trusting the manifest, because the manifest is +# exactly what was wrong before. +images: + - name: ghcr.io/milo-os/ipam + newName: ghcr.io/milo-os/ipam + # tag v0.0.0-13bd2c8 + digest: sha256:4fb11adf3f748d41f5e3d7f9d3357eb7aafe211e356297a0d845d579dacffe8f + +patches: + - path: patches/apiservice-patch.yaml + - path: patches/deployment-patch.yaml + target: + kind: Deployment + name: ipam-apiserver + - path: patches/single-replica-patch.yaml + target: + kind: Deployment + name: ipam-apiserver + - path: patches/networkpolicy-apiserver-egress-patch.yaml + target: + kind: NetworkPolicy + name: ipam-apiserver + +labels: + - includeSelectors: false + includeTemplates: true + pairs: + environment: test-infra diff --git a/config/dependencies/postgres-operator/kustomization.yaml b/config/dependencies/postgres-operator/kustomization.yaml new file mode 100644 index 00000000..c190dc6a --- /dev/null +++ b/config/dependencies/postgres-operator/kustomization.yaml @@ -0,0 +1,43 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The CloudNativePG operator, which runs IPAM's database. +# +# IPAM's own bundle installs CNPG at dependencies/postgres-operator as a Flux +# HelmRelease. This env runs no Flux, so it takes the same chart through +# kustomize --enable-helm — the way cert-manager is installed here — and skips +# the HelmRelease. The name mirrors IPAM's so the correspondence is obvious. +# +# The operator installs CRDs and a webhook; nothing may apply a Cluster until +# all three are live. task test-infra:cnpg-operator gates on that. +# +# Pinned rather than floating (IPAM's HelmRelease tracks "0.x"): a CI run should +# not change underneath us because upstream published a release. + +resources: + - namespace.yaml + +helmCharts: + - name: cloudnative-pg + repo: https://cloudnative-pg.github.io/charts + # chart 0.29.0 == CloudNativePG 1.30.0 + version: 0.29.0 + releaseName: cnpg + namespace: cnpg-system + # The chart declares kubeVersion >=1.29. Rendering happens offline, where + # helm assumes a default Kubernetes far older than that and refuses the + # chart — so state the version this env actually runs (K8S_NODE_IMAGE in + # Taskfile.test-infra.yml). Bump both together. + kubeVersion: 1.35.5 + valuesInline: + # One operator replica on a single-node kind; the default requests are + # sized for a real cluster and crowd the node alongside everything else + # this env runs. + replicaCount: 1 + resources: + requests: + cpu: 50m + memory: 100Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/config/dependencies/postgres-operator/namespace.yaml b/config/dependencies/postgres-operator/namespace.yaml new file mode 100644 index 00000000..8deac4c7 --- /dev/null +++ b/config/dependencies/postgres-operator/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cnpg-system diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index 9e9744c4..3b02e6c0 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -36,10 +36,24 @@ patches: volumeMounts: - name: downstream-cluster-kubeconfig mountPath: /etc/downstream-cluster + - name: ipam-cluster-kubeconfig + mountPath: /etc/ipam-cluster volumes: - name: downstream-cluster-kubeconfig secret: secretName: downstream-cluster-kubeconfig + # Optional only so a missing secret fails in the manager rather + # than in the kubelet. It does NOT make the manager tolerate the + # secret's absence: with networkInterface.enabled true, loading + # the IPAM kubeconfig is part of setup, and failing it exits the + # process — so the manager crashloops and takes every other + # controller with it. Deliberate: a clear "unable to load IPAM + # kubeconfig" beats a manager that comes up half-functional. + # test-infra:up creates the secret before prepare-upstream. + - name: ipam-cluster-kubeconfig + secret: + secretName: ipam-cluster-kubeconfig + optional: true replacements: - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs diff --git a/hack/gen-ipam-impersonation-kubeconfig.sh b/hack/gen-ipam-impersonation-kubeconfig.sh new file mode 100755 index 00000000..7ede8ac9 --- /dev/null +++ b/hack/gen-ipam-impersonation-kubeconfig.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Write a kubeconfig whose contexts drive the IPAM aggregated apiserver as a +# specific project. +# +# IPAM scopes storage per tenant from three UserInfo.Extra keys +# (iam.miloapis.com/parent-api-group, parent-type, parent-name). Milo's front +# gate normally supplies them as X-Remote-Extra-* requestheader extras. A +# kubeconfig's as-user-extra becomes Impersonate-Extra-*, which the front proxy +# re-emits as X-Remote-Extra-* — the same place IPAM reads from. A request +# carrying none of the three reads nothing and its writes are refused, so every +# fixture and every assertion has to go through one of these contexts. +# +# Adapted from github.com/milo-os/ipam test/e2e/lib/gen-impersonation-kubeconfig.sh +# at ref 20865afe018c. The cross-project group is dropped: this env seeds no +# shared pools. +# +# Usage: gen-ipam-impersonation-kubeconfig.sh [project...] + +set -euo pipefail + +OUT="${1:?usage: gen-ipam-impersonation-kubeconfig.sh [project...]}" +SRC_CTX="${2:?source context required}" +shift 2 +PROJECTS=("$@") +if [ ${#PROJECTS[@]} -eq 0 ]; then + PROJECTS=(project-alpha project-beta) +fi + +TENANT_USER="e2e-tenant-tester" +PARENT_API_GROUP="resourcemanager.miloapis.com" +PARENT_TYPE="Project" + +# The output is assembled by hand rather than round-tripped through +# `kubectl config view`: kubectl v1.36 drops as / as-user-extra when it +# re-serialises a kubeconfig, which would silently remove the impersonation. +base="$(mktemp)" +trap 'rm -f "$base"' EXIT +kubectl --context "$SRC_CTX" config view --minify --flatten >"$base" + +BASE_CLUSTER="$(KUBECONFIG="$base" kubectl config view -o jsonpath="{.contexts[?(@.name=='${SRC_CTX}')].context.cluster}")" +BASE_USER="$(KUBECONFIG="$base" kubectl config view -o jsonpath="{.contexts[?(@.name=='${SRC_CTX}')].context.user}")" + +C_SERVER="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.server}")" +C_CA="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.certificate-authority-data}")" +C_INSECURE="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.insecure-skip-tls-verify}")" + +B_CERT="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.client-certificate-data}")" +B_KEY="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.client-key-data}")" +B_TOKEN="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.token}")" + +emit_base_credentials() { + [ -n "$B_CERT" ] && printf '%s\n' " client-certificate-data: ${B_CERT}" + [ -n "$B_KEY" ] && printf '%s\n' " client-key-data: ${B_KEY}" + [ -n "$B_TOKEN" ] && printf '%s\n' " token: ${B_TOKEN}" + return 0 +} + +{ + printf '%s\n' "apiVersion: v1" + printf '%s\n' "kind: Config" + printf '%s\n' "current-context: tenant-platform" + printf '%s\n' "clusters:" + printf '%s\n' "- name: ${BASE_CLUSTER}" + printf '%s\n' " cluster:" + printf '%s\n' " server: ${C_SERVER}" + [ -n "$C_CA" ] && printf '%s\n' " certificate-authority-data: ${C_CA}" + [ "$C_INSECURE" = "true" ] && printf '%s\n' " insecure-skip-tls-verify: true" + printf '%s\n' "users:" + printf '%s\n' "- name: ${BASE_USER}" + printf '%s\n' " user:" + emit_base_credentials + for proj in "${PROJECTS[@]}"; do + printf '%s\n' "- name: tenant-${proj}-as" + printf '%s\n' " user:" + emit_base_credentials + printf '%s\n' " as: ${TENANT_USER}" + printf '%s\n' " as-user-extra:" + printf '%s\n' " iam.miloapis.com/parent-api-group:" + printf '%s\n' " - ${PARENT_API_GROUP}" + printf '%s\n' " iam.miloapis.com/parent-type:" + printf '%s\n' " - ${PARENT_TYPE}" + printf '%s\n' " iam.miloapis.com/parent-name:" + printf '%s\n' " - ${proj}" + done + printf '%s\n' "contexts:" + for proj in "${PROJECTS[@]}"; do + printf '%s\n' "- name: tenant-${proj}" + printf '%s\n' " context:" + printf '%s\n' " cluster: ${BASE_CLUSTER}" + printf '%s\n' " user: tenant-${proj}-as" + done + printf '%s\n' "- name: tenant-platform" + printf '%s\n' " context:" + printf '%s\n' " cluster: ${BASE_CLUSTER}" + printf '%s\n' " user: ${BASE_USER}" +} >"$OUT" + +echo "wrote ${OUT} (source context ${SRC_CTX})" +echo " contexts: tenant-platform, $(printf 'tenant-%s ' "${PROJECTS[@]}")" diff --git a/hack/ipam-clear-fixtures.sh b/hack/ipam-clear-fixtures.sh new file mode 100755 index 00000000..8d256dbc --- /dev/null +++ b/hack/ipam-clear-fixtures.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Delete every IPAM object in the fixture projects. +# +# IPPool and IPClass are cluster-scoped, so chainsaw's namespace teardown never +# reaches them. A suite that dies partway leaves its pools and classes behind, +# and the next run fails on "already exists" for a reason unrelated to what it +# was testing. +# +# Order matters: a bound allocation refuses deletion of the pool it came from, +# and a class with pools offering it refuses deletion too. Claims first, +# allocations next, then pools, then classes. +# +# This script exists to guarantee a clean slate, so it must never report one it +# did not produce. It proves it can reach each tenant before deleting anything, +# and it verifies all four kinds are empty afterwards. +# +# Adapted from github.com/milo-os/ipam test/e2e/lib/clear-tenant-fixtures.sh at +# ref 20865afe018c. +# +# Usage: ipam-clear-fixtures.sh [project...] + +set -euo pipefail + +KCFG="${1:?usage: ipam-clear-fixtures.sh [project...]}" +shift +PROJECTS=("$@") +if [ ${#PROJECTS[@]} -eq 0 ]; then + PROJECTS=(project-alpha project-beta) +fi + +DELETE_TIMEOUT="${IPAM_CLEAR_TIMEOUT:-60s}" +KINDS="ipclaims ipallocations ippools ipclasses" + +if [ ! -f "$KCFG" ]; then + echo "❌ no impersonation kubeconfig at ${KCFG}" >&2 + exit 1 +fi + +k() { + KUBECONFIG="$KCFG" kubectl --context "$1" "${@:2}" +} + +# Positive control. Reading a type that always resolves proves the aggregated +# API is reachable AND that this context's impersonation is accepted. Without +# it, an unreachable cluster or a wrong context name would make every delete +# below a no-op and this script would report a clean tenant it never touched. +verify_reachable() { + local ctx="$1" out + if ! out="$(k "$ctx" get ipclasses 2>&1)"; then + echo "❌ cannot reach IPAM as ${ctx}; refusing to report a clean tenant" >&2 + echo " ${out}" >&2 + exit 1 + fi +} + +# A delete that fails because the object is already gone is success. A pool +# pinned by a child that has not been deleted yet is expected mid-loop and is +# resolved by the next pass — the residue check at the end is what decides +# whether clearing actually worked. Anything else is surfaced rather than +# swallowed. +attempt_delete() { + local ctx="$1" kind="$2" out rc + set +e + out="$(k "$ctx" delete "$kind" --all --all-namespaces --ignore-not-found \ + --timeout="$DELETE_TIMEOUT" 2>&1)" + rc=$? + set -e + if [ "$rc" -ne 0 ] && ! printf '%s' "$out" | grep -qiE 'not found|no matches for|cannot delete IPPool'; then + echo " ⚠️ deleting ${kind} in ${ctx}: ${out}" >&2 + fi +} + +residue() { + local ctx="$1" kind="$2" + k "$ctx" get "$kind" --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {end}' 2>/dev/null || true +} + +for proj in "${PROJECTS[@]}"; do + ctx="tenant-${proj}" + verify_reachable "$ctx" + + attempt_delete "$ctx" ipclaims + attempt_delete "$ctx" ipallocations + + # Pools need more than one pass. A cascade leaves child pools carved out of + # their parent, and the parent refuses to go while a carve is outstanding — + # but `delete --all` works through the list in name order, so a parent is + # usually attempted before the children that pin it. Each pass frees one + # level of the chain; the chain is at most a few deep. + for _ in 1 2 3 4; do + [ -z "$(residue "$ctx" ippools)" ] && break + attempt_delete "$ctx" ippools + done + + attempt_delete "$ctx" ipclasses + + # The authoritative check, over every kind. Pools and classes block the next + # run with "already exists"; a stranded claim or allocation is worse, because + # a bound allocation refuses deletion of the pool it came from and wedges the + # run after that one. + left="" + for kind in $KINDS; do + found="$(residue "$ctx" "$kind")" + [ -n "$found" ] && left="${left}${kind}: ${found}"$'\n' + done + if [ -n "$left" ]; then + echo "❌ ${proj} still holds IPAM fixtures after clearing:" >&2 + printf '%s' "$left" >&2 + exit 1 + fi + + echo "cleared IPAM fixtures in ${proj}" +done diff --git a/hack/ipam-fetch-bundle.sh b/hack/ipam-fetch-bundle.sh new file mode 100755 index 00000000..902ea2b0 --- /dev/null +++ b/hack/ipam-fetch-bundle.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Fetch the published IPAM manifest bundle from an OCI registry and extract it. +# +# The bundle is a flux OCI artifact: a single layer of media type +# application/vnd.cncf.flux.content.v1.tar+gzip carrying the repo's config/ +# tree. It has no org.opencontainers.image.title annotation, so `oras pull` +# extracts nothing; reading the layer blob directly works with any OCI client +# and needs no flux-specific tooling. We use crane because it installs with +# `go install` like every other pinned tool here, where the flux CLI's own +# go.mod replace directives make that fail. +# +# Pull is anonymous — the repository is public, so CI needs no registry +# credentials. +# +# Usage: +# ipam-fetch-bundle.sh + +set -euo pipefail + +CRANE="${1:?crane binary path required}" +CRANE_VERSION="${2:?crane version required}" +REPO="${3:?bundle repository required}" +DIGEST="${4:?bundle digest required}" +COMMIT="${5:?expected commit required}" +OUT="${6:?output directory required}" + +# Install crane on demand rather than making the caller remember a prep step. +# `up` does not depend on the tools task, and the workflow that runs the e2e +# suite installs its binaries from an explicit list, so requiring crane to be +# there already means it works on whichever machine happened to install it and +# nowhere else. Same shape as the tools task's own per-binary guard. +if [ ! -x "$CRANE" ]; then + echo "🔧 installing crane ${CRANE_VERSION} into $(dirname "$CRANE")" >&2 + if ! GOBIN="$(cd "$(dirname "$CRANE")" && pwd)" \ + go install "github.com/google/go-containerregistry/cmd/crane@${CRANE_VERSION}" >&2; then + echo "❌ could not install crane ${CRANE_VERSION}; needs Go and network access" >&2 + exit 1 + fi +fi + +if [ ! -x "$CRANE" ]; then + echo "❌ no crane binary at ${CRANE} after installing" >&2 + exit 1 +fi + +# ghcr.io occasionally times out on the first dial. A registry read is now on +# the critical path of every `up`, so retry rather than fail the whole bring-up +# on one flaky connection. +retry() { + local attempt=1 + until "$@"; do + if [ "$attempt" -ge 3 ]; then + echo "❌ '$*' failed after ${attempt} attempts" >&2 + return 1 + fi + echo " retrying (${attempt}/3) after registry error" >&2 + attempt=$((attempt + 1)) + sleep $((attempt * 3)) + done +} + +manifest="$(retry "$CRANE" manifest "${REPO}@${DIGEST}")" + +field() { + printf '%s' "$manifest" | python3 -c "import json,sys; print($1)" +} + +# The artifact must be the same commit go.mod pins, or the manifests and the Go +# types NSO compiles against have drifted apart — which is exactly the failure +# the digest pin exists to prevent, so it is checked rather than assumed. +revision="$(field "json.load(sys.stdin).get('annotations',{}).get('org.opencontainers.image.revision','')")" +case "$revision" in + *"$COMMIT"*) ;; + *) + echo "❌ bundle revision '${revision}' does not carry the pinned commit ${COMMIT}" >&2 + exit 1 + ;; +esac + +layer="$(field "json.load(sys.stdin)['layers'][0]['digest']")" + +rm -rf "$OUT" +mkdir -p "$OUT" +# Buffer the blob before extracting: piping crane straight into tar hides a +# mid-transfer registry failure behind tar's exit status. +blob="$(mktemp)" +trap 'rm -f "$blob"' EXIT +retry "$CRANE" blob "${REPO}@${layer}" > "$blob" +tar xz -C "$OUT" < "$blob" + +if [ ! -d "$OUT/base" ]; then + echo "❌ extracted bundle has no base/ directory" >&2 + exit 1 +fi + +echo "$revision" diff --git a/hack/module-dir.sh b/hack/module-dir.sh new file mode 100755 index 00000000..eeb2f03e --- /dev/null +++ b/hack/module-dir.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Print the source directory of a Go module in the build list, downloading it +# into the module cache first if it is not already extracted there. +# +# `go list -m -f {{.Dir}}` does NOT download. On a cold module cache it prints +# an empty line and exits 0 — a silent failure, which is the dangerous part: +# interpolated into a path it produces something like "/config/crd/bases/..." +# with a leading slash, and the error surfaces as a missing file somewhere +# unrelated to the actual cause. That is exactly what broke CI, on runners whose +# cache happened to be cold, while passing everywhere the cache was warm. +# +# Usage: module-dir.sh +# e.g. module-dir.sh go.miloapis.com/dns-operator + +set -euo pipefail + +MODULE="${1:?usage: module-dir.sh }" + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$repo_root" + +resolve() { + # Discard stderr: a cold cache makes `go list` chatter about downloading the + # toolchain, which would otherwise end up in the path. + go list -m -f '{{.Dir}}' "$MODULE" 2>/dev/null || true +} + +dir="$(resolve)" + +if [ -z "$dir" ]; then + echo "module ${MODULE} is not extracted in the module cache; downloading" >&2 + if ! go mod download "$MODULE" >&2; then + echo "❌ could not download ${MODULE}." >&2 + echo " Either it is not in this module's build list — \`go list -m ${MODULE}\`" >&2 + echo " should name it — or the module proxy is unreachable." >&2 + exit 1 + fi + dir="$(resolve)" +fi + +if [ -z "$dir" ]; then + echo "❌ could not resolve a directory for ${MODULE} after downloading it." >&2 + echo " Is it in the build list? \`go list -m ${MODULE}\` should name it." >&2 + exit 1 +fi + +if [ ! -d "$dir" ]; then + echo "❌ ${MODULE} resolved to ${dir}, which is not a directory" >&2 + exit 1 +fi + +echo "$dir" diff --git a/test/e2e/fixtures/ipam/RANGES.md b/test/e2e/fixtures/ipam/RANGES.md new file mode 100644 index 00000000..6cb38427 --- /dev/null +++ b/test/e2e/fixtures/ipam/RANGES.md @@ -0,0 +1,83 @@ +# Fixture range allocation + +`IPPool` and `IPClass` are cluster-scoped and every suite in a project shares +them, so root pool CIDRs must be disjoint — a root pool that overlaps another +in the same project is refused with 409, and chainsaw runs suites concurrently +within one project. The enforcing script upstream (`hack/verify-fixture-ranges.sh`) +does not exist at the pinned ref, so the discipline is this table. Add a range +here before you add a pool. + +| project | pool | CIDR | class it backs | +|---|---|---|---| +| project-alpha | `datum-network-v6-root` | `2001:db8:a000::/36` | `datum-network-v6` | +| project-alpha | `datum-endpoint-v4-root` | `10.128.0.0/16` | `datum-endpoint-v4` | +| project-alpha | `datum-public-v4-root` | `198.51.100.0/24` | `datum-public-v4` | +| project-beta | `datum-network-v6-root` | `2001:db8:b000::/36` | `datum-network-v6` | +| project-beta | `datum-endpoint-v4-root` | `10.129.0.0/16` | `datum-endpoint-v4` | +| project-beta | `datum-public-v4-root` | `203.0.113.0/24` | `datum-public-v4` | + +Class names are deliberately IDENTICAL across the two projects: a controller +routing by project must reach different address space through the same class +name, and identical names are what makes a routing bug visible. + +## Default classes + +A claim naming no class but setting `spec.ipFamily` resolves through +`ipam.miloapis.com/is-default-class`. Exactly one class per family per project +carries it: + +| project | IPv4 default | IPv6 default | +|---|---|---| +| project-alpha | `datum-endpoint-v4` | `datum-endpoint-v6` | +| project-beta | `datum-endpoint-v4` | `datum-endpoint-v6` | + +The annotation goes on the LEAF of the IPv6 chain, not its root. Resolution +returns a class and allocation proceeds from there, so annotating +`datum-network-v6` would hand a claim a `/48` where an endpoint block was +wanted — and report it as a successful allocation. + +`datum-public-v4` is deliberately NOT annotated: it is the named-class case and +must be reachable only through an explicit class name. Two defaults for one +family is not an error — IPAM lists the annotated classes `ORDER BY key` within +the project and takes the first whose family matches, silently ignoring the +rest — so a stray second annotation is a fixture that looks like it works. + +## The IPv6 chain + +`datum-network-v6` (`poolPer: [network]`, `/48`) + → `datum-subnet-v6` (`poolPer: [network, location]`, `/64`) + → `datum-endpoint-v6` (leaf, `/96`) + +Only the root class is backed by an operator-authored pool. The `/48` and `/64` +pools are provisioned by the allocator on first claim. A claim of +`datum-endpoint-v6` must therefore carry both scope roles, `network` and +`location`; one missing a role is rejected rather than widened. + +`datum-subnet-v6` repeats `network` in its `poolPer` on purpose. Pool identity +is keyed on (class name, scope digest) alone, with no reference to the parent +chain, so a subnet class scoped only by location would hand two networks in one +location the same pool. + +## Host addresses vs blocks + +A host-address class is one whose `allowedPrefixLengths` pins min == max at the +family's full width — 32 for IPv4, 128 for IPv6. The two IPv4 classes here are +host-address classes; every class in the IPv6 chain hands out blocks. + +**At the pinned IPAM ref, nothing ever writes `status.address`.** The field +exists on IPClaim and IPAllocation and round-trips through conversion, but no +code path populates it, so it reads empty for host-address classes too. A +host address arrives as a `/32` in `status.allocatedCIDR` and a consumer must +take it from there — reading `status.address` gets `""` and looks like a claim +that did not bind. + +## Scope roles + +Free strings, compared and never interpreted. The conventional pairs: + +| role | apiGroup | kind | +|---|---|---| +| `network` | `networking.datumapis.com` | `Network` | +| `location` | `networking.datumapis.com` | `Location` | + +Scope values are object NAMES, not UIDs. diff --git a/test/e2e/fixtures/ipam/README.md b/test/e2e/fixtures/ipam/README.md new file mode 100644 index 00000000..0ddb1125 --- /dev/null +++ b/test/e2e/fixtures/ipam/README.md @@ -0,0 +1,25 @@ +# IPAM test fixtures + +Test data for the suites that exercise NetworkInterfaceClaim allocation against +a real IPAM. None of it is needed to deploy IPAM — that lives in +`config/dependencies/ipam/`. + +This directory holds no `chainsaw-test.yaml`, so chainsaw walks past it: it +recurses looking for test files and ignores directories without one, the same +way it ignores the `networkbinding/` grouping directory. + +| file | what it is | +|---|---| +| `RANGES.md` | The range allocation per project, the class model, and the rules that keep concurrent suites from colliding. **Read this before adding a pool.** | +| `project-alpha/`, `project-beta/` | `IPClass` and `IPPool` seeds. Identical class names in both projects, disjoint address space, so a controller routing to the wrong project allocates from visibly wrong space instead of quietly succeeding. | +| `namespaces.yaml` | The `ipam-e2e-*` namespaces a claim resolves its project from, carrying the real encoded `meta.datumapis.com/upstream-cluster-name` label — plus one deliberately without it, for the fail-closed case. | +| `rbac.yaml` | Binds the identity the suites impersonate to the operator's own tenant role. | + +Applied by `task test-infra:ipam-fixtures` and `task test-infra:ipam-namespaces`, +both of which run as part of `test-infra:up` and again at the head of every +suite-running task. + +These are seeded, not asserted on directly: they are the address space the +suites allocate from. `IPPool` and `IPClass` are cluster-scoped, so chainsaw's +namespace teardown never reaches them — `task test-infra:ipam-fixtures-clear` +is what stops a run that died mid-suite from poisoning the next one. diff --git a/test/e2e/fixtures/ipam/namespaces.yaml b/test/e2e/fixtures/ipam/namespaces.yaml new file mode 100644 index 00000000..e48e9421 --- /dev/null +++ b/test/e2e/fixtures/ipam/namespaces.yaml @@ -0,0 +1,58 @@ +# Namespaces carrying the labels the mapped-namespace strategy really writes, +# so a scenario exercises the production decode path rather than a hand-written +# approximation of it. +# +# Applied to BOTH clusters. The label is written on downstream namespaces in +# production, but the consumer here is the NetworkInterfaceClaim reconciler: +# it runs on the upstream cell and resolveProject reads the namespace of the +# CLAIM through the cell client. Claims live upstream, so upstream is where +# these must exist for a project to resolve at all. +# +# meta.datumapis.com/upstream-cluster-name is ENCODED: a "cluster-" prefix with +# every "/" written as "_" (see internal/downstreamclient/mappednamespace.go, +# UpstreamClusterNameFromLabel). A fixture that writes the bare cluster name +# passes on its own and fails against the real path. +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-alpha + labels: + meta.datumapis.com/upstream-cluster-name: cluster-project-alpha + meta.datumapis.com/upstream-namespace: default +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-beta + labels: + meta.datumapis.com/upstream-cluster-name: cluster-project-beta + meta.datumapis.com/upstream-namespace: default +--- +# What this env's single-cluster provider actually stamps: the provider engages +# one cluster named "single". +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-single + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + meta.datumapis.com/upstream-namespace: default +--- +# Pre-#196 encoding: the cluster name carried a leading slash, so "/project-alpha" +# encodes to "cluster-_project-alpha" and must still decode to "project-alpha". +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-legacy + labels: + meta.datumapis.com/upstream-cluster-name: cluster-_project-alpha + meta.datumapis.com/upstream-namespace: default +--- +# No cluster-name label at all. Anything resolving a project from a downstream +# namespace must fail closed here, not fall back to a default project. +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-unlabeled + labels: + meta.datumapis.com/upstream-namespace: default diff --git a/test/e2e/fixtures/ipam/project-alpha/classes.yaml b/test/e2e/fixtures/ipam/project-alpha/classes.yaml new file mode 100644 index 00000000..9536a955 --- /dev/null +++ b/test/e2e/fixtures/ipam/project-alpha/classes.yaml @@ -0,0 +1,89 @@ +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-network-v6 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + poolPer: + - network + defaultPrefixLength: 48 + allowedPrefixLengths: + min: 48 + max: 48 + routing: + external: Aggregate +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-subnet-v6 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + parentClassName: datum-network-v6 + poolPer: + - network + - location + defaultPrefixLength: 64 + allowedPrefixLengths: + min: 64 + max: 64 +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-endpoint-v6 + annotations: + # The default class for this family: a claim naming no class but + # setting ipFamily resolves to this one. + ipam.miloapis.com/is-default-class: "true" + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + parentClassName: datum-subnet-v6 + defaultPrefixLength: 96 + allowedPrefixLengths: + min: 96 + max: 96 + reclaimPolicy: Delete + routing: + internal: Host +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-endpoint-v4 + annotations: + # The default class for this family: a claim naming no class but + # setting ipFamily resolves to this one. + ipam.miloapis.com/is-default-class: "true" + labels: + nso-fixture: "true" +spec: + ipFamily: IPv4 + allowedPrefixLengths: + min: 32 + max: 32 + reclaimPolicy: Delete + routing: + internal: Host +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-public-v4 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv4 + allowedPrefixLengths: + min: 32 + max: 32 + reclaimPolicy: Delete + routing: + internal: Host + external: Aggregate diff --git a/test/e2e/fixtures/ipam/project-alpha/pools.yaml b/test/e2e/fixtures/ipam/project-alpha/pools.yaml new file mode 100644 index 00000000..5ea74a3c --- /dev/null +++ b/test/e2e/fixtures/ipam/project-alpha/pools.yaml @@ -0,0 +1,52 @@ +# Ranges are registered in ../RANGES.md. Root pools of one project may not +# overlap. +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-network-v6-root + labels: + nso-fixture: "true" +spec: + cidr: 2001:db8:a000::/36 + ipFamily: IPv6 + visibility: consumer + classNames: + - datum-network-v6 + allocation: + minPrefixLength: 48 + maxPrefixLength: 48 + strategy: FirstFit +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-endpoint-v4-root + labels: + nso-fixture: "true" +spec: + cidr: 10.128.0.0/16 + ipFamily: IPv4 + visibility: consumer + classNames: + - datum-endpoint-v4 + allocation: + minPrefixLength: 32 + maxPrefixLength: 32 + strategy: FirstFit +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-public-v4-root + labels: + nso-fixture: "true" +spec: + cidr: 198.51.100.0/24 + ipFamily: IPv4 + visibility: consumer + classNames: + - datum-public-v4 + allocation: + minPrefixLength: 32 + maxPrefixLength: 32 + strategy: FirstFit diff --git a/test/e2e/fixtures/ipam/project-beta/classes.yaml b/test/e2e/fixtures/ipam/project-beta/classes.yaml new file mode 100644 index 00000000..9536a955 --- /dev/null +++ b/test/e2e/fixtures/ipam/project-beta/classes.yaml @@ -0,0 +1,89 @@ +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-network-v6 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + poolPer: + - network + defaultPrefixLength: 48 + allowedPrefixLengths: + min: 48 + max: 48 + routing: + external: Aggregate +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-subnet-v6 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + parentClassName: datum-network-v6 + poolPer: + - network + - location + defaultPrefixLength: 64 + allowedPrefixLengths: + min: 64 + max: 64 +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-endpoint-v6 + annotations: + # The default class for this family: a claim naming no class but + # setting ipFamily resolves to this one. + ipam.miloapis.com/is-default-class: "true" + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + parentClassName: datum-subnet-v6 + defaultPrefixLength: 96 + allowedPrefixLengths: + min: 96 + max: 96 + reclaimPolicy: Delete + routing: + internal: Host +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-endpoint-v4 + annotations: + # The default class for this family: a claim naming no class but + # setting ipFamily resolves to this one. + ipam.miloapis.com/is-default-class: "true" + labels: + nso-fixture: "true" +spec: + ipFamily: IPv4 + allowedPrefixLengths: + min: 32 + max: 32 + reclaimPolicy: Delete + routing: + internal: Host +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-public-v4 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv4 + allowedPrefixLengths: + min: 32 + max: 32 + reclaimPolicy: Delete + routing: + internal: Host + external: Aggregate diff --git a/test/e2e/fixtures/ipam/project-beta/pools.yaml b/test/e2e/fixtures/ipam/project-beta/pools.yaml new file mode 100644 index 00000000..11d50b0f --- /dev/null +++ b/test/e2e/fixtures/ipam/project-beta/pools.yaml @@ -0,0 +1,52 @@ +# Ranges are registered in ../RANGES.md. Root pools of one project may not +# overlap. +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-network-v6-root + labels: + nso-fixture: "true" +spec: + cidr: 2001:db8:b000::/36 + ipFamily: IPv6 + visibility: consumer + classNames: + - datum-network-v6 + allocation: + minPrefixLength: 48 + maxPrefixLength: 48 + strategy: FirstFit +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-endpoint-v4-root + labels: + nso-fixture: "true" +spec: + cidr: 10.129.0.0/16 + ipFamily: IPv4 + visibility: consumer + classNames: + - datum-endpoint-v4 + allocation: + minPrefixLength: 32 + maxPrefixLength: 32 + strategy: FirstFit +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-public-v4-root + labels: + nso-fixture: "true" +spec: + cidr: 203.0.113.0/24 + ipFamily: IPv4 + visibility: consumer + classNames: + - datum-public-v4 + allocation: + minPrefixLength: 32 + maxPrefixLength: 32 + strategy: FirstFit diff --git a/test/e2e/fixtures/ipam/rbac.yaml b/test/e2e/fixtures/ipam/rbac.yaml new file mode 100644 index 00000000..4f5e8b40 --- /dev/null +++ b/test/e2e/fixtures/ipam/rbac.yaml @@ -0,0 +1,28 @@ +# The test identity, kept out of the IPAM deployment. +# +# The suites and the fixture seeder reach IPAM by impersonating +# e2e-tenant-tester with the three iam.miloapis.com project extras. IPAM +# authorizes through a delegated SubjectAccessReview against the host +# apiserver, so that impersonated user needs ordinary RBAC of its own. +# +# It binds the nso-ipam-tenant ClusterRole from +# config/dependencies/ipam/overlay/rbac.yaml rather than restating the rules: +# the tester must hold exactly what the operator holds, and two copies of a +# permission set drift. +# +# Nothing grants the *right to impersonate* here. The seeder runs as the kind +# admin credential, which is in system:masters and already authorized for +# everything; only the NSO manager's ServiceAccount needs an explicit +# impersonation grant, and that ships with the deployment. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nso-ipam-tenant-e2e +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nso-ipam-tenant +subjects: + - kind: User + apiGroup: rbac.authorization.k8s.io + name: e2e-tenant-tester diff --git a/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml new file mode 100644 index 00000000..d3c0463d --- /dev/null +++ b/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml @@ -0,0 +1,220 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Deleting an interface a live claim still holds must not take the addresses +# with it. +# +# The delete is allowed — it is not refused and it does not cascade to the +# claim — but the release must NOT happen, because the addresses belong to a +# workload that is still running. Releasing here and re-allocating later is +# silent renumbering, which is worse than either refusing the delete or +# cascading it. +# +# The guarantee is asserted where it is real: the IPClaims in project-alpha +# still hold the same CIDRs after the interface is gone. NSO's own copy of the +# addresses proves nothing here — it is written once at bind and would look +# identical either way. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-bound-interface-delete +spec: + cluster: nso-standard + steps: + - name: A bound claim holds three addresses + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-boundifdel-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + spec: + network: + name: nic-boundifdel-net + ipFamilies: + - IPv6 + - IPv4 + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + record="${TMPDIR:-/tmp}/nic-boundifdel-addresses" + KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default get ipclaim \ + -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}' > "$record" + kubectl -n ipam-e2e-alpha get networkinterface nic-boundifdel -o jsonpath='{.metadata.uid}' > "${record}.uid" + cat "$record" + grep -q 'nic-boundifdel-f-ipv6=' "$record" || { echo "the v6 address was never allocated in IPAM"; exit 1; } + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - description: > + Safety net. The count it prints is the signal: a claim now releases + its addresses from what it asked for rather than from an interface + it may no longer have, so anything above zero here means a claim + went away without handing its addresses back. + script: + content: | + set -u + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-boundifdel --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-boundifdel --ignore-not-found --timeout=60s + stranded=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name} {end}' | tr ' ' '\n' | grep -E '^nic-boundifdel-' || true) + echo "IPClaims stranded by the deleted interface: $(echo "$stranded" | grep -c . || true)" + [ -z "$stranded" ] || ipam delete ipclaim $stranded --ignore-not-found --timeout=60s + orphans=$(ipam get ipallocation -o name) + [ -z "$orphans" ] || ipam delete $orphans --ignore-not-found + rm -f "${TMPDIR:-/tmp}/nic-boundifdel-addresses" "${TMPDIR:-/tmp}/nic-boundifdel-addresses.uid" + + - name: Deleting the interface under a live claim is allowed + description: | + Not refused and not cascaded: the interface's finalizer clears at once + because a live claim still holds it, and the claim itself is untouched. + try: + - script: + timeout: 120s + content: | + set -eu + # --wait=false on purpose: the claim controller rebuilds the + # interface within seconds, so a waiting delete can watch the + # replacement appear under the same name and never return. + kubectl -n ipam-e2e-alpha delete networkinterface nic-boundifdel --wait=false + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + + - name: The addresses were not released + description: | + The assertion that matters. Every IPClaim must still exist and still + hold the CIDR recorded before the delete — a release here would hand the + workload new addresses on the next reconcile. + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + before=$(cat "${TMPDIR:-/tmp}/nic-boundifdel-addresses") + after=$(ipam get ipclaim -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}') + echo "before: ${before}" + echo "after: ${after}" + [ "$before" = "$after" ] || { echo "the addresses changed when the interface was deleted"; exit 1; } + + - name: The claim rebuilds the interface on the same addresses + description: | + The rebuild is driven by the claim controller watching NetworkInterface + and enqueuing the holder, so no nudge is needed here; a claim left + Bound against a missing interface would be the failure. + + The rebuilt interface must carry the addresses IPAM already holds, not + newly allocated ones — that reuse is the whole reason deleting a bound + interface is allowed to be survivable. + try: + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + spec: + claimRef: + name: nic-boundifdel + status: + phase: Bound + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + before=$(cat "${TMPDIR:-/tmp}/nic-boundifdel-addresses") + after=$(ipam get ipclaim -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}') + [ "$before" = "$after" ] || { echo "the rebuild reallocated: before ${before}, after ${after}"; exit 1; } + + # A rebuild, not a delete that never took: the object carrying + # these addresses is a different one. + was=$(cat "${TMPDIR:-/tmp}/nic-boundifdel-addresses.uid") + now=$(kubectl -n ipam-e2e-alpha get networkinterface nic-boundifdel -o jsonpath='{.metadata.uid}') + [ "$was" != "$now" ] || { echo "the interface was never actually deleted (same uid ${now})"; exit 1; } + + for entry in $before; do + name=${entry%%=*} + cidr=${entry#*=} + case "$name" in + *-c-datum-public-v4) published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-boundifdel -o jsonpath='{.status.externalAddresses[0].address}')/32 ;; + *-f-ipv6) published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-boundifdel -o 'jsonpath={.status.addresses[?(@.family=="IPv6")].address}') ;; + *-f-ipv4) published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-boundifdel -o 'jsonpath={.status.addresses[?(@.family=="IPv4")].address}') ;; + esac + [ "$published" = "$cidr" ] || { echo "the rebuilt interface publishes ${published} where IPAM holds ${cidr}"; exit 1; } + done + + - name: The rebuilt claim releases everything on delete + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-boundifdel + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + ipam get ipclass datum-public-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-boundifdel-' || true); do + echo "IPClaim ${found} survived the claim that held it" + exit 1 + done diff --git a/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml new file mode 100644 index 00000000..100d26f1 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml @@ -0,0 +1,146 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A dual-stack claim binds, and the addresses it publishes are real allocations +# in the upstream IPAM service rather than values NSO invented. +# +# The claim lives in ipam-e2e-alpha, whose upstream-cluster-name label routes to +# project-alpha; the IPClaims are therefore read back through the +# tenant-project-alpha impersonation context, in the project namespace named by +# the cell namespace's upstream-namespace label (default). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-dual-stack +spec: + cluster: nso-standard + steps: + - name: Create a dual-stack network + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-dualstack-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + + - name: A dual-stack claim binds and publishes one address per family + description: | + Programmed stays Unknown because no data plane reports the attachment in + this env, so Ready is never True. Waiting for Ready here would hang. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-dualstack + namespace: ipam-e2e-alpha + spec: + network: + name: nic-dualstack-net + ipFamilies: + - IPv6 + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-dualstack + namespace: ipam-e2e-alpha + status: + networkInterfaceRef: + name: nic-dualstack + addresses: + - family: IPv6 + primary: true + - family: IPv4 + (length(addresses || `[]`) == `2`): true + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0].status): "True" + (conditions[?type == 'Programmed'] | [0].status): Unknown + (length(conditions[?type == 'Ready' && status == 'True']) == `0`): true + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: The interface the claim names exists and is bound to it + try: + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-dualstack + namespace: ipam-e2e-alpha + spec: + claimRef: + name: nic-dualstack + status: + phase: Bound + + - name: Every published address is an IPClaim held in project-alpha + description: | + The assertion this environment exists to make. Each address is looked up + in IPAM under its deterministic name — -f-ipv6 / -f-ipv4 — + and must be Bound to exactly the CIDR NSO published. + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { ctx=$1; shift; KUBECONFIG=$IPAM_KUBECONFIG kubectl --context "tenant-$ctx" -n default "$@"; } + + for family in ipv6 ipv4; do + upper=$(echo "$family" | tr '[:lower:]' '[:upper:]' | sed 's/IPV/IPv/') + published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-dualstack \ + -o "jsonpath={.status.addresses[?(@.family=='${upper}')].address}") + allocated=$(ipam project-alpha get ipclaim "nic-dualstack-f-${family}" -o jsonpath='{.status.allocatedCIDR}') + phase=$(ipam project-alpha get ipclaim "nic-dualstack-f-${family}" -o jsonpath='{.status.phase}') + + echo "${upper}: claim=${published} ipam=${allocated} (${phase})" + [ -n "$published" ] || { echo "claim published no ${upper} address"; exit 1; } + [ "$phase" = "Bound" ] || { echo "IPClaim nic-dualstack-f-${family} is ${phase}, want Bound"; exit 1; } + [ "$published" = "$allocated" ] || { echo "claim published ${published}, IPAM holds ${allocated}"; exit 1; } + done + + # Addresses on the interface are always CIDRs, host prefix included. + # A bare address here is a regression. + v6=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-dualstack -o "jsonpath={.status.addresses[?(@.family=='IPv6')].address}") + v4=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-dualstack -o "jsonpath={.status.addresses[?(@.family=='IPv4')].address}") + echo "$v6" | grep -Eq '^2001:db8:a[0-9a-f]{3}:.*/96$' || { echo "IPv6 ${v6} is not a /96 out of project-alpha's 2001:db8:a000::/36"; exit 1; } + echo "$v4" | grep -Eq '^10\.128\.[0-9]+\.[0-9]+/32$' || { echo "IPv4 ${v4} is not a /32 out of project-alpha's 10.128.0.0/16"; exit 1; } + + - name: Deleting the claim releases both addresses + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-dualstack + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + ipam get ipclass datum-endpoint-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-dualstack-' || true); do + echo "IPClaim ${found} still exists in project-alpha after the claim was deleted" + exit 1 + done diff --git a/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml new file mode 100644 index 00000000..2a92867f --- /dev/null +++ b/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml @@ -0,0 +1,86 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A class named in spec.addresses[] yields an EXTERNAL address, and it arrives +# bare. +# +# datum-public-v4 carries no default-class annotation, so the only way to reach +# 198.51.100.0/24 is to name it. It is a /32 class: IPAM hands back +# "198.51.100.x/32" and the interface must publish "198.51.100.x". A prefix +# surviving into externalAddresses is a regression, not a fixture quirk. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-external-address +spec: + cluster: nso-standard + steps: + - name: A claim naming datum-public-v4 binds + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-external-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-external + namespace: ipam-e2e-alpha + spec: + network: + name: nic-external-net + ipFamilies: + - IPv4 + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-external + namespace: ipam-e2e-alpha + status: + externalAddresses: + - family: IPv4 + class: datum-public-v4 + (length(externalAddresses || `[]`) == `1`): true + (length(addresses || `[]`) == `1`): true + (conditions[?type == 'Allocated'] | [0].status): "True" + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: The external address is bare and comes from project-alpha's public range + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + + external=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-external -o jsonpath='{.status.externalAddresses[0].address}') + internal=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-external -o jsonpath='{.status.addresses[0].address}') + held=$(KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default \ + get ipclaim nic-external-c-datum-public-v4 -o jsonpath='{.status.allocatedCIDR}') + echo "external=${external} internal=${internal} ipam=${held}" + + echo "$external" | grep -Eq '^198\.51\.100\.[0-9]+$' || { echo "external address ${external} is not a bare address from 198.51.100.0/24"; exit 1; } + echo "$internal" | grep -Eq '^10\.128\.[0-9]+\.[0-9]+/32$' || { echo "internal address ${internal} is not a /32 from 10.128.0.0/16"; exit 1; } + [ "$held" = "${external}/32" ] || { echo "IPAM holds ${held}, claim published ${external}"; exit 1; } + + # The named class must not have leaked into the internal address + # list, and the family entry must not have taken the public class. + class=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-external -o jsonpath='{.status.addresses[0].class}') + [ -z "$class" ] || { echo "the family address was allocated from class ${class}, want the default class"; exit 1; } diff --git a/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml new file mode 100644 index 00000000..9b7e96be --- /dev/null +++ b/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml @@ -0,0 +1,134 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A claim asking for a family its network does not carry fails, naming the +# family, and allocates nothing. +# +# This is the accidental case, not an exotic one: Network.spec.ipFamilies +# defaults to [IPv4] and a claim's defaults to [IPv6], so the most minimal claim +# anyone can write against the most minimal network is exactly this mismatch. +# The check is NSO's own — it compares the two specs before it ever reaches +# IPAM — so a green result here says nothing about IPAM and everything about +# NSO refusing to ask for space it knows is wrong. +# +# A second claim on the SAME network, differing only in the family it asks for, +# allocates for real. It is the control that gives the absence its meaning: an +# IPAM this suite cannot read at all would otherwise look identical to an IPAM +# that was never asked. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-family-not-carried +spec: + cluster: nso-standard + steps: + - name: A minimal claim on a default network is refused + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-family-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-family-net + namespace: ipam-e2e-alpha + spec: + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-family-mismatch + namespace: ipam-e2e-alpha + spec: + network: + name: nic-family-net + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-family-mismatch + namespace: ipam-e2e-alpha + spec: + ipFamilies: + - IPv6 + status: + (conditions[?type == 'Bound'] | [0]): + status: "False" + reason: AddressFamilyNotCarried + (conditions[?type == 'Allocated'] | [0].status): "False" + (conditions[?type == 'Ready'] | [0].status): "False" + (contains(conditions[?type == 'Bound'] | [0].message, 'IPv6')): true + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get network,networkinterfaceclaim -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: The same network satisfies the family it does carry + description: | + The positive control. Identical to the refused claim but for the family + it asks for. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-family-canary + namespace: ipam-e2e-alpha + spec: + network: + name: nic-family-net + ipFamilies: + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-family-canary + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + + - name: The refusal allocated nothing + description: | + The whole IPClaim list is matched against the refused claim's prefix + rather than the one name today's derivation produces, so an allocation + made under a name this suite did not predict is still caught. + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + claims=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + echo "project-alpha IPClaims: $(echo $claims)" + + echo "$claims" | grep -qx 'nic-family-canary-f-ipv4' \ + || { echo "the canary's IPClaim is not visible in project-alpha — this suite cannot see allocations, so it cannot prove their absence"; exit 1; } + + for found in $(echo "$claims" | grep -E '^nic-family-mismatch-' || true); do + echo "a refused claim still allocated ${found}" + exit 1 + done + + if kubectl -n ipam-e2e-alpha get networkinterface nic-family-mismatch >/dev/null 2>&1; then + echo "a refused claim still created a NetworkInterface" + exit 1 + fi diff --git a/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml new file mode 100644 index 00000000..2e6d5170 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml @@ -0,0 +1,191 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Deleting a retained interface hands its addresses back. +# +# A retained interface sits unbound with no claim left to carry a finalizer on +# its behalf, which is exactly when it looks disposable. Before the interface +# grew its own finalizer, deleting one left its IPClaims alive with nothing in +# NSO naming them: capacity gone, nothing reporting it, and the obvious manual +# remedy — deleting those IPClaims by hand — stranded an IPAllocation per claim +# and wedged the name permanently. +# +# So this asserts the release in IPAM, and asserts that the state which made +# the manual remedy tempting no longer exists. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-interface-release +spec: + cluster: nso-standard + steps: + - name: A retained claim binds and its interface carries a finalizer + description: | + The finalizer and the allocation-claim annotation are what make the + release possible at all: the first holds the delete open, the second + records which claim the IPClaims were minted under so they can still be + found once the claim is gone. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-ifrelease-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + spec: + network: + name: nic-ifrelease-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + annotations: + networking.datumapis.com/allocation-claim: nic-ifrelease + (contains(finalizers, 'networking.datumapis.com/networkinterface-release')): true + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - description: > + Safety net. Everything this test creates outlives its claim by + design, so a failure part-way through would otherwise hold three + addresses in project-alpha for every later run. + script: + content: | + set -u + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-ifrelease --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-ifrelease --ignore-not-found --timeout=60s + leftover=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name} {end}' | tr ' ' '\n' | grep -E '^nic-ifrelease-' || true) + [ -z "$leftover" ] || ipam delete ipclaim $leftover --ignore-not-found --timeout=60s + orphans=$(ipam get ipallocation -o name) + [ -z "$orphans" ] || ipam delete $orphans --ignore-not-found + echo "orphaned IPAllocations purged: $(echo "$orphans" | grep -c . || true)" + + - name: The addresses are held in IPAM and survive the claim + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + held() { ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true; } + + [ "$(held)" -eq 3 ] || { echo "expected 3 IPClaims for this interface, found $(held)"; exit 1; } + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-ifrelease + namespace: ipam-e2e-alpha + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + spec: + (claimRef == null): true + status: + phase: Available + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + count=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true) + [ "$count" -eq 3 ] || { echo "a retained interface should still hold its 3 addresses, found ${count}"; exit 1; } + + - name: Deleting the unheld interface releases every address + description: | + The delete blocks on the interface's finalizer until the release has + happened, so an assertion after it returns is asserting a completed + release rather than racing one. + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + name: nic-ifrelease + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + ipam get ipclass datum-public-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-ifrelease-' || true); do + echo "IPClaim ${found} survived the interface that held it" + exit 1 + done + + - name: Nothing is left for an operator to clean up by hand + description: | + The composition that made this dangerous: stranded IPClaims invite a + manual delete, and deleting a Retain-allocated IPClaim leaves an + IPAllocation holding the address and blocking the name forever. There + is no longer a stranded IPClaim to invite it. + + The IPAllocations themselves still outlive the release — IPAM freezes + Retain onto the allocation and nothing revisits it — so the cleanup on + the first step purges them and prints the count. That residue is + IPAM-side and is not something NSO can assert away today. + try: + - error: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + stranded=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true) + [ "$stranded" -eq 0 ] || { echo "${stranded} IPClaims are stranded with nothing in NSO naming them"; exit 1; } + echo "IPAllocations still holding addresses after the release: $(ipam get ipallocation -o name | grep -c . || true)" diff --git a/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml new file mode 100644 index 00000000..951b4224 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml @@ -0,0 +1,132 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Delete the interface, then the claim. Nothing is left holding an address. +# +# This order used to leak everything: release() read the addresses off the +# interface, so a claim whose interface was already gone found nothing to +# release, dropped its finalizer and left every IPClaim alive with no NSO object +# naming it. A claim now releases what it ASKED for — its ipFamilies and its +# addresses[].class — which is what the names were minted from, so the interface +# no longer has to survive for the addresses to come back. +# +# The interface delete does not wait: whether the claim controller rebuilds the +# interface before the claim delete lands is a race, and the end state has to be +# empty either way. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-interface-then-claim-delete +spec: + cluster: nso-standard + steps: + - name: A claim holds a family address and a named class + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-reverse-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-reverse + namespace: ipam-e2e-alpha + spec: + network: + name: nic-reverse-net + ipFamilies: + - IPv6 + - IPv4 + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-reverse + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + count=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-reverse-' || true) + [ "$count" -eq 3 ] || { echo "expected 3 IPClaims before the deletes, found ${count}"; exit 1; } + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - description: > + Safety net. If the release regresses, the addresses would be held in + project-alpha for every later run; the count printed here is zero + whenever the test itself passed. + script: + content: | + set -u + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-reverse --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-reverse --ignore-not-found --timeout=60s + stranded=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name} {end}' | tr ' ' '\n' | grep -E '^nic-reverse-' || true) + echo "IPClaims left holding addresses: $(echo "$stranded" | grep -c . || true)" + [ -z "$stranded" ] || ipam delete ipclaim $stranded --ignore-not-found --timeout=60s + orphans=$(ipam get ipallocation -o name) + [ -z "$orphans" ] || ipam delete $orphans --ignore-not-found + + - name: Deleting the interface and then the claim releases everything + try: + - script: + timeout: 120s + content: | + set -eu + # --wait=false on purpose: the claim controller rebuilds the + # interface within seconds, so a waiting delete can watch the + # replacement appear under the same name and never return. + kubectl -n ipam-e2e-alpha delete networkinterface nic-reverse --wait=false + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-reverse + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + ipam get ipclass datum-public-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-reverse-' || true); do + echo "IPClaim ${found} outlived both the interface and the claim" + exit 1 + done + + orphans=$(ipam get ipallocation -o name | grep -c . || true) + [ "$orphans" -eq 0 ] || { echo "${orphans} IPAllocations are still holding addresses"; exit 1; } + - error: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-reverse + namespace: ipam-e2e-alpha diff --git a/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml new file mode 100644 index 00000000..12a9240f --- /dev/null +++ b/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml @@ -0,0 +1,276 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# An interface advertising an address IPAM no longer holds is reported loudly +# and gates nothing. +# +# Both halves matter. IPAM considers the address free and will hand it to the +# next claimant — measured: a fresh datum-endpoint-v4 claim in project-alpha +# came back with the very address the interface was still advertising — so an +# operator has to find out. But re-allocating would renumber a workload that is +# running on that address right now, which is worse than the duplicate, so the +# claim must stay Bound and keep advertising what it is actually using. +# +# The state is built by deleting the IPClaim out from under a bound interface. +# That is not a state NSO can reach on its own; it stands in for an address lost +# to an IPAM-side repair, restore, or operator error. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-missing-allocation +spec: + cluster: nso-standard + steps: + - name: A claim binds and holds one address + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-missingalloc-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-missingalloc + namespace: ipam-e2e-alpha + spec: + network: + name: nic-missingalloc-net + ipFamilies: + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-missingalloc + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0].status): "True" + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n ipam-e2e-alpha get events --field-selector involvedObject.name=nic-missingalloc + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - script: + content: | + set -u + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-missingalloc --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-missingalloc --ignore-not-found --timeout=60s + leftover=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name} {end}' | tr ' ' '\n' | grep -E '^nic-missingalloc-' || true) + [ -z "$leftover" ] || ipam delete ipclaim $leftover --ignore-not-found --timeout=60s + orphans=$(ipam get ipallocation -o name) + [ -z "$orphans" ] || ipam delete $orphans --ignore-not-found + rm -f "${TMPDIR:-/tmp}/nic-missingalloc-address" + + - name: The allocation disappears from IPAM + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + address=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-missingalloc -o jsonpath='{.status.addresses[0].address}') + [ -n "$address" ] || { echo "the claim never published an address"; exit 1; } + echo "$address" > "${TMPDIR:-/tmp}/nic-missingalloc-address" + + held=$(ipam get ipclaim nic-missingalloc-f-ipv4 -o jsonpath='{.status.allocatedCIDR}') + [ "$held" = "$address" ] || { echo "IPAM holds ${held}, the claim publishes ${address}"; exit 1; } + + ipam delete ipclaim nic-missingalloc-f-ipv4 --timeout=60s + ! ipam get ipclaim nic-missingalloc-f-ipv4 >/dev/null 2>&1 || { echo "the IPClaim is still there"; exit 1; } + - description: > + Every event for this claim is cleared first. The claim name is fixed + and events outlive a run by about an hour, so an event from an + earlier run would otherwise satisfy the next one — the assertion + would stop testing anything the day the controller went quiet. + + Then the claim is nudged, because nothing watches IPAM: this forces + the reconcile an operator's next edit would have caused anyway. + script: + content: | + set -eu + kubectl -n ipam-e2e-alpha delete events \ + --field-selector involvedObject.name=nic-missingalloc --ignore-not-found + kubectl -n ipam-e2e-alpha annotate networkinterfaceclaim nic-missingalloc \ + e2e.datum.net/nudge="$(date +%s)" --overwrite + + - name: The loss is reported as a Warning on the claim + description: | + The event is re-emitted every reconcile so the signal stays alive rather + than ageing out, so this asserts at least one rather than exactly one. + try: + - script: + timeout: 120s + content: | + set -eu + address=$(cat "${TMPDIR:-/tmp}/nic-missingalloc-address") + + for _ in $(seq 1 30); do + count=$(kubectl -n ipam-e2e-alpha get events \ + --field-selector involvedObject.name=nic-missingalloc,reason=AddressAllocationMissing \ + -o jsonpath='{range .items[*]}{.type}{"\n"}{end}' | grep -c '^Warning$' || true) + [ "$count" -ge 1 ] && break + sleep 3 + done + + [ "$count" -ge 1 ] || { echo "no Warning/AddressAllocationMissing event was recorded on the claim"; exit 1; } + + message=$(kubectl -n ipam-e2e-alpha get events \ + --field-selector involvedObject.name=nic-missingalloc,reason=AddressAllocationMissing \ + -o jsonpath='{.items[0].message}') + echo "event: ${message}" + case "$message" in + *"$address"*) ;; + *) echo "the event does not name the address ${address}"; exit 1 ;; + esac + case "$message" in + *nic-missingalloc-f-ipv4*) ;; + *) echo "the event does not name the missing IPClaim"; exit 1 ;; + esac + case "$message" in + *project-alpha*) ;; + *) echo "the event does not name the project"; exit 1 ;; + esac + + - name: Allocated goes False and the address is kept anyway + description: | + Allocated says every requested address is held. It is not, so it reads + False — reporting True here would be the status contradicting a check the + controller had just failed. Ready follows it down. + + Bound stays True and the address does not move: a workload is running on + it, and re-allocating to make the status tidy would renumber a live + interface to report a problem the operator has to resolve in IPAM either + way. Loud, and still not destructive. + try: + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-missingalloc + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0]): + status: "False" + reason: AddressAllocationMissing + (conditions[?type == 'Ready'] | [0]): + status: "False" + reason: NotAllocated + - script: + content: | + set -eu + address=$(cat "${TMPDIR:-/tmp}/nic-missingalloc-address") + published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-missingalloc -o jsonpath='{.status.addresses[0].address}') + advertised=$(kubectl -n ipam-e2e-alpha get networkinterface nic-missingalloc -o jsonpath='{.spec.addresses[0].address}') + [ "$published" = "$address" ] || { echo "the claim renumbered to ${published}"; exit 1; } + [ "$advertised" = "$address" ] || { echo "the interface renumbered to ${advertised}"; exit 1; } + + message=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-missingalloc \ + -o "jsonpath={.status.conditions[?(@.type=='Allocated')].message}") + echo "Allocated: ${message}" + case "$message" in + *"$address"*) ;; + *) echo "the condition does not name the address ${address}"; exit 1 ;; + esac + + - name: An allocation that moved to another address is caught too + description: | + The harder half of the same check: the IPClaim exists, under the right + name, and holds a DIFFERENT address. Only comparing what IPAM holds + against what the interface publishes catches it — a check that stopped + at "the IPClaim is there" would call this healthy. + + The blocker claim takes the freed address first, so the recreated one is + guaranteed to land somewhere else; without it IPAM hands back the very + address that was just released and there is nothing to detect. + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + address=$(cat "${TMPDIR:-/tmp}/nic-missingalloc-address") + + for name in nic-missingalloc-blocker nic-missingalloc-f-ipv4; do + ipam apply -f - >/dev/null </dev/null 2>&1; } + + visible tenant-project-alpha nic-isolation-alpha-f-ipv4 || { echo "project-alpha cannot see its own IPClaim"; exit 1; } + visible tenant-project-beta nic-isolation-beta-f-ipv4 || { echo "project-beta cannot see its own IPClaim"; exit 1; } + visible tenant-project-alpha nic-isolation-legacy-f-ipv4 || { echo "the legacy-labelled namespace did not allocate in project-alpha"; exit 1; } + + KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-platform get --raw /apis/ipam.miloapis.com/v1alpha1 >/dev/null \ + || { echo "tenant-platform cannot reach IPAM at all, so it proves nothing about scoping"; exit 1; } + classes=$(ipam tenant-project-alpha get ipclass -o name | wc -l | tr -d ' ') + [ "$classes" -gt 0 ] || { echo "project-alpha sees no IPClass, so the fixtures are not readable through this context"; exit 1; } + platform_classes=$(KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-platform get ipclass -o name | wc -l | tr -d ' ') + [ "$platform_classes" -eq 0 ] || { echo "tenant-platform reads ${platform_classes} IPClass objects; it is meant to hold no project"; exit 1; } + denial=$(ipam tenant-platform get ipclaim nic-isolation-alpha-f-ipv4 2>&1 || true) + echo "$denial" | grep -Eq 'NotFound|Forbidden|forbidden|not found' \ + || { echo "tenant-platform's refusal did not come from the server: ${denial}"; exit 1; } + + ! visible tenant-project-beta nic-isolation-alpha-f-ipv4 || { echo "project-beta can read project-alpha's IPClaim"; exit 1; } + ! visible tenant-project-alpha nic-isolation-beta-f-ipv4 || { echo "project-alpha can read project-beta's IPClaim"; exit 1; } + ! visible tenant-platform nic-isolation-alpha-f-ipv4 || { echo "the project-less context can read project-alpha's IPClaim"; exit 1; } + ! visible tenant-platform nic-isolation-beta-f-ipv4 || { echo "the project-less context can read project-beta's IPClaim"; exit 1; } + + - name: The address IPAM holds is the address the claim published + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + allocated() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context "$1" -n default get ipclaim "$2" -o jsonpath='{.status.allocatedCIDR}'; } + + check() { + published=$(kubectl -n "$1" get networkinterfaceclaim "$2" -o jsonpath='{.status.addresses[0].address}') + held=$(allocated "$3" "$2-f-ipv4") + [ "$published" = "$held" ] || { echo "$2 published ${published}, IPAM holds ${held}"; exit 1; } + } + + check ipam-e2e-alpha nic-isolation-alpha tenant-project-alpha + check ipam-e2e-beta nic-isolation-beta tenant-project-beta + check ipam-e2e-legacy nic-isolation-legacy tenant-project-alpha diff --git a/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml new file mode 100644 index 00000000..e294a6d3 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml @@ -0,0 +1,104 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# reclaimPolicy: Delete returns every address to IPAM when the claim goes away. +# +# The interface disappearing proves nothing on its own — an address that stays +# claimed in IPAM after its consumer is gone is leaked space that no one can +# name. The release is therefore asserted in IPAM, over all three claim kinds +# the interface holds: both family entries and the named class. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-reclaim-delete +spec: + cluster: nso-standard + steps: + - name: Claim two families and a named class with reclaimPolicy Delete + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-delete-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-delete + namespace: ipam-e2e-alpha + spec: + network: + name: nic-delete-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Delete + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-delete + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: All three addresses are held in IPAM + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + for name in nic-delete-f-ipv6 nic-delete-f-ipv4 nic-delete-c-datum-public-v4; do + phase=$(KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default \ + get ipclaim "$name" -o jsonpath='{.status.phase}' 2>/dev/null || true) + echo "${name}: ${phase:-missing}" + [ "$phase" = "Bound" ] || { echo "IPClaim ${name} is ${phase:-missing}, want Bound"; exit 1; } + done + + - name: Deleting the claim releases every address + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-delete + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + ipam get ipclass datum-public-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-delete-' || true); do + echo "IPClaim ${found} survived the claim it was allocated for" + exit 1 + done + - error: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-delete + namespace: ipam-e2e-alpha diff --git a/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml new file mode 100644 index 00000000..fa16ee13 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml @@ -0,0 +1,289 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# reclaimPolicy: Retain is a promise about specific addresses: a claim of the +# same name that comes back gets the SAME interface holding the SAME addresses, +# because a workload that is rescheduled keeps the address its peers, its DNS, +# and its firewall rules already know. +# +# The addresses are captured before the claim is deleted and compared verbatim +# after it is recreated. Asserting that the new claim merely has "an" address +# would pass against a controller that reallocated, which is the exact failure +# this policy exists to prevent. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-reclaim-retain +spec: + cluster: nso-standard + steps: + - name: A retained claim binds and its addresses are recorded + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-retain-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + network: + name: nic-retain-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + record="${TMPDIR:-/tmp}/nic-retain-addresses" + kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o jsonpath='{.status.addresses[?(@.family=="IPv6")].address} {.status.addresses[?(@.family=="IPv4")].address} {.status.externalAddresses[0].address} {.status.networkInterfaceRef.name}' > "$record" + cat "$record" + grep -Eq '^2001:db8:a[0-9a-f]{3}:[^ ]*/96 10\.128\.[0-9]+\.[0-9]+/32 198\.51\.100\.[0-9]+ nic-retain$' "$record" \ + || { echo "recorded an unexpected address set"; exit 1; } + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - description: > + Safety net, and not optional. A retained interface and its IPClaims + outlive the claim by design, and deleting a retained IPClaim leaves + its IPAllocation behind in IPAM — which keeps holding the address + AND blocks the name, so the next run of this test is refused with + RetainedAddressConflict. Every layer therefore has to be cleared + here: claim, interface, IPClaims, then the allocations that held the + recorded addresses. + script: + content: | + set -u + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + record="${TMPDIR:-/tmp}/nic-retain-addresses" + ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-retain --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-retain --ignore-not-found --timeout=60s + ipam delete ipclaim nic-retain-f-ipv6 nic-retain-f-ipv4 nic-retain-c-datum-public-v4 \ + --ignore-not-found --timeout=60s + + if [ -f "$record" ]; then + read -r want6 want4 wantext _ < "$record" || true + purged=0 + for cidr in "$want6" "$want4" "${wantext}/32"; do + leftover=$(ipam get ipallocation -o "jsonpath={range .items[?(@.status.allocatedCIDR=='${cidr}')]}{.metadata.name} {end}") + if [ -n "$leftover" ]; then + echo "orphaned IPAllocation holding ${cidr}: ${leftover}" + ipam delete ipallocation $leftover --ignore-not-found + purged=$((purged + 1)) + fi + done + echo "orphaned IPAllocations purged: ${purged}" + fi + rm -f "$record" + + - name: Deleting the claim leaves the interface and its addresses in place + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-retain + namespace: ipam-e2e-alpha + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + (claimRef == null): true + status: + phase: Available + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + read -r want6 want4 wantext _ < "${TMPDIR:-/tmp}/nic-retain-addresses" || true + + held() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default get ipclaim "$1" -o jsonpath='{.status.allocatedCIDR}'; } + + [ "$(held nic-retain-f-ipv6)" = "$want6" ] || { echo "IPv6 address was not retained in IPAM"; exit 1; } + [ "$(held nic-retain-f-ipv4)" = "$want4" ] || { echo "IPv4 address was not retained in IPAM"; exit 1; } + [ "$(held nic-retain-c-datum-public-v4)" = "${wantext}/32" ] || { echo "the external address was not retained in IPAM"; exit 1; } + + - name: A claim asking for a different reclaim policy cannot adopt it + description: | + The delete-and-recreate route to the wedge the CEL immutability closes + in place. These addresses were allocated under Retain and IPAM freezes + that onto the allocation, so a Delete claim adopting them would strand + what it was told to release. It has to be refused at bind. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + network: + name: nic-retain-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Delete + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Bound'] | [0]): + status: "False" + reason: ReclaimPolicyMismatch + (conditions[?type == 'Allocated'] | [0].status): "False" + (contains(conditions[?type == 'Bound'] | [0].message, 'reclaimPolicy')): true + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + reclaimPolicy: Retain + (claimRef == null): true + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-retain + namespace: ipam-e2e-alpha + + - name: A claim of the same name rebinds the same interface with the same addresses + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + network: + name: nic-retain-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + status: + networkInterfaceRef: + name: nic-retain + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + read -r want6 want4 wantext wantiface < "${TMPDIR:-/tmp}/nic-retain-addresses" || true + + got6=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o 'jsonpath={.status.addresses[?(@.family=="IPv6")].address}') + got4=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o 'jsonpath={.status.addresses[?(@.family=="IPv4")].address}') + gotext=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o jsonpath='{.status.externalAddresses[0].address}') + gotiface=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o jsonpath='{.status.networkInterfaceRef.name}') + echo "before: ${want6} ${want4} ${wantext} ${wantiface}" + echo "after: ${got6} ${got4} ${gotext} ${gotiface}" + + [ "$got6" = "$want6" ] || { echo "IPv6 address changed across the rebind"; exit 1; } + [ "$got4" = "$want4" ] || { echo "IPv4 address changed across the rebind"; exit 1; } + [ "$gotext" = "$wantext" ] || { echo "the external address changed across the rebind"; exit 1; } + [ "$gotiface" = "$wantiface" ] || { echo "the claim bound a different interface"; exit 1; } + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + annotations: + networking.datumapis.com/allocation-claim: nic-retain + spec: + claimRef: + name: nic-retain + status: + phase: Bound + + - name: The reclaim policy cannot be edited after creation + description: | + A retained interface is given up by deleting the interface, not by + relaxing the claim that holds it. Editing reclaimPolicy in place is + refused at admission: NSO freezes the policy into IPAM when the address + is first allocated and cannot revise it there, so a claim that changed + its mind would delete an IPClaim that IPAM still believes is retained + and strand the allocation. + try: + - patch: + expect: + - check: + ($error != null): true + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + reclaimPolicy: Delete + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + reclaimPolicy: Retain diff --git a/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml new file mode 100644 index 00000000..6fd559af --- /dev/null +++ b/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml @@ -0,0 +1,137 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A namespace that names no project allocates nothing. ipam-e2e-unlabeled +# carries no meta.datumapis.com/upstream-cluster-name label, and the only safe +# outcome is a refusal naming the missing label — a fallback to some default +# project would hand one tenant another tenant's address space. +# +# The absence of an allocation is only evidence if this suite can see an +# allocation when there is one, so a canary claim in ipam-e2e-alpha allocates +# for real and is read back through the same context, in the same step, before +# any absence is asserted. Without it a dead IPAM, a stale kubeconfig or a +# typo'd context all read as a pass. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-unlabeled-namespace +spec: + cluster: nso-standard + steps: + - name: A claim in an unlabelled namespace is refused + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-unlabeled-net + namespace: ipam-e2e-unlabeled + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-unlabeled + namespace: ipam-e2e-unlabeled + spec: + network: + name: nic-unlabeled-net + ipFamilies: + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-unlabeled + namespace: ipam-e2e-unlabeled + status: + (conditions[?type == 'Bound'] | [0]): + status: "False" + reason: ProjectUnresolved + (conditions[?type == 'Allocated'] | [0].status): "False" + (conditions[?type == 'Ready'] | [0].status): "False" + (contains(conditions[?type == 'Bound'] | [0].message, 'meta.datumapis.com/upstream-cluster-name')): true + catch: + - script: + content: | + kubectl -n ipam-e2e-unlabeled get networkinterfaceclaim -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: A labelled namespace allocates, so an empty result means empty + description: | + The positive control. This claim differs from the refused one only in + the namespace it lives in. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-unlabeled-canary-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-unlabeled-canary + namespace: ipam-e2e-alpha + spec: + network: + name: nic-unlabeled-canary-net + ipFamilies: + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-unlabeled-canary + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + + - name: Nothing was bound and nothing was allocated + description: | + Every IPClaim in both project namespaces is listed and matched against + the refused claim's name prefix, rather than probing the two names the + naming scheme happens to derive today — a leak under an unexpected name + is exactly what a derivation bug produces. + try: + - script: + content: | + set -eu + IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + ipam() { ctx=$1; shift; KUBECONFIG=$IPAM_KUBECONFIG kubectl --context "$ctx" -n default "$@"; } + names() { ipam "$1" get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'; } + + alpha=$(names tenant-project-alpha) + beta=$(names tenant-project-beta) + echo "project-alpha IPClaims: $(echo $alpha)" + echo "project-beta IPClaims: $(echo $beta)" + + echo "$alpha" | grep -qx 'nic-unlabeled-canary-f-ipv4' \ + || { echo "the canary's IPClaim is not visible in project-alpha — this suite cannot see allocations, so it cannot prove their absence"; exit 1; } + + for found in $(printf '%s\n%s\n' "$alpha" "$beta" | grep -E '^nic-unlabeled-(f|c)-' || true); do + echo "an unresolvable claim allocated ${found}" + exit 1 + done + + if kubectl -n ipam-e2e-unlabeled get networkinterface nic-unlabeled >/dev/null 2>&1; then + echo "an unresolvable claim still created a NetworkInterface" + exit 1 + fi From f6bd88da850a43953f3f0c3934e3a3b5b9362ec9 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 13:18:52 -0500 Subject: [PATCH 05/13] refactor: source dns CRDs from OCI, impersonate with kubectl flags Two hack/ scripts had better replacements. dns-crds resolved a Go module directory to find CRD files. dns-operator publishes the same CRDs in an OCI bundle, so fetch that instead, pinned by digest at the version go.mod pins and verified on fetch. That deletes module-dir.sh and the entire class of bug it existed for: there is no longer a `go list` that can return an empty string. ipam-fetch-bundle.sh now serves both bundles and is renamed accordingly. kubectl takes impersonation extras as flags, so nothing needs a generated kubeconfig to assert a project. ipam-tenant-kubectl.sh wraps the four flags; it replaces a script that flattened the caller's credentials into a file on disk and had to run before anything could talk to IPAM. The RBAC is unchanged: the impersonator still needs impersonate on users and on the three userextras/iam.miloapis.com/parent-* resources. Key changes: - Fetch the dns-operator bundle in dns-crds; delete module-dir.sh - Rename ipam-fetch-bundle.sh to fetch-oci-bundle.sh, taking any revision string rather than a commit sha, since one bundle pins a tag - Add ipam-tenant-kubectl.sh; move fixture seeding and clearing onto it - ipam-clear-fixtures.sh takes a context instead of a kubeconfig path gen-ipam-impersonation-kubeconfig.sh survives one more change: 11 suites still reference the contexts it writes. It is marked deprecated and now hangs off the suite tasks that need it rather than off fixture seeding. --- Taskfile.test-infra.yml | 64 +++++++++++-------- ...am-fetch-bundle.sh => fetch-oci-bundle.sh} | 31 +++++---- hack/ipam-clear-fixtures.sh | 44 ++++++------- hack/ipam-tenant-kubectl.sh | 42 ++++++++++++ hack/module-dir.sh | 52 --------------- 5 files changed, 122 insertions(+), 111 deletions(-) rename hack/{ipam-fetch-bundle.sh => fetch-oci-bundle.sh} (71%) create mode 100755 hack/ipam-tenant-kubectl.sh delete mode 100755 hack/module-dir.sh diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 93aa5da0..71215f36 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -70,6 +70,13 @@ vars: IPAM_BUNDLE_DIGEST: sha256:08418654958ea6861d54fe5f9e87d9cf306b87429b29fc41a7876dc143a13bf5 IPAM_COMMIT: 13bd2c8c077b12983fa16a8888dcd117b65a14d2 IPAM_STAGE: '{{.TMP_DIR}}/.ipam-bundle' + + # The dns-operator CRDs come from its published bundle at the version go.mod + # pins, so the CRDs installed and the types NSO compiles against match. + DNS_BUNDLE_REPO: ghcr.io/datum-cloud/dns-operator-kustomize + DNS_BUNDLE_TAG: v0.5.1 + DNS_BUNDLE_DIGEST: sha256:cd7dd3a0d2fbbbaa029f2b5e083723a675e88c89e56f999257ff4bd87221cd4a + DNS_STAGE: '{{.TMP_DIR}}/.dns-bundle' CRANE: '{{.REPO}}/bin/crane' CRANE_VERSION: v0.21.7 IPAM_KUBECONFIG: '{{.TMP_DIR}}/.ipam-tenant-impersonation.yaml' @@ -163,7 +170,7 @@ tasks: # The IPAM fixtures and their database die with the upstream cluster; only # the impersonation kubeconfig outlives it. - rm -f {{.IPAM_KUBECONFIG}} - - rm -rf {{.IPAM_STAGE}} + - rm -rf {{.IPAM_STAGE}} {{.DNS_STAGE}} - echo "✨ done." clusters: @@ -203,19 +210,25 @@ tasks: - '{{.KIND}} load docker-image {{.IMG}} --name {{.DOWNSTREAM_CLUSTER}}' dns-crds: - desc: "Install DNSZone / DNSRecordSet / DNSZoneClass CRDs from the dns-operator Go module on the upstream cluster. Required before prepare-upstream when gateway.enableDNSIntegration is true (the operator indexes and watches these types at startup)." - vars: - # Resolved through the helper, not `go list` directly: go list does not - # download, so on a cold module cache it returns an empty string and exits - # 0, and the apply below then fails on a path with a leading slash that - # says nothing about the real cause. - DNS_OPERATOR_CRD_DIR: - sh: '{{.REPO}}/hack/module-dir.sh go.miloapis.com/dns-operator' - cmds: - - echo "📜 installing dns-operator CRDs (upstream) from {{.DNS_OPERATOR_CRD_DIR}}" - - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_OPERATOR_CRD_DIR}}/config/crd/bases/dns.networking.miloapis.com_dnszoneclasses.yaml - - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_OPERATOR_CRD_DIR}}/config/crd/bases/dns.networking.miloapis.com_dnszones.yaml - - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_OPERATOR_CRD_DIR}}/config/crd/bases/dns.networking.miloapis.com_dnsrecordsets.yaml + desc: "Install the dns-operator CRDs (DNSZone / DNSRecordSet / DNSZoneClass) on the upstream cluster from the operator's published bundle, pinned by digest at the version go.mod pins. Required before prepare-upstream when gateway.enableDNSIntegration is true, since the operator indexes and watches these types at startup." + cmds: + - | + set -e + stage="{{.DNS_STAGE}}" + if [ -f "$stage/.digest" ] && [ "$(cat "$stage/.digest")" = "{{.DNS_BUNDLE_DIGEST}}" ]; then + echo "📜 dns-operator bundle already staged ({{.DNS_BUNDLE_TAG}})" >&2 + else + echo "📜 fetching {{.DNS_BUNDLE_REPO}}:{{.DNS_BUNDLE_TAG}}" >&2 + rev=$({{.REPO}}/hack/fetch-oci-bundle.sh \ + {{.CRANE}} {{.CRANE_VERSION}} \ + {{.DNS_BUNDLE_REPO}} {{.DNS_BUNDLE_DIGEST}} {{.DNS_BUNDLE_TAG}} "$stage/bundle") + echo "✅ fetched $rev" >&2 + echo "{{.DNS_BUNDLE_DIGEST}}" > "$stage/.digest" + fi + - echo "📜 installing dns-operator CRDs (upstream)" + - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_STAGE}}/bundle/crd/bases/dns.networking.miloapis.com_dnszoneclasses.yaml + - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_STAGE}}/bundle/crd/bases/dns.networking.miloapis.com_dnszones.yaml + - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_STAGE}}/bundle/crd/bases/dns.networking.miloapis.com_dnsrecordsets.yaml prepare-upstream: desc: "Deploy the NSO manager + webhook (config/e2e) on the upstream cluster, with the prod-base memory profile." @@ -449,7 +462,7 @@ tasks: echo "📦 IPAM bundle already staged ({{.IPAM_BUNDLE_TAG}})" >&2 else echo "📦 fetching {{.IPAM_BUNDLE_REPO}}:{{.IPAM_BUNDLE_TAG}}" >&2 - rev=$({{.REPO}}/hack/ipam-fetch-bundle.sh \ + rev=$({{.REPO}}/hack/fetch-oci-bundle.sh \ {{.CRANE}} {{.CRANE_VERSION}} \ {{.IPAM_BUNDLE_REPO}} {{.IPAM_BUNDLE_DIGEST}} {{.IPAM_COMMIT}} "$stage/bundle") echo "✅ fetched $rev" >&2 @@ -586,6 +599,10 @@ tasks: --from-file=kubeconfig={{.TMP_DIR}}/.ipam-cluster-kubeconfig.yaml \ | kubectl --context {{.UPSTREAM_CTX}} apply -f - + # DEPRECATED, pending the e2e suites moving to hack/ipam-tenant-kubectl.sh. + # Nothing in this Taskfile uses it any more; 11 suites still reference the + # contexts it writes. Delete this task, the script and IPAM_KUBECONFIG once + # they do. ipam-impersonation-kubeconfig: desc: "Write the per-project impersonation kubeconfig used to seed fixtures and to drive chainsaw suites as a tenant. Contexts: tenant-platform (no project, reads nothing by design), tenant-project-alpha, tenant-project-beta." cmds: @@ -595,8 +612,6 @@ tasks: ipam-fixtures: desc: "Seed the IPClass/IPPool fixtures for both projects, clearing any leftovers first. Classes are named identically in the two projects and their pools are disjoint (test/e2e/fixtures/ipam/RANGES.md), so a controller routing to the wrong project allocates from visibly wrong space instead of quietly succeeding. Do not run this while a suite is in flight: it removes the fixtures and the tenant RBAC that a running suite is mid-way through using." - deps: - - ipam-impersonation-kubeconfig cmds: - task: ipam-tenant-rbac - task: ipam-fixtures-clear @@ -604,8 +619,8 @@ tasks: set -e for proj in {{.IPAM_PROJECTS}}; do echo "🌱 seeding $proj" - KUBECONFIG={{.IPAM_KUBECONFIG}} kubectl --context "tenant-$proj" apply -f test/e2e/fixtures/ipam/$proj/classes.yaml - KUBECONFIG={{.IPAM_KUBECONFIG}} kubectl --context "tenant-$proj" apply -f test/e2e/fixtures/ipam/$proj/pools.yaml + IPAM_KUBE_CONTEXT={{.UPSTREAM_CTX}} {{.REPO}}/hack/ipam-tenant-kubectl.sh "$proj" apply -f test/e2e/fixtures/ipam/$proj/classes.yaml + IPAM_KUBE_CONTEXT={{.UPSTREAM_CTX}} {{.REPO}}/hack/ipam-tenant-kubectl.sh "$proj" apply -f test/e2e/fixtures/ipam/$proj/pools.yaml done ipam-tenant-rbac: @@ -618,12 +633,7 @@ tasks: - ipam-tenant-rbac desc: "Delete every IPAM object in the fixture projects, claims first. IPPool and IPClass are cluster-scoped, so chainsaw's namespace teardown never reaches them and a suite that dies mid-run leaves pools that make the NEXT run fail on 'already exists'. Runs ahead of every seed for that reason. Do not run this while a suite is in flight: it removes the fixtures and the tenant RBAC that a running suite is mid-way through using." cmds: - - | - if [ -f {{.IPAM_KUBECONFIG}} ]; then - {{.REPO}}/hack/ipam-clear-fixtures.sh {{.IPAM_KUBECONFIG}} {{.IPAM_PROJECTS}} - else - echo "no impersonation kubeconfig at {{.IPAM_KUBECONFIG}}; nothing to clear" - fi + - '{{.REPO}}/hack/ipam-clear-fixtures.sh {{.UPSTREAM_CTX}} {{.IPAM_PROJECTS}}' ipam-namespaces: desc: "Create the namespaces the interface scenarios resolve a project from, carrying the ENCODED meta.datumapis.com/upstream-cluster-name label the mapped-namespace strategy really writes, plus one namespace deliberately missing it for the fail-closed case. Applied to BOTH clusters: NetworkInterfaceClaims live on the upstream cell and resolveProject reads the claim's own namespace through the cell client, so upstream is the one that has to have them." @@ -721,6 +731,8 @@ tasks: SELECTED: '{{.CLI_ARGS | default .SCENARIOS | default .DEFAULT_SCENARIOS}}' deps: - kubeconfigs + # Only the suites still need this; see the task's own note. + - ipam-impersonation-kubeconfig cmds: # Reset the cluster-scoped fixtures before the first suite. A run that # died mid-suite leaves pools and classes that chainsaw's namespace @@ -758,6 +770,8 @@ tasks: REPORT_NAME: '{{.REPORT_NAME | default "chainsaw-report"}}' deps: - kubeconfigs + # Only the suites still need this; see the task's own note. + - ipam-impersonation-kubeconfig cmds: # See the note in `e2e`: cluster-scoped fixtures outlive a failed run, so # the suite starts by putting them back to a known state. diff --git a/hack/ipam-fetch-bundle.sh b/hack/fetch-oci-bundle.sh similarity index 71% rename from hack/ipam-fetch-bundle.sh rename to hack/fetch-oci-bundle.sh index 902ea2b0..a5da7bad 100755 --- a/hack/ipam-fetch-bundle.sh +++ b/hack/fetch-oci-bundle.sh @@ -1,7 +1,12 @@ #!/usr/bin/env bash -# Fetch the published IPAM manifest bundle from an OCI registry and extract it. +# Fetch a published manifest bundle from an OCI registry and extract it. # -# The bundle is a flux OCI artifact: a single layer of media type +# Used for both the IPAM bundle and the dns-operator bundle. Sourcing manifests +# this way rather than from a Go module directory also removes a failure mode: +# `go list -m -f {{.Dir}}` returns an empty string on a cold module cache and +# exits 0, which silently produced paths with a leading slash. +# +# A bundle is a flux OCI artifact: a single layer of media type # application/vnd.cncf.flux.content.v1.tar+gzip carrying the repo's config/ # tree. It has no org.opencontainers.image.title annotation, so `oras pull` # extracts nothing; reading the layer blob directly works with any OCI client @@ -12,8 +17,12 @@ # Pull is anonymous — the repository is public, so CI needs no registry # credentials. # +# The revision check takes whatever substring identifies the pinned version — +# a commit sha for a bundle pinned to a commit, a tag for one pinned to a +# release — and fails if the artifact does not carry it. +# # Usage: -# ipam-fetch-bundle.sh +# fetch-oci-bundle.sh set -euo pipefail @@ -21,7 +30,7 @@ CRANE="${1:?crane binary path required}" CRANE_VERSION="${2:?crane version required}" REPO="${3:?bundle repository required}" DIGEST="${4:?bundle digest required}" -COMMIT="${5:?expected commit required}" +REVISION="${5:?expected revision required}" OUT="${6:?output directory required}" # Install crane on demand rather than making the caller remember a prep step. @@ -65,14 +74,14 @@ field() { printf '%s' "$manifest" | python3 -c "import json,sys; print($1)" } -# The artifact must be the same commit go.mod pins, or the manifests and the Go -# types NSO compiles against have drifted apart — which is exactly the failure -# the digest pin exists to prevent, so it is checked rather than assumed. +# The artifact must carry the revision go.mod pins, or the manifests and the Go +# types NSO compiles against have drifted apart — exactly the failure the digest +# pin exists to prevent, so it is checked rather than assumed. revision="$(field "json.load(sys.stdin).get('annotations',{}).get('org.opencontainers.image.revision','')")" case "$revision" in - *"$COMMIT"*) ;; + *"$REVISION"*) ;; *) - echo "❌ bundle revision '${revision}' does not carry the pinned commit ${COMMIT}" >&2 + echo "❌ bundle revision '${revision}' does not carry the pinned revision ${REVISION}" >&2 exit 1 ;; esac @@ -88,8 +97,8 @@ trap 'rm -f "$blob"' EXIT retry "$CRANE" blob "${REPO}@${layer}" > "$blob" tar xz -C "$OUT" < "$blob" -if [ ! -d "$OUT/base" ]; then - echo "❌ extracted bundle has no base/ directory" >&2 +if [ -z "$(ls -A "$OUT" 2>/dev/null)" ]; then + echo "❌ extracted bundle at ${OUT} is empty" >&2 exit 1 fi diff --git a/hack/ipam-clear-fixtures.sh b/hack/ipam-clear-fixtures.sh index 8d256dbc..986002ef 100755 --- a/hack/ipam-clear-fixtures.sh +++ b/hack/ipam-clear-fixtures.sh @@ -17,11 +17,11 @@ # Adapted from github.com/milo-os/ipam test/e2e/lib/clear-tenant-fixtures.sh at # ref 20865afe018c. # -# Usage: ipam-clear-fixtures.sh [project...] +# Usage: ipam-clear-fixtures.sh [project...] set -euo pipefail -KCFG="${1:?usage: ipam-clear-fixtures.sh [project...]}" +CONTEXT="${1:?usage: ipam-clear-fixtures.sh [project...]}" shift PROJECTS=("$@") if [ ${#PROJECTS[@]} -eq 0 ]; then @@ -31,13 +31,12 @@ fi DELETE_TIMEOUT="${IPAM_CLEAR_TIMEOUT:-60s}" KINDS="ipclaims ipallocations ippools ipclasses" -if [ ! -f "$KCFG" ]; then - echo "❌ no impersonation kubeconfig at ${KCFG}" >&2 - exit 1 -fi +here="$(cd "$(dirname "$0")" && pwd)" +# Every call reaches IPAM as a project; a request without the project extras +# reads nothing, which would make an empty result look like a clean tenant. k() { - KUBECONFIG="$KCFG" kubectl --context "$1" "${@:2}" + IPAM_KUBE_CONTEXT="$CONTEXT" "${here}/ipam-tenant-kubectl.sh" "$@" } # Positive control. Reading a type that always resolves proves the aggregated @@ -45,9 +44,9 @@ k() { # it, an unreachable cluster or a wrong context name would make every delete # below a no-op and this script would report a clean tenant it never touched. verify_reachable() { - local ctx="$1" out - if ! out="$(k "$ctx" get ipclasses 2>&1)"; then - echo "❌ cannot reach IPAM as ${ctx}; refusing to report a clean tenant" >&2 + local proj="$1" out + if ! out="$(k "$proj" get ipclasses 2>&1)"; then + echo "❌ cannot reach IPAM as ${proj}; refusing to report a clean tenant" >&2 echo " ${out}" >&2 exit 1 fi @@ -59,29 +58,28 @@ verify_reachable() { # whether clearing actually worked. Anything else is surfaced rather than # swallowed. attempt_delete() { - local ctx="$1" kind="$2" out rc + local proj="$1" kind="$2" out rc set +e - out="$(k "$ctx" delete "$kind" --all --all-namespaces --ignore-not-found \ + out="$(k "$proj" delete "$kind" --all --all-namespaces --ignore-not-found \ --timeout="$DELETE_TIMEOUT" 2>&1)" rc=$? set -e if [ "$rc" -ne 0 ] && ! printf '%s' "$out" | grep -qiE 'not found|no matches for|cannot delete IPPool'; then - echo " ⚠️ deleting ${kind} in ${ctx}: ${out}" >&2 + echo " ⚠️ deleting ${kind} in ${proj}: ${out}" >&2 fi } residue() { - local ctx="$1" kind="$2" - k "$ctx" get "$kind" --all-namespaces \ + local proj="$1" kind="$2" + k "$proj" get "$kind" --all-namespaces \ -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {end}' 2>/dev/null || true } for proj in "${PROJECTS[@]}"; do - ctx="tenant-${proj}" - verify_reachable "$ctx" + verify_reachable "$proj" - attempt_delete "$ctx" ipclaims - attempt_delete "$ctx" ipallocations + attempt_delete "$proj" ipclaims + attempt_delete "$proj" ipallocations # Pools need more than one pass. A cascade leaves child pools carved out of # their parent, and the parent refuses to go while a carve is outstanding — @@ -89,11 +87,11 @@ for proj in "${PROJECTS[@]}"; do # usually attempted before the children that pin it. Each pass frees one # level of the chain; the chain is at most a few deep. for _ in 1 2 3 4; do - [ -z "$(residue "$ctx" ippools)" ] && break - attempt_delete "$ctx" ippools + [ -z "$(residue "$proj" ippools)" ] && break + attempt_delete "$proj" ippools done - attempt_delete "$ctx" ipclasses + attempt_delete "$proj" ipclasses # The authoritative check, over every kind. Pools and classes block the next # run with "already exists"; a stranded claim or allocation is worse, because @@ -101,7 +99,7 @@ for proj in "${PROJECTS[@]}"; do # run after that one. left="" for kind in $KINDS; do - found="$(residue "$ctx" "$kind")" + found="$(residue "$proj" "$kind")" [ -n "$found" ] && left="${left}${kind}: ${found}"$'\n' done if [ -n "$left" ]; then diff --git a/hack/ipam-tenant-kubectl.sh b/hack/ipam-tenant-kubectl.sh new file mode 100755 index 00000000..379d0f2a --- /dev/null +++ b/hack/ipam-tenant-kubectl.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Run kubectl against IPAM as a project tenant. +# +# IPAM scopes storage from three UserInfo.Extra keys. Milo's front gate supplies +# them as X-Remote-Extra-* requestheader extras; kubectl's --as-user-extra +# becomes Impersonate-Extra-*, which the front proxy re-emits as the same +# X-Remote-Extra-*, so it lands in UserInfo.Extra identically. A request +# carrying none of the three reads nothing and its writes are refused. +# +# This replaced a generated kubeconfig holding a flattened copy of the caller's +# credentials. kubectl takes the extras as flags directly, so there is no +# generated artefact to create before anything can talk to IPAM, and no token +# written to disk. +# +# Usage: ipam-tenant-kubectl.sh [kubectl args...] +# e.g. ipam-tenant-kubectl.sh project-alpha -n default get ipclaims + +set -euo pipefail + +PROJECT="${1:?usage: ipam-tenant-kubectl.sh [kubectl args...]}" +shift + +# The identity impersonated. Deliberately not cluster-admin: its access comes +# from the nso-ipam-tenant binding in test/e2e/fixtures/ipam/rbac.yaml, so the +# cross-project deny assertions cannot pass by privilege. +: "${IPAM_TENANT_USER:=e2e-tenant-tester}" +# Set by callers that already know which cluster; otherwise kubectl's current +# context applies. +CONTEXT="${IPAM_KUBE_CONTEXT:-}" + +set -- \ + --as="${IPAM_TENANT_USER}" \ + --as-user-extra="iam.miloapis.com/parent-api-group=resourcemanager.miloapis.com" \ + --as-user-extra="iam.miloapis.com/parent-type=Project" \ + --as-user-extra="iam.miloapis.com/parent-name=${PROJECT}" \ + "$@" + +if [ -n "$CONTEXT" ]; then + set -- --context "$CONTEXT" "$@" +fi + +exec kubectl "$@" diff --git a/hack/module-dir.sh b/hack/module-dir.sh deleted file mode 100755 index eeb2f03e..00000000 --- a/hack/module-dir.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Print the source directory of a Go module in the build list, downloading it -# into the module cache first if it is not already extracted there. -# -# `go list -m -f {{.Dir}}` does NOT download. On a cold module cache it prints -# an empty line and exits 0 — a silent failure, which is the dangerous part: -# interpolated into a path it produces something like "/config/crd/bases/..." -# with a leading slash, and the error surfaces as a missing file somewhere -# unrelated to the actual cause. That is exactly what broke CI, on runners whose -# cache happened to be cold, while passing everywhere the cache was warm. -# -# Usage: module-dir.sh -# e.g. module-dir.sh go.miloapis.com/dns-operator - -set -euo pipefail - -MODULE="${1:?usage: module-dir.sh }" - -repo_root="$(cd "$(dirname "$0")/.." && pwd)" -cd "$repo_root" - -resolve() { - # Discard stderr: a cold cache makes `go list` chatter about downloading the - # toolchain, which would otherwise end up in the path. - go list -m -f '{{.Dir}}' "$MODULE" 2>/dev/null || true -} - -dir="$(resolve)" - -if [ -z "$dir" ]; then - echo "module ${MODULE} is not extracted in the module cache; downloading" >&2 - if ! go mod download "$MODULE" >&2; then - echo "❌ could not download ${MODULE}." >&2 - echo " Either it is not in this module's build list — \`go list -m ${MODULE}\`" >&2 - echo " should name it — or the module proxy is unreachable." >&2 - exit 1 - fi - dir="$(resolve)" -fi - -if [ -z "$dir" ]; then - echo "❌ could not resolve a directory for ${MODULE} after downloading it." >&2 - echo " Is it in the build list? \`go list -m ${MODULE}\` should name it." >&2 - exit 1 -fi - -if [ ! -d "$dir" ]; then - echo "❌ ${MODULE} resolved to ${dir}, which is not a directory" >&2 - exit 1 -fi - -echo "$dir" From 1581bbb9a49702f946aa3626ef7dc5ba63abfd9e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 13:20:50 -0500 Subject: [PATCH 06/13] refactor: import the parent extras milo publishes The impersonation extras and the resource manager group were spelled out here, duplicating constants a dependency already exports. Hardcoding the values also hid that the key milo names for the parent kind carries the string "parent-type". The project kind stays a literal. Milo exports no kind-name constant in any public package, so that is a convention gap rather than something to import. --- internal/controller/ipam_project_client.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/internal/controller/ipam_project_client.go b/internal/controller/ipam_project_client.go index dd7589c6..22a9957f 100644 --- a/internal/controller/ipam_project_client.go +++ b/internal/controller/ipam_project_client.go @@ -14,18 +14,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + iamv1alpha1 "go.miloapis.com/milo/pkg/apis/iam/v1alpha1" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" "go.datum.net/network-services-operator/internal/downstreamclient" ) -const ( - ipamParentAPIGroupExtraKey = "iam.miloapis.com/parent-api-group" - ipamParentTypeExtraKey = "iam.miloapis.com/parent-type" - ipamParentNameExtraKey = "iam.miloapis.com/parent-name" - - ipamParentAPIGroup = "resourcemanager.miloapis.com" - ipamParentType = "Project" -) +// Milo exports no constant for a kind name, so the project kind is spelled +// here. +const ipamParentType = "Project" // IPAMClientFactory returns a client bound to one project. Every IPAM request // goes through one, so no request can reach IPAM without naming a project. @@ -75,9 +72,9 @@ func (f *impersonatingIPAMClientFactory) ClientForProject(project string) (clien cfg.Impersonate = rest.ImpersonationConfig{ UserName: f.actAsUsername, Extra: map[string][]string{ - ipamParentAPIGroupExtraKey: {ipamParentAPIGroup}, - ipamParentTypeExtraKey: {ipamParentType}, - ipamParentNameExtraKey: {project}, + iamv1alpha1.ParentAPIGroupExtraKey: {resourcemanagerv1alpha1.GroupVersion.Group}, + iamv1alpha1.ParentKindExtraKey: {ipamParentType}, + iamv1alpha1.ParentNameExtraKey: {project}, }, } From 0cde29179d616e385a83a457b5106ef5465755fc Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 13:31:01 -0500 Subject: [PATCH 07/13] test: reach IPAM through the tenant kubectl wrapper The suites drove IPAM through a generated kubeconfig holding a flattened ServiceAccount token. kubectl takes the impersonation extras as flags, so the same requests can be made with no generated artefact and no credential on disk. This is the last consumer of that kubeconfig. Key changes: - Replace 36 call sites across 11 suites with hack/ipam-tenant-kubectl.sh, resolved from the repo root so nothing depends on chainsaw's working directory - Write the project-less request out inline rather than routing it through the wrapper: it is the same user carrying none of the parent extras, and what it proves is the absence of the impersonation the wrapper exists to add - Take the cluster from the kubeconfig chainsaw already supplies, so no suite names a context --- .../chainsaw-test.yaml | 20 +++--- .../chainsaw-test.yaml | 14 ++-- .../chainsaw-test.yaml | 4 +- .../chainsaw-test.yaml | 4 +- .../chainsaw-test.yaml | 20 +++--- .../chainsaw-test.yaml | 12 ++-- .../chainsaw-test.yaml | 16 ++--- .../chainsaw-test.yaml | 66 +++++++++++-------- .../chainsaw-test.yaml | 8 +-- .../chainsaw-test.yaml | 8 +-- .../chainsaw-test.yaml | 8 +-- 11 files changed, 94 insertions(+), 86 deletions(-) diff --git a/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml index d3c0463d..a39c11ce 100644 --- a/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml @@ -63,9 +63,9 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh record="${TMPDIR:-/tmp}/nic-boundifdel-addresses" - KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default get ipclaim \ + "$IPAM" project-alpha -n default get ipclaim \ -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}' > "$record" kubectl -n ipam-e2e-alpha get networkinterface nic-boundifdel -o jsonpath='{.metadata.uid}' > "${record}.uid" cat "$record" @@ -84,8 +84,8 @@ spec: script: content: | set -u - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-boundifdel --ignore-not-found --timeout=60s kubectl -n ipam-e2e-alpha delete networkinterface nic-boundifdel --ignore-not-found --timeout=60s @@ -126,8 +126,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } before=$(cat "${TMPDIR:-/tmp}/nic-boundifdel-addresses") after=$(ipam get ipclaim -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}') @@ -172,8 +172,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } before=$(cat "${TMPDIR:-/tmp}/nic-boundifdel-addresses") after=$(ipam get ipclaim -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}') @@ -208,8 +208,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } ipam get ipclass datum-public-v4 >/dev/null \ || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } diff --git a/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml index 100d26f1..b0f5e898 100644 --- a/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml @@ -4,9 +4,9 @@ # in the upstream IPAM service rather than values NSO invented. # # The claim lives in ipam-e2e-alpha, whose upstream-cluster-name label routes to -# project-alpha; the IPClaims are therefore read back through the -# tenant-project-alpha impersonation context, in the project namespace named by -# the cell namespace's upstream-namespace label (default). +# project-alpha; the IPClaims are therefore read back as project-alpha, in the +# project namespace named by the cell namespace's upstream-namespace label +# (default). apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: @@ -99,8 +99,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { ctx=$1; shift; KUBECONFIG=$IPAM_KUBECONFIG kubectl --context "tenant-$ctx" -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { proj=$1; shift; "$IPAM" "$proj" -n default "$@"; } for family in ipv6 ipv4; do upper=$(echo "$family" | tr '[:lower:]' '[:upper:]' | sed 's/IPV/IPv/') @@ -134,8 +134,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } ipam get ipclass datum-endpoint-v4 >/dev/null \ || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } diff --git a/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml index 2a92867f..06b1abb2 100644 --- a/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml @@ -68,11 +68,11 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh external=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-external -o jsonpath='{.status.externalAddresses[0].address}') internal=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-external -o jsonpath='{.status.addresses[0].address}') - held=$(KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default \ + held=$("$IPAM" project-alpha -n default \ get ipclaim nic-external-c-datum-public-v4 -o jsonpath='{.status.allocatedCIDR}') echo "external=${external} internal=${internal} ipam=${held}" diff --git a/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml index 9b7e96be..3524e07a 100644 --- a/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml @@ -114,8 +114,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } claims=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') echo "project-alpha IPClaims: $(echo $claims)" diff --git a/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml index 2e6d5170..95bf5475 100644 --- a/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml @@ -87,8 +87,8 @@ spec: script: content: | set -u - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-ifrelease --ignore-not-found --timeout=60s kubectl -n ipam-e2e-alpha delete networkinterface nic-ifrelease --ignore-not-found --timeout=60s @@ -103,8 +103,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } held() { ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true; } [ "$(held)" -eq 3 ] || { echo "expected 3 IPClaims for this interface, found $(held)"; exit 1; } @@ -130,8 +130,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } count=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true) [ "$count" -eq 3 ] || { echo "a retained interface should still hold its 3 addresses, found ${count}"; exit 1; } @@ -151,8 +151,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } ipam get ipclass datum-public-v4 >/dev/null \ || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } @@ -184,8 +184,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } stranded=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true) [ "$stranded" -eq 0 ] || { echo "${stranded} IPClaims are stranded with nothing in NSO naming them"; exit 1; } echo "IPAllocations still holding addresses after the release: $(ipam get ipallocation -o name | grep -c . || true)" diff --git a/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml index 951b4224..f9846ea2 100644 --- a/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml @@ -62,8 +62,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } count=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-reverse-' || true) [ "$count" -eq 3 ] || { echo "expected 3 IPClaims before the deletes, found ${count}"; exit 1; } catch: @@ -79,8 +79,8 @@ spec: script: content: | set -u - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-reverse --ignore-not-found --timeout=60s kubectl -n ipam-e2e-alpha delete networkinterface nic-reverse --ignore-not-found --timeout=60s @@ -110,8 +110,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } ipam get ipclass datum-public-v4 >/dev/null \ || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } diff --git a/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml index 12a9240f..588810a5 100644 --- a/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml @@ -67,8 +67,8 @@ spec: - script: content: | set -u - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-missingalloc --ignore-not-found --timeout=60s kubectl -n ipam-e2e-alpha delete networkinterface nic-missingalloc --ignore-not-found --timeout=60s @@ -83,8 +83,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } address=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-missingalloc -o jsonpath='{.status.addresses[0].address}') [ -n "$address" ] || { echo "the claim never published an address"; exit 1; } @@ -206,8 +206,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } address=$(cat "${TMPDIR:-/tmp}/nic-missingalloc-address") for name in nic-missingalloc-blocker nic-missingalloc-f-ipv4; do @@ -255,8 +255,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } address=$(cat "${TMPDIR:-/tmp}/nic-missingalloc-address") moved=$(ipam get ipclaim nic-missingalloc-f-ipv4 -o jsonpath='{.status.allocatedCIDR}') diff --git a/test/e2e/networkinterfaceclaim-project-isolation/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-project-isolation/chainsaw-test.yaml index eb2003c2..d7fab264 100644 --- a/test/e2e/networkinterfaceclaim-project-isolation/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-project-isolation/chainsaw-test.yaml @@ -148,50 +148,58 @@ spec: - name: Neither project can see the other's IPClaims description: | - Read back through the per-project impersonation contexts. A project that - can read the other's IPClaim means the impersonation is not scoping - anything and every other assertion in this suite is coincidence. + Read back as each project in turn. A project that can read the other's + IPClaim means the impersonation is not scoping anything, and every other + assertion in this suite is coincidence. - Each context earns the right to be believed first: the two project - contexts must find their own IPClaim, and tenant-platform — which is + Each identity earns the right to be believed first: the two projects + must find their own IPClaim, and the project-less request — which is meant to read nothing — must still reach IPAM, proving its blindness is - scoping rather than a broken kubeconfig. Its NotFound comes from the - server, and the IPClasses it cannot see are the same ones a project can. + scoping rather than a broken call. Its NotFound comes from the server, + and the IPClasses it cannot see are the same ones a project can. + + That request is the same user with none of the three parent extras, so + it is written out rather than routed through the wrapper: what is being + tested is precisely the absence of the impersonation the wrapper exists + to add. try: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { ctx=$1; shift; KUBECONFIG=$IPAM_KUBECONFIG kubectl --context "$ctx" -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { proj=$1; shift; "$IPAM" "$proj" -n default "$@"; } + # The same identity carrying none of the parent extras: no project, + # so IPAM scopes it to nothing. + noproject() { kubectl --as=e2e-tenant-tester "$@"; } visible() { ipam "$1" get ipclaim "$2" >/dev/null 2>&1; } - visible tenant-project-alpha nic-isolation-alpha-f-ipv4 || { echo "project-alpha cannot see its own IPClaim"; exit 1; } - visible tenant-project-beta nic-isolation-beta-f-ipv4 || { echo "project-beta cannot see its own IPClaim"; exit 1; } - visible tenant-project-alpha nic-isolation-legacy-f-ipv4 || { echo "the legacy-labelled namespace did not allocate in project-alpha"; exit 1; } + visible project-alpha nic-isolation-alpha-f-ipv4 || { echo "project-alpha cannot see its own IPClaim"; exit 1; } + visible project-beta nic-isolation-beta-f-ipv4 || { echo "project-beta cannot see its own IPClaim"; exit 1; } + visible project-alpha nic-isolation-legacy-f-ipv4 || { echo "the legacy-labelled namespace did not allocate in project-alpha"; exit 1; } - KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-platform get --raw /apis/ipam.miloapis.com/v1alpha1 >/dev/null \ - || { echo "tenant-platform cannot reach IPAM at all, so it proves nothing about scoping"; exit 1; } - classes=$(ipam tenant-project-alpha get ipclass -o name | wc -l | tr -d ' ') - [ "$classes" -gt 0 ] || { echo "project-alpha sees no IPClass, so the fixtures are not readable through this context"; exit 1; } - platform_classes=$(KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-platform get ipclass -o name | wc -l | tr -d ' ') - [ "$platform_classes" -eq 0 ] || { echo "tenant-platform reads ${platform_classes} IPClass objects; it is meant to hold no project"; exit 1; } - denial=$(ipam tenant-platform get ipclaim nic-isolation-alpha-f-ipv4 2>&1 || true) + noproject get --raw /apis/ipam.miloapis.com/v1alpha1 >/dev/null \ + || { echo "the project-less request cannot reach IPAM at all, so it proves nothing about scoping"; exit 1; } + classes=$(ipam project-alpha get ipclass -o name | wc -l | tr -d ' ') + [ "$classes" -gt 0 ] || { echo "project-alpha sees no IPClass, so the fixtures are not readable as that project"; exit 1; } + platform_classes=$(noproject get ipclass -o name | wc -l | tr -d ' ') + [ "$platform_classes" -eq 0 ] || { echo "the project-less request reads ${platform_classes} IPClass objects; it is meant to hold no project"; exit 1; } + denial=$(noproject -n default get ipclaim nic-isolation-alpha-f-ipv4 2>&1 || true) echo "$denial" | grep -Eq 'NotFound|Forbidden|forbidden|not found' \ - || { echo "tenant-platform's refusal did not come from the server: ${denial}"; exit 1; } + || { echo "the project-less refusal did not come from the server: ${denial}"; exit 1; } - ! visible tenant-project-beta nic-isolation-alpha-f-ipv4 || { echo "project-beta can read project-alpha's IPClaim"; exit 1; } - ! visible tenant-project-alpha nic-isolation-beta-f-ipv4 || { echo "project-alpha can read project-beta's IPClaim"; exit 1; } - ! visible tenant-platform nic-isolation-alpha-f-ipv4 || { echo "the project-less context can read project-alpha's IPClaim"; exit 1; } - ! visible tenant-platform nic-isolation-beta-f-ipv4 || { echo "the project-less context can read project-beta's IPClaim"; exit 1; } + ! visible project-beta nic-isolation-alpha-f-ipv4 || { echo "project-beta can read project-alpha's IPClaim"; exit 1; } + ! visible project-alpha nic-isolation-beta-f-ipv4 || { echo "project-alpha can read project-beta's IPClaim"; exit 1; } + ! noproject -n default get ipclaim nic-isolation-alpha-f-ipv4 >/dev/null 2>&1 || { echo "the project-less request can read project-alpha's IPClaim"; exit 1; } + ! noproject -n default get ipclaim nic-isolation-beta-f-ipv4 >/dev/null 2>&1 || { echo "the project-less request can read project-beta's IPClaim"; exit 1; } - name: The address IPAM holds is the address the claim published try: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - allocated() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context "$1" -n default get ipclaim "$2" -o jsonpath='{.status.allocatedCIDR}'; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + allocated() { "$IPAM" "$1" -n default get ipclaim "$2" -o jsonpath='{.status.allocatedCIDR}'; } check() { published=$(kubectl -n "$1" get networkinterfaceclaim "$2" -o jsonpath='{.status.addresses[0].address}') @@ -199,6 +207,6 @@ spec: [ "$published" = "$held" ] || { echo "$2 published ${published}, IPAM holds ${held}"; exit 1; } } - check ipam-e2e-alpha nic-isolation-alpha tenant-project-alpha - check ipam-e2e-beta nic-isolation-beta tenant-project-beta - check ipam-e2e-legacy nic-isolation-legacy tenant-project-alpha + check ipam-e2e-alpha nic-isolation-alpha project-alpha + check ipam-e2e-beta nic-isolation-beta project-beta + check ipam-e2e-legacy nic-isolation-legacy project-alpha diff --git a/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml index e294a6d3..0116f739 100644 --- a/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml @@ -65,9 +65,9 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh for name in nic-delete-f-ipv6 nic-delete-f-ipv4 nic-delete-c-datum-public-v4; do - phase=$(KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default \ + phase=$("$IPAM" project-alpha -n default \ get ipclaim "$name" -o jsonpath='{.status.phase}' 2>/dev/null || true) echo "${name}: ${phase:-missing}" [ "$phase" = "Bound" ] || { echo "IPClaim ${name} is ${phase:-missing}, want Bound"; exit 1; } @@ -85,8 +85,8 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } ipam get ipclass datum-public-v4 >/dev/null \ || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } diff --git a/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml index fa16ee13..98a99fe9 100644 --- a/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml @@ -82,9 +82,9 @@ spec: script: content: | set -u - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh record="${TMPDIR:-/tmp}/nic-retain-addresses" - ipam() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default "$@"; } + ipam() { "$IPAM" project-alpha -n default "$@"; } kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-retain --ignore-not-found --timeout=60s kubectl -n ipam-e2e-alpha delete networkinterface nic-retain --ignore-not-found --timeout=60s @@ -130,10 +130,10 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh read -r want6 want4 wantext _ < "${TMPDIR:-/tmp}/nic-retain-addresses" || true - held() { KUBECONFIG=$IPAM_KUBECONFIG kubectl --context tenant-project-alpha -n default get ipclaim "$1" -o jsonpath='{.status.allocatedCIDR}'; } + held() { "$IPAM" project-alpha -n default get ipclaim "$1" -o jsonpath='{.status.allocatedCIDR}'; } [ "$(held nic-retain-f-ipv6)" = "$want6" ] || { echo "IPv6 address was not retained in IPAM"; exit 1; } [ "$(held nic-retain-f-ipv4)" = "$want4" ] || { echo "IPv4 address was not retained in IPAM"; exit 1; } diff --git a/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml index 6fd559af..5e31562f 100644 --- a/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml @@ -114,12 +114,12 @@ spec: - script: content: | set -eu - IPAM_KUBECONFIG="${TMPDIR:-/tmp}/.ipam-tenant-impersonation.yaml" - ipam() { ctx=$1; shift; KUBECONFIG=$IPAM_KUBECONFIG kubectl --context "$ctx" -n default "$@"; } + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { proj=$1; shift; "$IPAM" "$proj" -n default "$@"; } names() { ipam "$1" get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'; } - alpha=$(names tenant-project-alpha) - beta=$(names tenant-project-beta) + alpha=$(names project-alpha) + beta=$(names project-beta) echo "project-alpha IPClaims: $(echo $alpha)" echo "project-beta IPClaims: $(echo $beta)" From 28b2ae74cca4565c66544bc1e9cb8878a1dc56eb Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 13:46:25 -0500 Subject: [PATCH 08/13] chore: bump IPAM to the published error taxonomy Picks up the public package that names why an allocation was refused, so the operator can stop inferring it from status codes. --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 4f21ba48..cbcc9066 100644 --- a/go.mod +++ b/go.mod @@ -273,7 +273,7 @@ require ( go.etcd.io/etcd/api/v3 v3.6.8 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect - go.miloapis.com/ipam v0.3.2-0.20260813020700-13bd2c8c077b + go.miloapis.com/ipam v0.3.2-0.20260813184449-4fac0aa96194 go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect @@ -326,4 +326,4 @@ require ( sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) -replace go.miloapis.com/ipam => github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b +replace go.miloapis.com/ipam => github.com/milo-os/ipam v0.3.2-0.20260813184449-4fac0aa96194 diff --git a/go.sum b/go.sum index 6d66fd3d..e2b458c4 100644 --- a/go.sum +++ b/go.sum @@ -453,8 +453,8 @@ github.com/microsoft/go-mssqldb v1.10.0/go.mod h1:mnG7lGa9iYJbzJqGCXyuQCegStKMr3 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= -github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b h1:iJ0zDKVRgC4cz0/eOX8dp+wBNw8iqaGI87zwgqdBZH8= -github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b/go.mod h1:Jj7xg4lJi9psE0+4PuOg/GQOG8rG13h112xYoM994rc= +github.com/milo-os/ipam v0.3.2-0.20260813184449-4fac0aa96194 h1:4AUWL/7kuD5i0zT4zvy/oGcH6eDISJaOqPc9LdYKuNI= +github.com/milo-os/ipam v0.3.2-0.20260813184449-4fac0aa96194/go.mod h1:Jj7xg4lJi9psE0+4PuOg/GQOG8rG13h112xYoM994rc= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= From 6a019e3fd9b83acbef29f4cd90ac9e730826ae68 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 13:55:15 -0500 Subject: [PATCH 09/13] refactor: ask IPAM why an allocation was refused The classifier read HTTP status codes, so it could not tell an unconfigured class from a bad prefix length from a scope short a role: IPAM answered all of them with the same 400. Exhaustion was recognised by a hardcoded 507, knowledge that belongs to IPAM rather than to us. IPAM now publishes its taxonomy, so the classifier asks for a reason and maps it onto our condition reasons, which are unchanged. A reason this build does not know, and a server too old to send one, both land on the existing catch-all rather than being guessed at. Key changes: - classify on the published reason and drop the local 507 constant - name the pool that ran out in the exhaustion message, and the retained IPAllocation in the conflict message, when IPAM says which - say which piece of configuration is missing instead of hedging that it "usually means" a class or pool is not configured; the hedge survives only for a refusal this build cannot name - build the fake's refusals with the upstream constructors so it cannot drift from what the server emits, and classify through the wire form --- internal/controller/ipam_errors.go | 120 ++++++++--- internal/controller/ipam_errors_test.go | 187 ++++++++++++++++++ .../networkinterfaceclaim_controller_test.go | 15 +- 3 files changed, 291 insertions(+), 31 deletions(-) create mode 100644 internal/controller/ipam_errors_test.go diff --git a/internal/controller/ipam_errors.go b/internal/controller/ipam_errors.go index 805db6c5..2b43dcf4 100644 --- a/internal/controller/ipam_errors.go +++ b/internal/controller/ipam_errors.go @@ -3,55 +3,127 @@ package controller import ( - "errors" - "net/http" + "fmt" + "strings" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "go.miloapis.com/ipam/pkg/ipamerrors" ) -// allocationFailureReason classifies why IPAM refused an allocation, by status -// code only. IPAM's own message is passed through unread. +// allocationFailureReason classifies why IPAM refused an allocation. The values +// are condition reasons on our own API, so they are ours to keep stable and +// deliberately coarser than IPAM's taxonomy. type allocationFailureReason string const ( allocationFailureExhausted allocationFailureReason = "AddressPoolExhausted" allocationFailureConflict allocationFailureReason = "RetainedAddressConflict" - // Usually an address class or pool that is not configured, rather than a - // bad request, and the status code does not tell the two apart. - allocationFailureRejected allocationFailureReason = "AddressAllocationRejected" - allocationFailureUnknown allocationFailureReason = "AllocationFailed" + allocationFailureRejected allocationFailureReason = "AddressAllocationRejected" + allocationFailureUnknown allocationFailureReason = "AllocationFailed" ) -const httpStatusInsufficientStorage = 507 - +// classifyAllocationFailure asks IPAM why it refused rather than reading the +// status code. A reason this build does not know, and a server too old to send +// one, both land on the catch-all instead of being guessed at. func classifyAllocationFailure(err error) allocationFailureReason { - var status apierrors.APIStatus - if !errors.As(err, &status) { - return allocationFailureUnknown - } - - switch int(status.Status().Code) { - case httpStatusInsufficientStorage: + switch ipamerrors.ReasonFor(err) { + case ipamerrors.ReasonExhausted: return allocationFailureExhausted - case http.StatusConflict: + case ipamerrors.ReasonAllocationRetained, ipamerrors.ReasonClaimExists: return allocationFailureConflict - case http.StatusBadRequest, http.StatusUnprocessableEntity: + case ipamerrors.ReasonClassNotFound, + ipamerrors.ReasonNoDefaultClass, + ipamerrors.ReasonNoOfferingPool, + ipamerrors.ReasonPrefixLengthRejected, + ipamerrors.ReasonScopeRolesMissing, + ipamerrors.ReasonNoProjectScope: return allocationFailureRejected default: return allocationFailureUnknown } } +// allocationFailureMessage writes the condition message. IPAM's own message is +// appended verbatim in every branch: it is the only account of the failure, and +// parsing it would tie us to its wording. func allocationFailureMessage(reason allocationFailureReason, request allocationRequest, err error) string { switch reason { case allocationFailureExhausted: - return "No address is left for " + request.describe() + ": " + err.Error() + return "No address is left for " + request.describe() + exhaustedPoolSuffix(err) + ": " + err.Error() case allocationFailureConflict: - return "A retained allocation still holds the name for " + request.describe() + ": " + err.Error() + return "A retained allocation still holds the name for " + request.describe() + + retainedAllocationSuffix(err) + ": " + err.Error() case allocationFailureRejected: - return "IPAM would not allocate " + request.describe() + - ". This usually means the address class, or a pool backing it, is not configured in this project: " + err.Error() + return rejectionMessage(request, err) + ": " + err.Error() default: return "Allocating " + request.describe() + " failed: " + err.Error() } } + +// rejectionMessage says which piece of configuration is missing. IPAM used to +// answer every one of these with an indistinguishable 400, so this could only +// guess; it now asks. +func rejectionMessage(request allocationRequest, err error) string { + switch ipamerrors.ReasonFor(err) { + case ipamerrors.ReasonClassNotFound: + return "IPAM has no address class for " + request.describe() + + ". The class must exist in this project" + case ipamerrors.ReasonNoDefaultClass: + return "IPAM has no default address class for " + request.describe() + + ", and the claim named no class" + case ipamerrors.ReasonNoOfferingPool: + return "No address pool offers the class for " + request.describe() + case ipamerrors.ReasonPrefixLengthRejected: + return "IPAM would not allocate " + request.describe() + + " at the requested size, which its class does not allow" + case ipamerrors.ReasonScopeRolesMissing: + return "The claim for " + request.describe() + " is missing the scope " + + missingScopeRoles(err) + " its class requires" + case ipamerrors.ReasonNoProjectScope: + return "The claim for " + request.describe() + " reached IPAM without a project" + default: + // A refusal this build cannot name, from a server that sends no reason + // or one added since. The hedge belongs here and nowhere else. + return "IPAM would not allocate " + request.describe() + + ". This usually means the address class, or a pool backing it, is not configured in this project" + } +} + +// exhaustedPoolSuffix names the pool that ran out. A claim names a class rather +// than a pool, so without this an operator is told an address ran out with +// nothing to widen. Exhaustion while provisioning an ancestor pool names none, +// and reporting the wrong one would be worse than reporting nothing. +func exhaustedPoolSuffix(err error) string { + pool, ok := ipamerrors.ExhaustedPool(err) + if !ok { + return "" + } + return fmt.Sprintf(" in IPPool %q", pool) +} + +// retainedAllocationSuffix names the IPAllocation holding the address, which is +// the object that has to go for the name to be reusable. +func retainedAllocationSuffix(err error) string { + allocation, ok := ipamerrors.RetainedAllocation(err) + if !ok { + return "" + } + return fmt.Sprintf(" (IPAllocation %q)", allocation) +} + +// missingScopeRoles names every role the claim was short, in one message. IPAM +// reports them all at once so an operator is not sent back a second time. +func missingScopeRoles(err error) string { + roles := ipamerrors.MissingScopeRoles(err) + if len(roles) == 0 { + return "roles" + } + quoted := make([]string, len(roles)) + for i, role := range roles { + quoted[i] = fmt.Sprintf("%q", role) + } + noun := "roles" + if len(roles) == 1 { + noun = "role" + } + return noun + " " + strings.Join(quoted, ", ") +} diff --git a/internal/controller/ipam_errors_test.go b/internal/controller/ipam_errors_test.go new file mode 100644 index 00000000..42119bc5 --- /dev/null +++ b/internal/controller/ipam_errors_test.go @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "encoding/json" + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "go.miloapis.com/ipam/pkg/ipamerrors" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +var ipclaimsResource = schema.GroupResource{Group: "ipam.miloapis.com", Resource: "ipclaims"} + +// fromTheWire is the error the controller actually classifies: the refusal as +// it arrives after the apiserver serialised it and client-go decoded it. A +// classifier proven only against an in-process value proves nothing. +func fromTheWire(t *testing.T, err *apierrors.StatusError) error { + t.Helper() + data, marshalErr := json.Marshal(err.ErrStatus) + require.NoError(t, marshalErr) + var status metav1.Status + require.NoError(t, json.Unmarshal(data, &status)) + return &apierrors.StatusError{ErrStatus: status} +} + +func v4Request() allocationRequest { + return allocationRequest{family: networkingv1alpha.IPv4Protocol} +} + +func classRequest() allocationRequest { + return allocationRequest{className: "public-v4", family: networkingv1alpha.IPv4Protocol} +} + +// The condition reasons are our API surface. IPAM's taxonomy is finer than +// ours, and this is where the two are pinned together. +func TestClassifyAllocationFailure(t *testing.T) { + for _, tc := range []struct { + name string + err error + want allocationFailureReason + }{ + {"pool exhausted", ipamerrors.NewPoolExhausted("datum-v4", `IPPool "datum-v4" is exhausted`), allocationFailureExhausted}, + {"ancestor exhausted", ipamerrors.New(ipamerrors.ReasonExhausted, "ipam: pool exhausted"), allocationFailureExhausted}, + {"retained allocation", ipamerrors.NewRetainedAllocation(ipclaimsResource, "eth0-f-ipv4", "alloc-9f2c", "retained"), allocationFailureConflict}, + {"claim exists", ipamerrors.NewClaimExists(ipclaimsResource, "eth0-f-ipv4", "already holds an allocation"), allocationFailureConflict}, + {"class not found", ipamerrors.New(ipamerrors.ReasonClassNotFound, "ipam: class not found"), allocationFailureRejected}, + {"no default class", ipamerrors.New(ipamerrors.ReasonNoDefaultClass, "ipam: no default class"), allocationFailureRejected}, + {"no offering pool", ipamerrors.New(ipamerrors.ReasonNoOfferingPool, "ipam: no pool offers this class"), allocationFailureRejected}, + {"prefix length", ipamerrors.New(ipamerrors.ReasonPrefixLengthRejected, "prefixLength 20 is outside the class range"), allocationFailureRejected}, + {"scope roles", ipamerrors.NewScopeRolesMissing([]string{"location"}, "scope is missing role"), allocationFailureRejected}, + {"no project scope", ipamerrors.New(ipamerrors.ReasonNoProjectScope, "ipam: request carries no project scope"), allocationFailureRejected}, + } { + t.Run(tc.name, func(t *testing.T) { + status, ok := tc.err.(*apierrors.StatusError) + require.True(t, ok) + require.Equal(t, tc.want, classifyAllocationFailure(fromTheWire(t, status))) + }) + } +} + +// A reason added to IPAM after this build, and anything that never came from +// IPAM, land on the catch-all. Guessing would put a wrong reason on our API. +func TestClassifyAllocationFailureDoesNotGuess(t *testing.T) { + added := &apierrors.StatusError{ErrStatus: metav1.Status{ + Status: metav1.StatusFailure, + Code: http.StatusBadRequest, + Details: &metav1.StatusDetails{Causes: []metav1.StatusCause{{ + Type: metav1.CauseType("AddedAfterThisBuild"), + }}}, + }} + require.Equal(t, allocationFailureUnknown, classifyAllocationFailure(fromTheWire(t, added))) + require.Equal(t, allocationFailureUnknown, classifyAllocationFailure(errors.New("connection refused"))) + require.Equal(t, allocationFailureUnknown, + classifyAllocationFailure(apierrors.NewInternalError(errors.New("boom")))) +} + +// An operator may upgrade this operator before IPAM. An older server answers +// exhaustion with a bare 507 and no reason, and that must still read as +// exhaustion rather than as an unexplained failure. +func TestClassifyAllocationFailureReadsAnOlderServer(t *testing.T) { + legacy := apierrors.NewGenericServerResponse( + 507, "POST", ipclaimsResource, "", "IPPool exhausted", 0, false) + require.Equal(t, allocationFailureExhausted, classifyAllocationFailure(fromTheWire(t, legacy))) +} + +// Naming the pool that ran out was the point of asking IPAM for it. A message +// that says an address ran out without saying what ran out leaves an operator +// with nothing to widen. +func TestExhaustionMessageNamesThePool(t *testing.T) { + err := fromTheWire(t, ipamerrors.NewPoolExhausted("datum-v4", `IPPool "datum-v4" is exhausted`)) + message := allocationFailureMessage(classifyAllocationFailure(err), v4Request(), err) + + require.Contains(t, message, "datum-v4", "the pool that ran out belongs in the condition") + require.Contains(t, message, "an IPv4 address") + require.Contains(t, message, `IPPool "datum-v4" is exhausted`, "IPAM's message is carried verbatim") +} + +// Exhaustion while a level of the class chain is provisioned names no pool. An +// invented name would send an operator to the wrong place. +func TestExhaustionMessageWithoutAPoolNamesNone(t *testing.T) { + err := fromTheWire(t, ipamerrors.New(ipamerrors.ReasonExhausted, "ipam: pool exhausted")) + message := allocationFailureMessage(classifyAllocationFailure(err), classRequest(), err) + + require.NotContains(t, message, "IPPool") + require.Contains(t, message, `an address of class "public-v4"`) + require.Contains(t, message, "ipam: pool exhausted") +} + +// The refusals that used to be one indistinguishable 400 now say which piece of +// configuration is missing. +func TestRejectionMessagesSayWhichConfigurationIsMissing(t *testing.T) { + for _, tc := range []struct { + name string + err *apierrors.StatusError + contains string + }{ + {"class not found", ipamerrors.New(ipamerrors.ReasonClassNotFound, "ipam: class not found"), "no address class"}, + {"no default class", ipamerrors.New(ipamerrors.ReasonNoDefaultClass, "ipam: no default class"), "no default address class"}, + {"no offering pool", ipamerrors.New(ipamerrors.ReasonNoOfferingPool, "ipam: no pool offers this class"), "No address pool offers"}, + {"prefix length", ipamerrors.New(ipamerrors.ReasonPrefixLengthRejected, "prefixLength 20 is outside"), "at the requested size"}, + {"no project scope", ipamerrors.New(ipamerrors.ReasonNoProjectScope, "ipam: request carries no project scope"), "without a project"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := fromTheWire(t, tc.err) + message := allocationFailureMessage(classifyAllocationFailure(err), classRequest(), err) + + require.Contains(t, message, tc.contains) + require.Contains(t, message, tc.err.ErrStatus.Message, "IPAM's message is carried verbatim") + require.NotContains(t, message, "usually means", + "the reason is known now, so the message must not hedge") + }) + } +} + +// A claim short two roles is one refusal naming both, so the condition names +// both rather than sending an operator back for the second. +func TestScopeRolesMessageNamesEveryMissingRole(t *testing.T) { + err := fromTheWire(t, ipamerrors.NewScopeRolesMissing( + []string{"network", "location"}, "scope is missing roles")) + message := allocationFailureMessage(classifyAllocationFailure(err), v4Request(), err) + + require.Contains(t, message, `"network"`) + require.Contains(t, message, `"location"`) + require.Contains(t, message, "scope roles") +} + +// The hedge survives for exactly one case: a refusal this build cannot name. +func TestUnnamedRejectionStillHedges(t *testing.T) { + err := fromTheWire(t, apierrors.NewBadRequest("something IPAM has not classified")) + require.Equal(t, allocationFailureUnknown, classifyAllocationFailure(err)) + require.Contains(t, + allocationFailureMessage(allocationFailureRejected, classRequest(), err), + "usually means") +} + +// The two conflicts are one condition reason to our consumers, but the message +// names the allocation that has to go when IPAM says which one it is. +func TestConflictMessageNamesTheRetainedAllocation(t *testing.T) { + retained := fromTheWire(t, ipamerrors.NewRetainedAllocation( + ipclaimsResource, "eth0-f-ipv4", "alloc-9f2c", "retained by an earlier claim")) + message := allocationFailureMessage(classifyAllocationFailure(retained), v4Request(), retained) + require.Contains(t, message, "alloc-9f2c") + require.Contains(t, message, "retained by an earlier claim") + + exists := fromTheWire(t, ipamerrors.NewClaimExists( + ipclaimsResource, "eth0-f-ipv4", "already holds an allocation")) + require.NotContains(t, + allocationFailureMessage(classifyAllocationFailure(exists), v4Request(), exists), + "IPAllocation") +} + +// Whatever the reason, IPAM's account of the failure reaches the condition. +// Parsing it instead would tie our conditions to its wording. +func TestEveryMessageCarriesIPAMsOwnMessage(t *testing.T) { + err := fromTheWire(t, apierrors.NewInternalError(errors.New("postgres is unreachable"))) + message := allocationFailureMessage(classifyAllocationFailure(err), v4Request(), err) + require.Contains(t, message, "postgres is unreachable") +} diff --git a/internal/controller/networkinterfaceclaim_controller_test.go b/internal/controller/networkinterfaceclaim_controller_test.go index e62f1dfe..e175b30a 100644 --- a/internal/controller/networkinterfaceclaim_controller_test.go +++ b/internal/controller/networkinterfaceclaim_controller_test.go @@ -23,7 +23,6 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation" clientgoscheme "k8s.io/client-go/kubernetes/scheme" eventsv1client "k8s.io/client-go/kubernetes/typed/events/v1" @@ -37,6 +36,7 @@ import ( "sigs.k8s.io/yaml" ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + "go.miloapis.com/ipam/pkg/ipamerrors" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" "go.datum.net/network-services-operator/internal/config" @@ -173,10 +173,11 @@ func (f *fakeIPAM) retainedConflictLocked(ipClaim *ipamv1alpha1.IPClaim) error { if _, orphaned := f.orphans[allocationName]; !orphaned { return nil } - return apierrors.NewConflict( + return ipamerrors.NewRetainedAllocation( ipamv1alpha1.SchemeGroupVersion.WithResource("ipclaims").GroupResource(), ipClaim.Name, - fmt.Errorf("an allocation under this identity already exists: IPAllocation %q, retained by an earlier claim of the same name; delete it to reuse the name", allocationName), + allocationName, + fmt.Sprintf("an allocation under this identity already exists: IPAllocation %q, retained by an earlier claim of the same name; delete it to reuse the name", allocationName), ) } @@ -584,10 +585,8 @@ func TestNetworkInterfaceClaimRollsBackPartialAllocation(t *testing.T) { s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}) - s.ipam.refuse("half-eth0-f-ipv4", apierrors.NewGenericServerResponse( - httpStatusInsufficientStorage, "POST", - schema.GroupResource{Group: "ipam.miloapis.com", Resource: "ipclaims"}, - "", "IPPool exhausted", 0, false)) + s.ipam.refuse("half-eth0-f-ipv4", + ipamerrors.NewPoolExhausted("datum-v4", "IPPool exhausted")) claim := s.createClaim("half-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ InterfaceName: "eth0", @@ -611,6 +610,8 @@ func TestNetworkInterfaceClaimRollsBackPartialAllocation(t *testing.T) { require.Equal(t, string(allocationFailureExhausted), condition.Reason) require.Contains(t, condition.Message, "IPPool exhausted", "IPAM's own message is carried through verbatim") + require.Contains(t, condition.Message, "datum-v4", + "the pool that ran out is what an operator has to widen") } func TestNetworkInterfaceRetainRebindsSameAddresses(t *testing.T) { From c6d4decee1075db8717a29d8a69bcceab19840ef Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 14:27:51 -0500 Subject: [PATCH 10/13] docs: say why the address is read before it is claimed The comment justified reading an address before claiming it with an IPAM defect that has since been fixed, which would send the next reader looking for a bug that is no longer there. The read is still right, for a reason the comment did not give: it does not depend on how a particular IPAM version reports a duplicate, and the operator can run ahead of the server it talks to. --- internal/controller/networkinterfaceclaim_controller.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/controller/networkinterfaceclaim_controller.go b/internal/controller/networkinterfaceclaim_controller.go index c48827d1..d44e9505 100644 --- a/internal/controller/networkinterfaceclaim_controller.go +++ b/internal/controller/networkinterfaceclaim_controller.go @@ -557,8 +557,10 @@ func (r *NetworkInterfaceClaimReconciler) allocate( ipClaim.Spec.IPFamily = ipamv1alpha1.IPFamily(request.family) } - // IPAM reports a duplicate name inconsistently, so ask whether the - // address exists instead of reading the refusal. + // Ask whether the address exists rather than reading the refusal. This + // holds against any IPAM version, whatever that version reports for a + // duplicate, which matters while the operator can run ahead of the + // server it talks to. The cost is one read per address per reconcile. existing := &ipamv1alpha1.IPClaim{} getErr := ipamClient.Get(ctx, client.ObjectKeyFromObject(ipClaim), existing) if getErr != nil && !apierrors.IsNotFound(getErr) { @@ -569,6 +571,8 @@ func (r *NetworkInterfaceClaimReconciler) allocate( if getErr == nil { ipClaim = existing } else if createErr := ipamClient.Create(ctx, ipClaim); createErr != nil { + // The create can still lose a race with another writer, so ask + // again before calling this a failure to allocate. raced := &ipamv1alpha1.IPClaim{} if err := ipamClient.Get(ctx, client.ObjectKeyFromObject(ipClaim), raced); err != nil { rollback() From d5869dba4bf5fb8bb890e436ecc03152e732f76b Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 14:29:20 -0500 Subject: [PATCH 11/13] fix: stop naming the same thing twice in a condition The condition read "No address is left for an IPv4 address in IPPool "datum-v4": IPPool "datum-v4" is exhausted". We pulled detail out of the refusal, wrote prose around it, then appended IPAM's message, which already carried that detail. The pool, the missing scope roles and the retained allocation were each named twice. Appending verbatim was right while we could not classify a refusal. Now that we can, our half of the message says only what IPAM cannot know: which of the claim's addresses was being allocated. Key changes: - drop the per-reason prose that restated the class, project, roles and remedy IPAM's own message spells out - keep building the exhaustion message from the pool in the refusal's details, so the pool survives a rewording upstream - assert whole rendered messages, and that no message names a fact twice --- internal/controller/ipam_errors.go | 98 +++------- internal/controller/ipam_errors_test.go | 178 +++++++++--------- .../networkinterfaceclaim_controller_test.go | 10 +- 3 files changed, 119 insertions(+), 167 deletions(-) diff --git a/internal/controller/ipam_errors.go b/internal/controller/ipam_errors.go index 2b43dcf4..3b5f4ce0 100644 --- a/internal/controller/ipam_errors.go +++ b/internal/controller/ipam_errors.go @@ -4,7 +4,6 @@ package controller import ( "fmt" - "strings" "go.miloapis.com/ipam/pkg/ipamerrors" ) @@ -42,88 +41,33 @@ func classifyAllocationFailure(err error) allocationFailureReason { } } -// allocationFailureMessage writes the condition message. IPAM's own message is -// appended verbatim in every branch: it is the only account of the failure, and -// parsing it would tie us to its wording. +// allocationFailureMessage says which address we were asking for and hands over +// to IPAM's account of why it said no. +// +// IPAM's message already carries the detail a reader acts on: the pool that ran +// out, the roles a scope was short and the class that required them, the +// allocation holding a name. Restating any of it here would print it twice, and +// reconstructing it would drift from what IPAM actually said. What IPAM cannot +// know is which of this claim's addresses was being allocated, so that is our +// half and the whole of it. func allocationFailureMessage(reason allocationFailureReason, request allocationRequest, err error) string { switch reason { case allocationFailureExhausted: - return "No address is left for " + request.describe() + exhaustedPoolSuffix(err) + ": " + err.Error() + // The one message we write ourselves. IPAM's reads the same, but the + // pool is the thing an operator has to widen, and taking it from the + // error's details rather than its prose keeps it in the condition + // whatever IPAM's wording does later. + if pool, ok := ipamerrors.ExhaustedPool(err); ok { + return fmt.Sprintf("No address is left for %s: IPPool %q is exhausted", request.describe(), pool) + } + // Exhaustion while a level of the class chain was provisioned names no + // pool, and IPAM's message is the only account of which level ran out. + return "No address is left for " + request.describe() + ": " + err.Error() case allocationFailureConflict: - return "A retained allocation still holds the name for " + request.describe() + - retainedAllocationSuffix(err) + ": " + err.Error() + return "Another allocation still holds the name for " + request.describe() + ": " + err.Error() case allocationFailureRejected: - return rejectionMessage(request, err) + ": " + err.Error() + return "IPAM would not allocate " + request.describe() + ": " + err.Error() default: return "Allocating " + request.describe() + " failed: " + err.Error() } } - -// rejectionMessage says which piece of configuration is missing. IPAM used to -// answer every one of these with an indistinguishable 400, so this could only -// guess; it now asks. -func rejectionMessage(request allocationRequest, err error) string { - switch ipamerrors.ReasonFor(err) { - case ipamerrors.ReasonClassNotFound: - return "IPAM has no address class for " + request.describe() + - ". The class must exist in this project" - case ipamerrors.ReasonNoDefaultClass: - return "IPAM has no default address class for " + request.describe() + - ", and the claim named no class" - case ipamerrors.ReasonNoOfferingPool: - return "No address pool offers the class for " + request.describe() - case ipamerrors.ReasonPrefixLengthRejected: - return "IPAM would not allocate " + request.describe() + - " at the requested size, which its class does not allow" - case ipamerrors.ReasonScopeRolesMissing: - return "The claim for " + request.describe() + " is missing the scope " + - missingScopeRoles(err) + " its class requires" - case ipamerrors.ReasonNoProjectScope: - return "The claim for " + request.describe() + " reached IPAM without a project" - default: - // A refusal this build cannot name, from a server that sends no reason - // or one added since. The hedge belongs here and nowhere else. - return "IPAM would not allocate " + request.describe() + - ". This usually means the address class, or a pool backing it, is not configured in this project" - } -} - -// exhaustedPoolSuffix names the pool that ran out. A claim names a class rather -// than a pool, so without this an operator is told an address ran out with -// nothing to widen. Exhaustion while provisioning an ancestor pool names none, -// and reporting the wrong one would be worse than reporting nothing. -func exhaustedPoolSuffix(err error) string { - pool, ok := ipamerrors.ExhaustedPool(err) - if !ok { - return "" - } - return fmt.Sprintf(" in IPPool %q", pool) -} - -// retainedAllocationSuffix names the IPAllocation holding the address, which is -// the object that has to go for the name to be reusable. -func retainedAllocationSuffix(err error) string { - allocation, ok := ipamerrors.RetainedAllocation(err) - if !ok { - return "" - } - return fmt.Sprintf(" (IPAllocation %q)", allocation) -} - -// missingScopeRoles names every role the claim was short, in one message. IPAM -// reports them all at once so an operator is not sent back a second time. -func missingScopeRoles(err error) string { - roles := ipamerrors.MissingScopeRoles(err) - if len(roles) == 0 { - return "roles" - } - quoted := make([]string, len(roles)) - for i, role := range roles { - quoted[i] = fmt.Sprintf("%q", role) - } - noun := "roles" - if len(roles) == 1 { - noun = "role" - } - return noun + " " + strings.Join(quoted, ", ") -} diff --git a/internal/controller/ipam_errors_test.go b/internal/controller/ipam_errors_test.go index 42119bc5..af6f1a28 100644 --- a/internal/controller/ipam_errors_test.go +++ b/internal/controller/ipam_errors_test.go @@ -45,7 +45,7 @@ func classRequest() allocationRequest { func TestClassifyAllocationFailure(t *testing.T) { for _, tc := range []struct { name string - err error + err *apierrors.StatusError want allocationFailureReason }{ {"pool exhausted", ipamerrors.NewPoolExhausted("datum-v4", `IPPool "datum-v4" is exhausted`), allocationFailureExhausted}, @@ -60,9 +60,7 @@ func TestClassifyAllocationFailure(t *testing.T) { {"no project scope", ipamerrors.New(ipamerrors.ReasonNoProjectScope, "ipam: request carries no project scope"), allocationFailureRejected}, } { t.Run(tc.name, func(t *testing.T) { - status, ok := tc.err.(*apierrors.StatusError) - require.True(t, ok) - require.Equal(t, tc.want, classifyAllocationFailure(fromTheWire(t, status))) + require.Equal(t, tc.want, classifyAllocationFailure(fromTheWire(t, tc.err))) }) } } @@ -92,96 +90,106 @@ func TestClassifyAllocationFailureReadsAnOlderServer(t *testing.T) { require.Equal(t, allocationFailureExhausted, classifyAllocationFailure(fromTheWire(t, legacy))) } -// Naming the pool that ran out was the point of asking IPAM for it. A message -// that says an address ran out without saying what ran out leaves an operator -// with nothing to widen. -func TestExhaustionMessageNamesThePool(t *testing.T) { - err := fromTheWire(t, ipamerrors.NewPoolExhausted("datum-v4", `IPPool "datum-v4" is exhausted`)) - message := allocationFailureMessage(classifyAllocationFailure(err), v4Request(), err) - - require.Contains(t, message, "datum-v4", "the pool that ran out belongs in the condition") - require.Contains(t, message, "an IPv4 address") - require.Contains(t, message, `IPPool "datum-v4" is exhausted`, "IPAM's message is carried verbatim") -} - -// Exhaustion while a level of the class chain is provisioned names no pool. An -// invented name would send an operator to the wrong place. -func TestExhaustionMessageWithoutAPoolNamesNone(t *testing.T) { - err := fromTheWire(t, ipamerrors.New(ipamerrors.ReasonExhausted, "ipam: pool exhausted")) - message := allocationFailureMessage(classifyAllocationFailure(err), classRequest(), err) - - require.NotContains(t, message, "IPPool") - require.Contains(t, message, `an address of class "public-v4"`) - require.Contains(t, message, "ipam: pool exhausted") -} - -// The refusals that used to be one indistinguishable 400 now say which piece of -// configuration is missing. -func TestRejectionMessagesSayWhichConfigurationIsMissing(t *testing.T) { +// Every branch, rendered whole. A test matching a fragment passes while the +// rest of the message repeats itself, which is how a doubly-named pool shipped. +func TestAllocationFailureMessages(t *testing.T) { for _, tc := range []struct { - name string - err *apierrors.StatusError - contains string + name string + request allocationRequest + err *apierrors.StatusError + want string }{ - {"class not found", ipamerrors.New(ipamerrors.ReasonClassNotFound, "ipam: class not found"), "no address class"}, - {"no default class", ipamerrors.New(ipamerrors.ReasonNoDefaultClass, "ipam: no default class"), "no default address class"}, - {"no offering pool", ipamerrors.New(ipamerrors.ReasonNoOfferingPool, "ipam: no pool offers this class"), "No address pool offers"}, - {"prefix length", ipamerrors.New(ipamerrors.ReasonPrefixLengthRejected, "prefixLength 20 is outside"), "at the requested size"}, - {"no project scope", ipamerrors.New(ipamerrors.ReasonNoProjectScope, "ipam: request carries no project scope"), "without a project"}, + { + name: "exhaustion names the pool once", + request: v4Request(), + err: ipamerrors.NewPoolExhausted("datum-v4", `IPPool "datum-v4" is exhausted`), + want: `No address is left for an IPv4 address: IPPool "datum-v4" is exhausted`, + }, + { + // No pool accounts for this one, so IPAM's account of which level + // of the chain ran out is all there is. + name: "exhaustion with no pool falls back to IPAM", + request: classRequest(), + err: ipamerrors.New(ipamerrors.ReasonExhausted, `provisioning IPPool "datum-v4-us-east": ipam: pool exhausted`), + want: `No address is left for an address of class "public-v4": provisioning IPPool "datum-v4-us-east": ipam: pool exhausted`, + }, + { + name: "retained allocation names the allocation once", + request: v4Request(), + err: ipamerrors.NewRetainedAllocation(ipclaimsResource, "eth0-f-ipv4", "alloc-9f2c", + `an allocation under this identity already exists: IPAllocation "alloc-9f2c", retained by an earlier claim of the same name; delete it to reuse the name`), + want: `Another allocation still holds the name for an IPv4 address: Operation cannot be fulfilled on ipclaims.ipam.miloapis.com "eth0-f-ipv4": an allocation under this identity already exists: IPAllocation "alloc-9f2c", retained by an earlier claim of the same name; delete it to reuse the name`, + }, + { + name: "scope roles are named once, by IPAM", + request: v4Request(), + err: ipamerrors.NewScopeRolesMissing([]string{"network", "location"}, + `scope is missing roles "network", "location" required by uniqueWithin (class "public-v4")`), + want: `IPAM would not allocate an IPv4 address: scope is missing roles "network", "location" required by uniqueWithin (class "public-v4")`, + }, + { + name: "an unconfigured class is named by IPAM, with its project", + request: classRequest(), + err: ipamerrors.New(ipamerrors.ReasonClassNotFound, `ipam: class not found: "public-v4" in project "shared-infra"`), + want: `IPAM would not allocate an address of class "public-v4": ipam: class not found: "public-v4" in project "shared-infra"`, + }, + { + name: "no pool offers the class", + request: classRequest(), + err: ipamerrors.New(ipamerrors.ReasonNoOfferingPool, `ipam: no pool offers this class: class "public-v4" in project "shared-infra"`), + want: `IPAM would not allocate an address of class "public-v4": ipam: no pool offers this class: class "public-v4" in project "shared-infra"`, + }, + { + name: "a failure we cannot classify is still reported", + request: v4Request(), + err: apierrors.NewInternalError(errors.New("postgres is unreachable")), + want: "Allocating an IPv4 address failed: Internal error occurred: postgres is unreachable", + }, } { t.Run(tc.name, func(t *testing.T) { err := fromTheWire(t, tc.err) - message := allocationFailureMessage(classifyAllocationFailure(err), classRequest(), err) - - require.Contains(t, message, tc.contains) - require.Contains(t, message, tc.err.ErrStatus.Message, "IPAM's message is carried verbatim") - require.NotContains(t, message, "usually means", - "the reason is known now, so the message must not hedge") + require.Equal(t, tc.want, + allocationFailureMessage(classifyAllocationFailure(err), tc.request, err)) }) } } -// A claim short two roles is one refusal naming both, so the condition names -// both rather than sending an operator back for the second. -func TestScopeRolesMessageNamesEveryMissingRole(t *testing.T) { - err := fromTheWire(t, ipamerrors.NewScopeRolesMissing( - []string{"network", "location"}, "scope is missing roles")) - message := allocationFailureMessage(classifyAllocationFailure(err), v4Request(), err) - - require.Contains(t, message, `"network"`) - require.Contains(t, message, `"location"`) - require.Contains(t, message, "scope roles") -} - -// The hedge survives for exactly one case: a refusal this build cannot name. -func TestUnnamedRejectionStillHedges(t *testing.T) { - err := fromTheWire(t, apierrors.NewBadRequest("something IPAM has not classified")) - require.Equal(t, allocationFailureUnknown, classifyAllocationFailure(err)) - require.Contains(t, - allocationFailureMessage(allocationFailureRejected, classRequest(), err), - "usually means") -} - -// The two conflicts are one condition reason to our consumers, but the message -// names the allocation that has to go when IPAM says which one it is. -func TestConflictMessageNamesTheRetainedAllocation(t *testing.T) { - retained := fromTheWire(t, ipamerrors.NewRetainedAllocation( - ipclaimsResource, "eth0-f-ipv4", "alloc-9f2c", "retained by an earlier claim")) - message := allocationFailureMessage(classifyAllocationFailure(retained), v4Request(), retained) - require.Contains(t, message, "alloc-9f2c") - require.Contains(t, message, "retained by an earlier claim") - - exists := fromTheWire(t, ipamerrors.NewClaimExists( - ipclaimsResource, "eth0-f-ipv4", "already holds an allocation")) - require.NotContains(t, - allocationFailureMessage(classifyAllocationFailure(exists), v4Request(), exists), - "IPAllocation") +// The message must never carry a fact twice. An operator reading a condition +// should learn something from every clause of it. +func TestMessagesDoNotRepeatThemselves(t *testing.T) { + for _, tc := range []struct { + name string + err *apierrors.StatusError + repeat string + }{ + {"pool", ipamerrors.NewPoolExhausted("datum-v4", `IPPool "datum-v4" is exhausted`), "datum-v4"}, + { + "roles", + ipamerrors.NewScopeRolesMissing([]string{"location"}, `scope is missing role "location" required by uniqueWithin`), + `"location"`, + }, + { + "allocation", + ipamerrors.NewRetainedAllocation(ipclaimsResource, "eth0-f-ipv4", "alloc-9f2c", `IPAllocation "alloc-9f2c" is retained`), + "alloc-9f2c", + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := fromTheWire(t, tc.err) + message := allocationFailureMessage(classifyAllocationFailure(err), v4Request(), err) + require.Contains(t, message, tc.repeat) + require.Equal(t, 1, countOccurrences(message, tc.repeat), + "%q appears more than once in %q", tc.repeat, message) + }) + } } -// Whatever the reason, IPAM's account of the failure reaches the condition. -// Parsing it instead would tie our conditions to its wording. -func TestEveryMessageCarriesIPAMsOwnMessage(t *testing.T) { - err := fromTheWire(t, apierrors.NewInternalError(errors.New("postgres is unreachable"))) - message := allocationFailureMessage(classifyAllocationFailure(err), v4Request(), err) - require.Contains(t, message, "postgres is unreachable") +func countOccurrences(haystack, needle string) int { + count := 0 + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + count++ + } + } + return count } diff --git a/internal/controller/networkinterfaceclaim_controller_test.go b/internal/controller/networkinterfaceclaim_controller_test.go index e175b30a..c2c7250a 100644 --- a/internal/controller/networkinterfaceclaim_controller_test.go +++ b/internal/controller/networkinterfaceclaim_controller_test.go @@ -586,7 +586,7 @@ func TestNetworkInterfaceClaimRollsBackPartialAllocation(t *testing.T) { []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}) s.ipam.refuse("half-eth0-f-ipv4", - ipamerrors.NewPoolExhausted("datum-v4", "IPPool exhausted")) + ipamerrors.NewPoolExhausted("datum-v4", `IPPool "datum-v4" is exhausted`)) claim := s.createClaim("half-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ InterfaceName: "eth0", @@ -608,10 +608,10 @@ func TestNetworkInterfaceClaimRollsBackPartialAllocation(t *testing.T) { condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimAllocated) require.Equal(t, metav1.ConditionFalse, condition.Status) require.Equal(t, string(allocationFailureExhausted), condition.Reason) - require.Contains(t, condition.Message, "IPPool exhausted", - "IPAM's own message is carried through verbatim") - require.Contains(t, condition.Message, "datum-v4", - "the pool that ran out is what an operator has to widen") + require.Equal(t, + `No address is left for an IPv4 address: IPPool "datum-v4" is exhausted`, + condition.Message, + "the condition names the address we wanted and the pool to widen, each once") } func TestNetworkInterfaceRetainRebindsSameAddresses(t *testing.T) { From deb9e3df45734fbc9d6b98cfbf7abf3ab6e72bac Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 14:34:23 -0500 Subject: [PATCH 12/13] refactor: delete the impersonation kubeconfig generator The e2e suites have moved to kubectl's impersonation flags, so nothing references the generated file, the contexts it wrote, or the task that produced it. Deleting it removes a ServiceAccount token from disk and an artefact that had to exist before anything could talk to IPAM. ipam-kubeconfig also wrote a token to a temp file to hand it to kubectl. It is piped now, so a token reaches the cluster and nothing else. That was the point of the exercise beyond the line count, and it was not true until this commit. Narrows ipam-clear-fixtures.sh to the two kinds that measurably leak. A full suite run starting with 15 stale IPPools and 5 IPClasses passes 25/25: seeding re-applies them as a no-op and cascade pools are reused by scope digest rather than duplicated. A run starting with one leftover IPClaim fails two suites, which enumerate the project's claims and assert against the stranger they find. So pools and classes are stable and claims and allocations are not. Key changes: - Delete hack/gen-ipam-impersonation-kubeconfig.sh, the ipam-impersonation-kubeconfig task, IPAM_KUBECONFIG and its cleanup - Pipe the ServiceAccount token into kubectl instead of a temp file - Clear only ipclaims and ipallocations; record in the task description what was measured, so the next person gets an answer not a rediscovery - Point the CI failure dump at the wrapper --- .github/workflows/test-e2e.yml | 4 +- Taskfile.test-infra.yml | 30 ++----- hack/gen-ipam-impersonation-kubeconfig.sh | 99 ----------------------- hack/ipam-clear-fixtures.sh | 67 ++++++--------- 4 files changed, 35 insertions(+), 165 deletions(-) delete mode 100755 hack/gen-ipam-impersonation-kubeconfig.sh diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index cc9bf2cd..c15b306b 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -78,8 +78,8 @@ jobs: KUBECONFIG="${UKC}" kubectl -n cnpg-system logs deploy/cnpg-cloudnative-pg --tail=200 echo "::endgroup::" echo "::group::IPAM fixtures" - KUBECONFIG=${TMPDIR}/.ipam-tenant-impersonation.yaml kubectl --context tenant-project-alpha get ipclasses,ippools -o wide - KUBECONFIG=${TMPDIR}/.ipam-tenant-impersonation.yaml kubectl --context tenant-project-beta get ipclasses,ippools -o wide + IPAM_KUBE_CONTEXT=kind-nso-upstream hack/ipam-tenant-kubectl.sh project-alpha get ipclasses,ippools -o wide + IPAM_KUBE_CONTEXT=kind-nso-upstream hack/ipam-tenant-kubectl.sh project-beta get ipclasses,ippools -o wide echo "::endgroup::" KC=${TMPDIR}/.kind-nso-downstream.yaml bin/kind-v0.32.0 get kubeconfig --name nso-downstream > "${KC}" diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 71215f36..960a13dc 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -79,7 +79,6 @@ vars: DNS_STAGE: '{{.TMP_DIR}}/.dns-bundle' CRANE: '{{.REPO}}/bin/crane' CRANE_VERSION: v0.21.7 - IPAM_KUBECONFIG: '{{.TMP_DIR}}/.ipam-tenant-impersonation.yaml' IPAM_PROJECTS: project-alpha project-beta tasks: @@ -167,9 +166,6 @@ tasks: - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} || true' - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} || true' - rm -f {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml - # The IPAM fixtures and their database die with the upstream cluster; only - # the impersonation kubeconfig outlives it. - - rm -f {{.IPAM_KUBECONFIG}} - rm -rf {{.IPAM_STAGE}} {{.DNS_STAGE}} - echo "✨ done." @@ -575,7 +571,10 @@ tasks: CA=$(kubectl --context {{.UPSTREAM_CTX}} -n kube-system get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 | tr -d '\n') TOKEN=$(kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system create token \ network-services-operator-controller-manager --duration=24h) - cat > {{.TMP_DIR}}/.ipam-cluster-kubeconfig.yaml < [project...] - -set -euo pipefail - -OUT="${1:?usage: gen-ipam-impersonation-kubeconfig.sh [project...]}" -SRC_CTX="${2:?source context required}" -shift 2 -PROJECTS=("$@") -if [ ${#PROJECTS[@]} -eq 0 ]; then - PROJECTS=(project-alpha project-beta) -fi - -TENANT_USER="e2e-tenant-tester" -PARENT_API_GROUP="resourcemanager.miloapis.com" -PARENT_TYPE="Project" - -# The output is assembled by hand rather than round-tripped through -# `kubectl config view`: kubectl v1.36 drops as / as-user-extra when it -# re-serialises a kubeconfig, which would silently remove the impersonation. -base="$(mktemp)" -trap 'rm -f "$base"' EXIT -kubectl --context "$SRC_CTX" config view --minify --flatten >"$base" - -BASE_CLUSTER="$(KUBECONFIG="$base" kubectl config view -o jsonpath="{.contexts[?(@.name=='${SRC_CTX}')].context.cluster}")" -BASE_USER="$(KUBECONFIG="$base" kubectl config view -o jsonpath="{.contexts[?(@.name=='${SRC_CTX}')].context.user}")" - -C_SERVER="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.server}")" -C_CA="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.certificate-authority-data}")" -C_INSECURE="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.insecure-skip-tls-verify}")" - -B_CERT="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.client-certificate-data}")" -B_KEY="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.client-key-data}")" -B_TOKEN="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.token}")" - -emit_base_credentials() { - [ -n "$B_CERT" ] && printf '%s\n' " client-certificate-data: ${B_CERT}" - [ -n "$B_KEY" ] && printf '%s\n' " client-key-data: ${B_KEY}" - [ -n "$B_TOKEN" ] && printf '%s\n' " token: ${B_TOKEN}" - return 0 -} - -{ - printf '%s\n' "apiVersion: v1" - printf '%s\n' "kind: Config" - printf '%s\n' "current-context: tenant-platform" - printf '%s\n' "clusters:" - printf '%s\n' "- name: ${BASE_CLUSTER}" - printf '%s\n' " cluster:" - printf '%s\n' " server: ${C_SERVER}" - [ -n "$C_CA" ] && printf '%s\n' " certificate-authority-data: ${C_CA}" - [ "$C_INSECURE" = "true" ] && printf '%s\n' " insecure-skip-tls-verify: true" - printf '%s\n' "users:" - printf '%s\n' "- name: ${BASE_USER}" - printf '%s\n' " user:" - emit_base_credentials - for proj in "${PROJECTS[@]}"; do - printf '%s\n' "- name: tenant-${proj}-as" - printf '%s\n' " user:" - emit_base_credentials - printf '%s\n' " as: ${TENANT_USER}" - printf '%s\n' " as-user-extra:" - printf '%s\n' " iam.miloapis.com/parent-api-group:" - printf '%s\n' " - ${PARENT_API_GROUP}" - printf '%s\n' " iam.miloapis.com/parent-type:" - printf '%s\n' " - ${PARENT_TYPE}" - printf '%s\n' " iam.miloapis.com/parent-name:" - printf '%s\n' " - ${proj}" - done - printf '%s\n' "contexts:" - for proj in "${PROJECTS[@]}"; do - printf '%s\n' "- name: tenant-${proj}" - printf '%s\n' " context:" - printf '%s\n' " cluster: ${BASE_CLUSTER}" - printf '%s\n' " user: tenant-${proj}-as" - done - printf '%s\n' "- name: tenant-platform" - printf '%s\n' " context:" - printf '%s\n' " cluster: ${BASE_CLUSTER}" - printf '%s\n' " user: ${BASE_USER}" -} >"$OUT" - -echo "wrote ${OUT} (source context ${SRC_CTX})" -echo " contexts: tenant-platform, $(printf 'tenant-%s ' "${PROJECTS[@]}")" diff --git a/hack/ipam-clear-fixtures.sh b/hack/ipam-clear-fixtures.sh index 986002ef..59d20d45 100755 --- a/hack/ipam-clear-fixtures.sh +++ b/hack/ipam-clear-fixtures.sh @@ -1,21 +1,24 @@ #!/usr/bin/env bash -# Delete every IPAM object in the fixture projects. +# Release the addresses a previous run left held in the fixture projects. # -# IPPool and IPClass are cluster-scoped, so chainsaw's namespace teardown never -# reaches them. A suite that dies partway leaves its pools and classes behind, -# and the next run fails on "already exists" for a reason unrelated to what it -# was testing. +# Only IPClaims and IPAllocations. That is measured, not assumed: # -# Order matters: a bound allocation refuses deletion of the pool it came from, -# and a class with pools offering it refuses deletion too. Claims first, -# allocations next, then pools, then classes. +# * A full suite run starting with 15 stale IPPools and 5 IPClasses and no +# leftover claims passes 25/25. Seeding uses `apply`, so re-applying +# identical classes and pools is a no-op, and cascade-provisioned pools are +# keyed by (class, scope digest) and get reused rather than duplicated — +# the pool count converges instead of growing. +# * A full suite run starting with one leftover IPClaim fails, twice over. +# Suites that enumerate the project's claims see the stranger and assert +# against it: "the rebuilt interface publishes 2001:db8:a005::/96 where +# IPAM holds 2001:db8:a006::/96". # -# This script exists to guarantee a clean slate, so it must never report one it -# did not produce. It proves it can reach each tenant before deleting anything, -# and it verifies all four kinds are empty afterwards. +# So pools and classes are stable and need no clearing; claims and allocations +# hold addresses and change what the next run sees. # -# Adapted from github.com/milo-os/ipam test/e2e/lib/clear-tenant-fixtures.sh at -# ref 20865afe018c. +# Order matters. An IPAllocation bound to an IPClaim refuses deletion — +# "IPAllocation is bound to IPClaim ...; delete the claim instead" — so claims +# go first and the allocations they held follow. # # Usage: ipam-clear-fixtures.sh [project...] @@ -29,7 +32,7 @@ if [ ${#PROJECTS[@]} -eq 0 ]; then fi DELETE_TIMEOUT="${IPAM_CLEAR_TIMEOUT:-60s}" -KINDS="ipclaims ipallocations ippools ipclasses" +KINDS="ipclaims ipallocations" here="$(cd "$(dirname "$0")" && pwd)" @@ -40,9 +43,9 @@ k() { } # Positive control. Reading a type that always resolves proves the aggregated -# API is reachable AND that this context's impersonation is accepted. Without -# it, an unreachable cluster or a wrong context name would make every delete -# below a no-op and this script would report a clean tenant it never touched. +# API is reachable AND that this project's impersonation is accepted. Without +# it, an unreachable cluster would make every delete below a no-op and this +# script would report a clean tenant it never touched. verify_reachable() { local proj="$1" out if ! out="$(k "$proj" get ipclasses 2>&1)"; then @@ -52,11 +55,6 @@ verify_reachable() { fi } -# A delete that fails because the object is already gone is success. A pool -# pinned by a child that has not been deleted yet is expected mid-loop and is -# resolved by the next pass — the residue check at the end is what decides -# whether clearing actually worked. Anything else is surfaced rather than -# swallowed. attempt_delete() { local proj="$1" kind="$2" out rc set +e @@ -64,7 +62,7 @@ attempt_delete() { --timeout="$DELETE_TIMEOUT" 2>&1)" rc=$? set -e - if [ "$rc" -ne 0 ] && ! printf '%s' "$out" | grep -qiE 'not found|no matches for|cannot delete IPPool'; then + if [ "$rc" -ne 0 ] && ! printf '%s' "$out" | grep -qiE 'not found|no matches for'; then echo " ⚠️ deleting ${kind} in ${proj}: ${out}" >&2 fi } @@ -78,35 +76,20 @@ residue() { for proj in "${PROJECTS[@]}"; do verify_reachable "$proj" - attempt_delete "$proj" ipclaims - attempt_delete "$proj" ipallocations - - # Pools need more than one pass. A cascade leaves child pools carved out of - # their parent, and the parent refuses to go while a carve is outstanding — - # but `delete --all` works through the list in name order, so a parent is - # usually attempted before the children that pin it. Each pass frees one - # level of the chain; the chain is at most a few deep. - for _ in 1 2 3 4; do - [ -z "$(residue "$proj" ippools)" ] && break - attempt_delete "$proj" ippools + for kind in $KINDS; do + attempt_delete "$proj" "$kind" done - attempt_delete "$proj" ipclasses - - # The authoritative check, over every kind. Pools and classes block the next - # run with "already exists"; a stranded claim or allocation is worse, because - # a bound allocation refuses deletion of the pool it came from and wedges the - # run after that one. left="" for kind in $KINDS; do found="$(residue "$proj" "$kind")" [ -n "$found" ] && left="${left}${kind}: ${found}"$'\n' done if [ -n "$left" ]; then - echo "❌ ${proj} still holds IPAM fixtures after clearing:" >&2 + echo "❌ ${proj} still holds addresses after clearing:" >&2 printf '%s' "$left" >&2 exit 1 fi - echo "cleared IPAM fixtures in ${proj}" + echo "cleared held addresses in ${proj}" done From 1a3810c05976063c4bf7bf1879ac3425f74dc390 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 13 Aug 2026 14:54:35 -0500 Subject: [PATCH 13/13] test: make the reconcile nudge actually change the claim This suite nudges the claim twice, and both nudges wrote the wall clock in seconds. Steps 3 and 4 routinely complete inside the same second, so the second annotate wrote the value already present. An unchanged object raises no watch event, so no reconcile ran, the condition kept the previous step's message, and the assert waited out its full 120s. Measured on the live env: a same-value annotate leaves resourceVersion at 18609, a nanosecond one moves it to 18613. Ten consecutive runs pass after the change, against roughly one failure in six before it. Key changes: - Nudge with nanosecond precision so the write is always a change --- .../chainsaw-test.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml index 588810a5..a4c019ea 100644 --- a/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml +++ b/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml @@ -102,14 +102,19 @@ spec: would stop testing anything the day the controller went quiet. Then the claim is nudged, because nothing watches IPAM: this forces - the reconcile an operator's next edit would have caused anyway. + the reconcile an operator's next edit would have caused anyway. The + nudge value is nanoseconds, not seconds. Both nudges in this suite + can land in the same second, and an annotate that writes the value + already there does not change the object, raises no watch event and + causes no reconcile — leaving the previous step's condition in place + until the next assert times out. script: content: | set -eu kubectl -n ipam-e2e-alpha delete events \ --field-selector involvedObject.name=nic-missingalloc --ignore-not-found kubectl -n ipam-e2e-alpha annotate networkinterfaceclaim nic-missingalloc \ - e2e.datum.net/nudge="$(date +%s)" --overwrite + e2e.datum.net/nudge="$(date +%s%N)" --overwrite - name: The loss is reported as a Warning on the claim description: | @@ -233,7 +238,7 @@ spec: kubectl -n ipam-e2e-alpha delete events \ --field-selector involvedObject.name=nic-missingalloc --ignore-not-found kubectl -n ipam-e2e-alpha annotate networkinterfaceclaim nic-missingalloc \ - e2e.datum.net/nudge="$(date +%s)" --overwrite + e2e.datum.net/nudge="$(date +%s%N)" --overwrite - assert: timeout: 120s resource: