From 3f1e637218330d3243fe259e5ab790c77471ab10 Mon Sep 17 00:00:00 2001 From: Pujol Date: Tue, 16 Jun 2026 23:35:12 +0200 Subject: [PATCH 01/10] feat: add gNMI test server for e2e testing Repackage the existing gNMI server to let us run it as an in-process library for fast envtest mode or as a deployed pod in a Kubernetes cluster for full end-to-end testing. Signed-off-by: Pujol --- Dockerfile | 1 + config/develop/gnmi-test-server.yaml | 2 +- go.mod | 2 + test/gnmi/Dockerfile | 19 +- test/gnmi/main.go | 367 ++-------------- test/gnmi/testserver/server.go | 636 +++++++++++++++++++++++++++ 6 files changed, 679 insertions(+), 348 deletions(-) create mode 100644 test/gnmi/testserver/server.go diff --git a/Dockerfile b/Dockerfile index 8619d4333..ab5dad76b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ WORKDIR /workspace RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=bind,source=go.mod,target=go.mod \ --mount=type=bind,source=go.sum,target=go.sum \ + --mount=type=bind,source=test/gnmi,target=test/gnmi \ go mod download -x RUN --mount=type=bind,target=. \ diff --git a/config/develop/gnmi-test-server.yaml b/config/develop/gnmi-test-server.yaml index 789b76681..9f2d4d1c8 100644 --- a/config/develop/gnmi-test-server.yaml +++ b/config/develop/gnmi-test-server.yaml @@ -33,7 +33,7 @@ spec: containers: - name: gnmi-test-server image: ghcr.io/ironcore-dev/gnmi-test-server:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Never ports: - containerPort: 9339 name: grpc diff --git a/go.mod b/go.mod index 31f543d34..7e41b4f98 100644 --- a/go.mod +++ b/go.mod @@ -129,3 +129,5 @@ require ( sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) + +replace github.com/ironcore-dev/gnmi-test-server => ./test/gnmi diff --git a/test/gnmi/Dockerfile b/test/gnmi/Dockerfile index d80e16ac4..bdc864cc6 100644 --- a/test/gnmi/Dockerfile +++ b/test/gnmi/Dockerfile @@ -12,17 +12,14 @@ ARG TARGETARCH WORKDIR /workspace -# Install dependencies -RUN --mount=type=cache,target=/go/pkg/mod \ - --mount=type=bind,source=go.mod,target=go.mod \ - --mount=type=bind,source=go.sum,target=go.sum \ - go mod download -x +# Copy source files +COPY go.mod go.sum ./ +RUN go mod download -x + +COPY . . # Build the application into a static executable while removing the symbol table and debugging information -RUN --mount=type=bind,target=. \ - --mount=type=cache,target=/go/pkg/mod \ - --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /usr/bin/server ./main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /usr/bin/server ./main.go FROM alpine:${ALPINE_VERSION} @@ -39,5 +36,5 @@ USER 65532:65532 # Switch into workspace WORKDIR / -# Start the server application -CMD ["/server", "--port=9339", "--http-port=8000"] +# Start the server application with NX-OS behavior enabled +CMD ["/server", "--port=9339", "--http-port=8000", "--nxos"] diff --git a/test/gnmi/main.go b/test/gnmi/main.go index ef2018cde..eb5408639 100644 --- a/test/gnmi/main.go +++ b/test/gnmi/main.go @@ -4,362 +4,57 @@ package main import ( - "bytes" "context" - "crypto/tls" - "encoding/json" "flag" - "fmt" - "io" "log" - "net" - "net/http" - "strconv" - "strings" - "sync" - "time" + "os" + "os/signal" + "syscall" - gpb "github.com/openconfig/gnmi/proto/gnmi" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/reflection" - "google.golang.org/grpc/status" - - gtls "github.com/openconfig/gnmi/testing/fake/testing/tls" + "github.com/ironcore-dev/gnmi-test-server/testserver" ) -var _ gpb.GNMIServer = (*Server)(nil) - -// Server implements the GNMI gRPC server -type Server struct { - gpb.UnimplementedGNMIServer - - State *State -} - -func (s *Server) Capabilities(_ context.Context, _ *gpb.CapabilityRequest) (*gpb.CapabilityResponse, error) { - return &gpb.CapabilityResponse{SupportedEncodings: []gpb.Encoding{gpb.Encoding_JSON}}, nil -} - -func (s *Server) Get(_ context.Context, req *gpb.GetRequest) (*gpb.GetResponse, error) { - notifications := make([]*gpb.Notification, 0, len(req.GetPath())) - for _, path := range req.GetPath() { - if len(path.GetElem()) == 0 { - return nil, status.Error(codes.InvalidArgument, "root path is not allowed") - } - log.Printf("Getting path: %v", path) - notifications = append(notifications, &gpb.Notification{ - Timestamp: time.Now().UnixNano(), - Update: []*gpb.Update{ - { - Path: path, - Val: &gpb.TypedValue{ - Value: &gpb.TypedValue_JsonVal{ - JsonVal: s.State.Get(path), - }, - }, - }, - }, - }) - } - return &gpb.GetResponse{ - Notification: notifications, - }, nil -} - -func (s *Server) Set(_ context.Context, req *gpb.SetRequest) (*gpb.SetResponse, error) { - log.Printf("Received Set request: %v", req) - res := make([]*gpb.UpdateResult, 0, len(req.GetDelete())+len(req.GetUpdate())) - for _, del := range req.GetDelete() { - log.Printf("Deleting path: %v", del) - res = append(res, &gpb.UpdateResult{ - Timestamp: time.Now().UnixNano(), - Path: del, - Op: gpb.UpdateResult_DELETE, - }) - s.State.Del(del) - } - for _, replace := range req.GetReplace() { - log.Printf("Replacing path: %v with value: %q", replace.GetPath(), replace.GetVal().GetJsonVal()) - res = append(res, &gpb.UpdateResult{ - Timestamp: time.Now().UnixNano(), - Path: replace.Path, - Op: gpb.UpdateResult_REPLACE, - }) - // Delete the existing value at the path and set the new value. - s.State.Del(replace.GetPath()) - s.State.Set(replace.GetPath(), replace.GetVal().GetJsonVal()) - } - for _, update := range req.GetUpdate() { - log.Printf("Updating path: %v with value: %q", update.GetPath(), update.GetVal().GetJsonVal()) - res = append(res, &gpb.UpdateResult{ - Timestamp: time.Now().UnixNano(), - Path: update.Path, - Op: gpb.UpdateResult_UPDATE, - }) - // The value will automatically be merged into the existing state. - s.State.Set(update.GetPath(), update.GetVal().GetJsonVal()) - } - // TODO: Handle UnionReplace - return &gpb.SetResponse{ - Response: res, - Timestamp: time.Now().UnixNano(), - }, nil -} - -func (s *Server) Subscribe(stream grpc.BidiStreamingServer[gpb.SubscribeRequest, gpb.SubscribeResponse]) error { - req, err := stream.Recv() - switch { - case err == io.EOF: - return nil - case err != nil: - return err - case req.GetSubscribe() == nil: - return status.Errorf(codes.InvalidArgument, "the subscribe request must contain a subscription definition") - } - - switch req.GetRequest().(type) { - case *gpb.SubscribeRequest_Poll: - return status.Errorf(codes.InvalidArgument, "invalid request type: %T", req.GetRequest()) - case *gpb.SubscribeRequest_Subscribe: - } - - switch mode := req.GetSubscribe().GetMode(); mode { - case gpb.SubscriptionList_ONCE: - log.Printf("Received Subscribe request with ONCE mode") - - paths := make([]*gpb.Path, 0, len(req.GetSubscribe().GetSubscription())) - for _, r := range req.GetSubscribe().GetSubscription() { - paths = append(paths, r.GetPath()) - } - - res, err := s.Get(stream.Context(), &gpb.GetRequest{ - Prefix: req.GetSubscribe().GetPrefix(), - Path: paths, - Encoding: req.GetSubscribe().GetEncoding(), - UseModels: req.GetSubscribe().GetUseModels(), - Extension: req.GetExtension(), - }) - if err != nil { - return err - } - - for _, notification := range res.GetNotification() { - if err := stream.Send(&gpb.SubscribeResponse{ - Response: &gpb.SubscribeResponse_Update{ - Update: notification, - }, - }); err != nil { - return status.Errorf(codes.Internal, "failed to send response: %v", err) - } - } - - case gpb.SubscriptionList_STREAM: - return status.Errorf(codes.Unimplemented, "subscribe method Stream not implemented") - case gpb.SubscriptionList_POLL: - return status.Errorf(codes.Unimplemented, "subscribe method Poll not implemented") - default: - return status.Errorf(codes.InvalidArgument, "unknown subscribe request mode: %v", mode) - } - - return nil -} - -// handleState handles HTTP requests to the /v1/state endpoint -func (s *Server) handleState(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodGet: - s.State.RLock() - defer s.State.RUnlock() - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - if len(s.State.Buf) == 0 { - w.Write([]byte("{}")) - return - } - var buf bytes.Buffer - if err := json.Compact(&buf, s.State.Buf); err != nil { - log.Printf("Failed to compact JSON: %v", err) - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("Internal Server Error")) - return - } - w.Write(buf.Bytes()) - case http.MethodDelete: - s.State.Lock() - defer s.State.Unlock() - s.State.Buf = nil - w.WriteHeader(http.StatusNoContent) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } -} - -// State represents a JSON body that can be manipulated using [sjson] syntax. -type State struct { - sync.RWMutex - - Buf []byte -} - -func (s State) Get(path *gpb.Path) []byte { - s.RLock() - defer s.RUnlock() - var sb strings.Builder - for _, elem := range path.GetElem() { - if elem.GetName() == "" { - continue - } - if sb.Len() > 0 { - sb.WriteByte('|') - } - sb.WriteString(elem.GetName()) - if len(elem.GetKey()) == 0 { - continue - } - for k, v := range elem.GetKey() { - sb.WriteByte('|') - sb.WriteString(`#(`) - sb.WriteString(k) - sb.WriteString(`=="`) - sb.WriteString(v) - sb.WriteString(`")#`) - } - } - res := gjson.GetBytes(s.Buf, sb.String()) - if !res.Exists() || (res.IsArray() && len(res.Array()) == 0) { - return []byte("null") - } - return []byte(res.Raw) -} - -func (s *State) Set(path *gpb.Path, raw []byte) { - s.Lock() - defer s.Unlock() - var sb strings.Builder - for _, elem := range path.GetElem() { - if elem.GetName() == "" { - continue - } - if sb.Len() > 0 { - sb.WriteByte('.') - } - sb.WriteString(elem.GetName()) - if len(elem.GetKey()) == 0 { - continue - } - var idx int - gjson.GetBytes(s.Buf, sb.String()).ForEach(func(_, r gjson.Result) bool { - for k, v := range elem.GetKey() { - if r.Get(k).String() != v { - idx++ - return true - } - } - return false - }) - sb.WriteByte('.') - sb.WriteString(strconv.Itoa(idx)) - } - s.Buf, _ = sjson.SetRawBytes(s.Buf, sb.String(), raw) //nolint:errcheck -} - -func (s *State) Del(path *gpb.Path) { - s.Lock() - defer s.Unlock() - var sb strings.Builder - for _, elem := range path.GetElem() { - if elem.GetName() == "" { - continue - } - if sb.Len() > 0 { - sb.WriteByte('.') - } - sb.WriteString(elem.GetName()) - if len(elem.GetKey()) == 0 { - continue - } - var ( - idx int - found bool - ) - gjson.GetBytes(s.Buf, sb.String()).ForEach(func(_, r gjson.Result) bool { - for k, v := range elem.GetKey() { - if r.Get(k).String() != v { - idx++ - return true - } - } - found = true - return false - }) - if !found { - return - } - sb.WriteByte('.') - sb.WriteString(strconv.Itoa(idx)) - } - - s.Buf, _ = sjson.DeleteBytes(s.Buf, sb.String()) //nolint:errcheck -} - func main() { // Parse command line flags port := flag.Int("port", 9339, "The gRPC server port") httpPort := flag.Int("http-port", 8000, "The HTTP server port") + nxos := flag.Bool("nxos", false, "Enable NX-OS behavior (strip DME markers)") flag.Parse() - // Create a listener on the specified port - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) - if err != nil { - log.Fatalf("Failed to listen on port %d: %v", *port, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Build server options + opts := []testserver.ServerOption{ + testserver.WithGRPCPort(*port), + testserver.WithHTTPPort(*httpPort), + testserver.WithBindAddress("0.0.0.0"), + } + if *nxos { + opts = append(opts, testserver.WithNXOSBehavior()) } - // Create a TLS certificate for gRPC server - // This is a self-signed certificate for testing purposes. - cert, err := gtls.NewCert() + // Start the server using the reusable NewTestServer function + // Bind to 0.0.0.0 to accept connections from other pods in the cluster + server, grpcAddr, httpAddr, err := testserver.NewTestServer(ctx, opts...) if err != nil { - log.Fatalf("Failed to create TLS certificate: %v", err) + log.Fatalf("Failed to start server: %v", err) } - // Create a new gRPC server with TLS - grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ - Certificates: []tls.Certificate{cert}, - }))) - - // Create our server implementation - server := &Server{State: &State{}} - - // Register the GNMIService with our server implementation - gpb.RegisterGNMIServer(grpcServer, server) - - // Enable reflection for easier testing with tools like grpcurl - reflection.Register(grpcServer) - - // Setup HTTP server - http.HandleFunc("/v1/state", server.handleState) - httpServer := &http.Server{Addr: fmt.Sprintf(":%d", *httpPort)} - - // Start HTTP server in a goroutine - go func() { - log.Printf("Starting HTTP server on port %d", *httpPort) - log.Printf("HTTP endpoint available at: /v1/state") - if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Fatalf("Failed to serve HTTP server: %v", err) - } - }() - - log.Printf("Starting gRPC server on port %d", *port) - log.Printf("Server is ready to accept connections...") + log.Printf("gRPC server listening on %s", grpcAddr) + log.Printf("HTTP server listening on %s", httpAddr) + log.Printf("HTTP endpoint available at: /v1/state") log.Printf("Use --port flag to specify a different gRPC port (default: 9339)") log.Printf("Use --http-port flag to specify a different HTTP port (default: 8000)") log.Printf("Available services: GNMI") - // Start serving - if err := grpcServer.Serve(lis); err != nil { - log.Fatalf("Failed to serve gRPC server: %v", err) + // Wait for interrupt signal + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + log.Println("Shutting down...") + if err := server.Close(); err != nil { + log.Printf("Error during shutdown: %v", err) } } diff --git a/test/gnmi/testserver/server.go b/test/gnmi/testserver/server.go new file mode 100644 index 000000000..c8a7aa734 --- /dev/null +++ b/test/gnmi/testserver/server.go @@ -0,0 +1,636 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testserver + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" + + gpb "github.com/openconfig/gnmi/proto/gnmi" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/status" + + gtls "github.com/openconfig/gnmi/testing/fake/testing/tls" +) + +var _ gpb.GNMIServer = (*Server)(nil) + +// Server implements the GNMI gRPC server +type Server struct { + gpb.UnimplementedGNMIServer + + State *State + + grpcServer *grpc.Server + httpServer *http.Server + grpcAddr string + httpAddr string +} + +// ServerOption configures the test server +type ServerOption func(*serverConfig) + +type serverConfig struct { + grpcPort int + httpPort int + bindAddress string + stripDMEMarkers bool + dmeMarkerValue string +} + +// WithGRPCPort sets a specific gRPC port (default: 0 for random) +func WithGRPCPort(port int) ServerOption { + return func(c *serverConfig) { + c.grpcPort = port + } +} + +// WithHTTPPort sets a specific HTTP port (default: 0 for random) +func WithHTTPPort(port int) ServerOption { + return func(c *serverConfig) { + c.httpPort = port + } +} + +// WithBindAddress sets the address to bind to (default: 127.0.0.1). +// Use "0.0.0.0" to listen on all interfaces (required for container/pod deployments). +func WithBindAddress(addr string) ServerOption { + return func(c *serverConfig) { + c.bindAddress = addr + } +} + +// WithNXOSBehavior configures the server to emulate NX-OS device behavior: +// - Strips fields with DME_UNSET_PROPERTY_MARKER value when storing (the marker +// means "unset this field", not "store this literal string") +// - Returns empty TypedValue for non-existent paths (instead of NOT_FOUND error) +func WithNXOSBehavior() ServerOption { + return func(c *serverConfig) { + c.stripDMEMarkers = true + c.dmeMarkerValue = "DME_UNSET_PROPERTY_MARKER" + } +} + +// NewTestServer starts an in-process gNMI + HTTP server. +// By default, it uses random available ports. Use WithGRPCPort/WithHTTPPort to specify ports. +// Returns the server, gRPC address, HTTP address, and any error. +func NewTestServer(ctx context.Context, opts ...ServerOption) (*Server, string, string, error) { + cfg := &serverConfig{ + grpcPort: 0, // Random port by default + httpPort: 0, // Random port by default + bindAddress: "127.0.0.1", // Localhost by default (safe for in-process tests) + } + for _, opt := range opts { + opt(cfg) + } + + // Create a listener on the specified port + grpcLis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", cfg.bindAddress, cfg.grpcPort)) + if err != nil { + return nil, "", "", fmt.Errorf("failed to listen for gRPC: %w", err) + } + + httpLis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", cfg.bindAddress, cfg.httpPort)) + if err != nil { + grpcLis.Close() + return nil, "", "", fmt.Errorf("failed to listen for HTTP: %w", err) + } + + // Create a TLS certificate for gRPC server + cert, err := gtls.NewCert() + if err != nil { + grpcLis.Close() + httpLis.Close() + return nil, "", "", fmt.Errorf("failed to create TLS certificate: %w", err) + } + + // Create a new gRPC server with TLS + grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + }))) + + // Create our server implementation + server := &Server{ + State: &State{ + stripDMEMarkers: cfg.stripDMEMarkers, + dmeMarkerValue: cfg.dmeMarkerValue, + }, + grpcServer: grpcServer, + grpcAddr: grpcLis.Addr().String(), + httpAddr: httpLis.Addr().String(), + } + + // Register the GNMIService with our server implementation + gpb.RegisterGNMIServer(grpcServer, server) + + // Enable reflection for easier testing + reflection.Register(grpcServer) + + // Setup HTTP server + mux := http.NewServeMux() + mux.HandleFunc("/v1/state", server.handleState) + mux.HandleFunc("/v1/clear", server.handleClear) + server.httpServer = &http.Server{Handler: mux} + + // Start HTTP server in a goroutine + go func() { + log.Printf("Starting HTTP server on %s", server.httpAddr) + if err := server.httpServer.Serve(httpLis); err != nil && err != http.ErrServerClosed { + log.Printf("HTTP server error: %v", err) + } + }() + + // Start gRPC server in a goroutine + go func() { + log.Printf("Starting gRPC server on %s", server.grpcAddr) + if err := grpcServer.Serve(grpcLis); err != nil { + log.Printf("gRPC server error: %v", err) + } + }() + + return server, server.grpcAddr, server.httpAddr, nil +} + +// GRPCAddr returns the gRPC server address +func (s *Server) GRPCAddr() string { + return s.grpcAddr +} + +// HTTPAddr returns the HTTP server address +func (s *Server) HTTPAddr() string { + return s.httpAddr +} + +// GetState returns the current JSON state +func (s *Server) GetState() ([]byte, error) { + s.State.RLock() + defer s.State.RUnlock() + if len(s.State.Buf) == 0 { + return []byte("{}"), nil + } + var buf bytes.Buffer + if err := json.Compact(&buf, s.State.Buf); err != nil { + return nil, fmt.Errorf("failed to compact JSON: %w", err) + } + return buf.Bytes(), nil +} + +// ClearState clears all accumulated state +func (s *Server) ClearState() { + s.State.Lock() + defer s.State.Unlock() + s.State.Buf = nil +} + +// Close gracefully shuts down the server +func (s *Server) Close() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var errs []error + if s.httpServer != nil { + if err := s.httpServer.Shutdown(ctx); err != nil { + errs = append(errs, fmt.Errorf("HTTP shutdown: %w", err)) + } + } + if s.grpcServer != nil { + s.grpcServer.GracefulStop() + } + if len(errs) > 0 { + return errs[0] + } + return nil +} + +func (s *Server) Capabilities(_ context.Context, _ *gpb.CapabilityRequest) (*gpb.CapabilityResponse, error) { + return &gpb.CapabilityResponse{SupportedEncodings: []gpb.Encoding{gpb.Encoding_JSON}}, nil +} + +func (s *Server) Get(_ context.Context, req *gpb.GetRequest) (*gpb.GetResponse, error) { + notifications := make([]*gpb.Notification, 0, len(req.GetPath())) + for _, path := range req.GetPath() { + if len(path.GetElem()) == 0 { + return nil, status.Error(codes.InvalidArgument, "root path is not allowed") + } + log.Printf("Getting path: %v", path) + notifications = append(notifications, &gpb.Notification{ + Timestamp: time.Now().UnixNano(), + Update: []*gpb.Update{ + { + Path: path, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonVal{ + JsonVal: s.State.Get(path), + }, + }, + }, + }, + }) + } + return &gpb.GetResponse{ + Notification: notifications, + }, nil +} + +func (s *Server) Set(_ context.Context, req *gpb.SetRequest) (*gpb.SetResponse, error) { + log.Printf("Received Set request: %v", req) + res := make([]*gpb.UpdateResult, 0, len(req.GetDelete())+len(req.GetUpdate())) + for _, del := range req.GetDelete() { + log.Printf("Deleting path: %v", del) + res = append(res, &gpb.UpdateResult{ + Timestamp: time.Now().UnixNano(), + Path: del, + Op: gpb.UpdateResult_DELETE, + }) + s.State.Del(del) + } + for _, replace := range req.GetReplace() { + log.Printf("Replacing path: %v with value: %q", replace.GetPath(), replace.GetVal().GetJsonVal()) + res = append(res, &gpb.UpdateResult{ + Timestamp: time.Now().UnixNano(), + Path: replace.Path, + Op: gpb.UpdateResult_REPLACE, + }) + // Delete the existing value at the path and set the new value. + s.State.Del(replace.GetPath()) + s.State.Set(replace.GetPath(), replace.GetVal().GetJsonVal()) + } + for _, update := range req.GetUpdate() { + log.Printf("Updating path: %v with value: %q", update.GetPath(), update.GetVal().GetJsonVal()) + res = append(res, &gpb.UpdateResult{ + Timestamp: time.Now().UnixNano(), + Path: update.Path, + Op: gpb.UpdateResult_UPDATE, + }) + // The value will automatically be merged into the existing state. + s.State.Set(update.GetPath(), update.GetVal().GetJsonVal()) + } + // TODO: Handle UnionReplace + return &gpb.SetResponse{ + Response: res, + Timestamp: time.Now().UnixNano(), + }, nil +} + +func (s *Server) Subscribe(stream grpc.BidiStreamingServer[gpb.SubscribeRequest, gpb.SubscribeResponse]) error { + req, err := stream.Recv() + switch { + case err == io.EOF: + return nil + case err != nil: + return err + case req.GetSubscribe() == nil: + return status.Errorf(codes.InvalidArgument, "the subscribe request must contain a subscription definition") + } + + switch req.GetRequest().(type) { + case *gpb.SubscribeRequest_Poll: + return status.Errorf(codes.InvalidArgument, "invalid request type: %T", req.GetRequest()) + case *gpb.SubscribeRequest_Subscribe: + } + + switch mode := req.GetSubscribe().GetMode(); mode { + case gpb.SubscriptionList_ONCE: + log.Printf("Received Subscribe request with ONCE mode") + + paths := make([]*gpb.Path, 0, len(req.GetSubscribe().GetSubscription())) + for _, r := range req.GetSubscribe().GetSubscription() { + paths = append(paths, r.GetPath()) + } + + res, err := s.Get(stream.Context(), &gpb.GetRequest{ + Prefix: req.GetSubscribe().GetPrefix(), + Path: paths, + Encoding: req.GetSubscribe().GetEncoding(), + UseModels: req.GetSubscribe().GetUseModels(), + Extension: req.GetExtension(), + }) + if err != nil { + return err + } + + for _, notification := range res.GetNotification() { + if err := stream.Send(&gpb.SubscribeResponse{ + Response: &gpb.SubscribeResponse_Update{ + Update: notification, + }, + }); err != nil { + return status.Errorf(codes.Internal, "failed to send response: %v", err) + } + } + + case gpb.SubscriptionList_STREAM: + return status.Errorf(codes.Unimplemented, "subscribe method Stream not implemented") + case gpb.SubscriptionList_POLL: + return status.Errorf(codes.Unimplemented, "subscribe method Poll not implemented") + default: + return status.Errorf(codes.InvalidArgument, "unknown subscribe request mode: %v", mode) + } + + return nil +} + +// handleState handles HTTP requests to the /v1/state endpoint +// GET: returns current state as JSON +// POST: preloads nested JSON into state +// DELETE: clears all state +// Supports X-HTTP-Method-Override header for clients that can't send DELETE. +func (s *Server) handleState(w http.ResponseWriter, r *http.Request) { + method := r.Method + if override := r.Header.Get("X-HTTP-Method-Override"); override != "" { + method = override + } + switch method { + case http.MethodGet: + state, err := s.GetState() + if err != nil { + log.Printf("Failed to get state: %v", err) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Internal Server Error")) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(state) + case http.MethodPost: + body, err := io.ReadAll(r.Body) + if err != nil { + log.Printf("Failed to read body: %v", err) + http.Error(w, "Failed to read body", http.StatusBadRequest) + return + } + if len(body) == 0 { + w.WriteHeader(http.StatusNoContent) + return + } + if !gjson.ValidBytes(body) { + http.Error(w, "invalid JSON", http.StatusBadRequest) + return + } + // Use Set with empty path to merge JSON into root of state + s.State.Set(&gpb.Path{}, body) + log.Printf("Merged state from JSON") + w.WriteHeader(http.StatusNoContent) + case http.MethodDelete: + s.ClearState() + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +// handleClear handles POST /v1/clear to clear all state. +func (s *Server) handleClear(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + s.ClearState() + w.WriteHeader(http.StatusNoContent) +} + +// mergeJSON merges src JSON into dst JSON at the root level. +// Keys in src overwrite keys in dst. +func mergeJSON(dst, src []byte) []byte { + srcParsed := gjson.ParseBytes(src) + if !srcParsed.IsObject() { + return src + } + result := dst + srcParsed.ForEach(func(key, value gjson.Result) bool { + result, _ = sjson.SetRawBytes(result, key.String(), []byte(value.Raw)) + return true + }) + return result +} + +// State represents a JSON body that can be manipulated using [sjson] syntax. +type State struct { + sync.RWMutex + + Buf []byte + + // NX-OS behavior options + stripDMEMarkers bool + dmeMarkerValue string +} + +// stripMarkerFields removes fields with the DME marker value from JSON recursively. +// This emulates NX-OS behavior where these markers mean "unset this field" +// rather than "store this literal string". +func (s *State) stripMarkerFields(data []byte) []byte { + if !s.stripDMEMarkers || s.dmeMarkerValue == "" { + return data + } + return s.stripMarkersRecursive(data) +} + +// stripMarkersRecursive walks the JSON structure and removes marker fields at all levels. +func (s *State) stripMarkersRecursive(data []byte) []byte { + parsed := gjson.ParseBytes(data) + if !parsed.IsObject() && !parsed.IsArray() { + return data + } + + if parsed.IsArray() { + // Process each array element + var results []string + parsed.ForEach(func(_, value gjson.Result) bool { + processed := s.stripMarkersRecursive([]byte(value.Raw)) + results = append(results, string(processed)) + return true + }) + return []byte("[" + strings.Join(results, ",") + "]") + } + + // It's an object - process fields + var toDelete []string + parsed.ForEach(func(key, value gjson.Result) bool { + keyStr := key.String() + if value.Type == gjson.String && value.String() == s.dmeMarkerValue { + toDelete = append(toDelete, keyStr) + } else if value.IsObject() || value.IsArray() { + // Recurse into nested structures + processed := s.stripMarkersRecursive([]byte(value.Raw)) + data, _ = sjson.SetRawBytes(data, keyStr, processed) + } + return true + }) + for _, key := range toDelete { + data, _ = sjson.DeleteBytes(data, key) + } + return data +} + +func (s *State) Get(path *gpb.Path) []byte { + s.RLock() + defer s.RUnlock() + var sb strings.Builder + for _, elem := range path.GetElem() { + if elem.GetName() == "" { + continue + } + if sb.Len() > 0 { + sb.WriteByte('|') + } + sb.WriteString(elem.GetName()) + if len(elem.GetKey()) == 0 { + continue + } + for k, v := range elem.GetKey() { + sb.WriteByte('|') + sb.WriteString(`#(`) + sb.WriteString(k) + sb.WriteString(`=="`) + sb.WriteString(v) + sb.WriteString(`")#`) + } + } + res := gjson.GetBytes(s.Buf, sb.String()) + if !res.Exists() || (res.IsArray() && len(res.Array()) == 0) { + // Return empty bytes for non-existent paths. This triggers gnmiext's + // ErrNil handling (len(b) == 0), matching real NX-OS behavior which + // returns empty TypedValue for paths that don't exist yet. + return []byte{} + } + return []byte(res.Raw) +} + +func (s *State) Set(path *gpb.Path, raw []byte) { + s.Lock() + defer s.Unlock() + + // Strip DME marker fields if NX-OS behavior is enabled + raw = s.stripMarkerFields(raw) + + elems := path.GetElem() + + // Handle empty path - merge raw into state at root level + if len(elems) == 0 { + if len(s.Buf) == 0 { + s.Buf = raw + } else { + s.Buf = mergeJSON(s.Buf, raw) + } + return + } + + var sb strings.Builder + + for i, elem := range elems { + if elem.GetName() == "" { + continue + } + if sb.Len() > 0 { + sb.WriteByte('.') + } + sb.WriteString(elem.GetName()) + + if len(elem.GetKey()) == 0 { + continue + } + + // Find existing array index or append + var idx int + gjson.GetBytes(s.Buf, sb.String()).ForEach(func(_, r gjson.Result) bool { + for k, v := range elem.GetKey() { + if r.Get(k).String() != v { + idx++ + return true + } + } + return false + }) + sb.WriteByte('.') + sb.WriteString(strconv.Itoa(idx)) + + // Inject keys into this list element if it's not the final element + // (for the final element, keys go into raw below) + if i < len(elems)-1 { + currentPath := sb.String() + current := gjson.GetBytes(s.Buf, currentPath) + if !current.Exists() || current.Raw == "null" { + // Create the element with its keys + keyObj := make(map[string]string) + for k, v := range elem.GetKey() { + keyObj[k] = v + } + keyJSON, _ := json.Marshal(keyObj) + s.Buf, _ = sjson.SetRawBytes(s.Buf, currentPath, keyJSON) + } else { + // Element exists, ensure keys are set + for k, v := range elem.GetKey() { + if !gjson.GetBytes(s.Buf, currentPath+"."+k).Exists() { + s.Buf, _ = sjson.SetBytes(s.Buf, currentPath+"."+k, v) + } + } + } + } + } + + // For the final element, inject its keys (from the last keyed element) into raw + lastElem := elems[len(elems)-1] + for k, v := range lastElem.GetKey() { + if !gjson.GetBytes(raw, k).Exists() { + raw, _ = sjson.SetBytes(raw, k, v) + } + } + + s.Buf, _ = sjson.SetRawBytes(s.Buf, sb.String(), raw) //nolint:errcheck +} + +func (s *State) Del(path *gpb.Path) { + s.Lock() + defer s.Unlock() + var sb strings.Builder + for _, elem := range path.GetElem() { + if elem.GetName() == "" { + continue + } + if sb.Len() > 0 { + sb.WriteByte('.') + } + sb.WriteString(elem.GetName()) + if len(elem.GetKey()) == 0 { + continue + } + var ( + idx int + found bool + ) + gjson.GetBytes(s.Buf, sb.String()).ForEach(func(_, r gjson.Result) bool { + for k, v := range elem.GetKey() { + if r.Get(k).String() != v { + idx++ + return true + } + } + found = true + return false + }) + if !found { + return + } + sb.WriteByte('.') + sb.WriteString(strconv.Itoa(idx)) + } + + s.Buf, _ = sjson.DeleteBytes(s.Buf, sb.String()) //nolint:errcheck +} From db4c465c6e18b6800812d484777ab2796888d74d Mon Sep 17 00:00:00 2001 From: Pujol Date: Tue, 16 Jun 2026 23:35:18 +0200 Subject: [PATCH 02/10] feat: add shared utilities for e2e tests Add common test infrastructure that both cluster and envtest modes will use. This includes helper functions for running kubectl commands and comparing JSON outputs, a provider registry that maps test directories to their provider types, resource cleanup logic that respects Kubernetes finalizers, and centralized timeout constants so all tests behave consistently. Signed-off-by: Pujol --- .license-scan-overrides.jsonl | 1 + test/e2e/testutil/doc.go | 16 + test/e2e/testutil/helpers.go | 547 ++++++++++++++++++++++++++++++++++ test/e2e/testutil/provider.go | 156 ++++++++++ test/e2e/testutil/timeouts.go | 17 ++ 5 files changed, 737 insertions(+) create mode 100644 test/e2e/testutil/doc.go create mode 100644 test/e2e/testutil/helpers.go create mode 100644 test/e2e/testutil/provider.go create mode 100644 test/e2e/testutil/timeouts.go diff --git a/.license-scan-overrides.jsonl b/.license-scan-overrides.jsonl index faa62bcb5..f6e9877b5 100644 --- a/.license-scan-overrides.jsonl +++ b/.license-scan-overrides.jsonl @@ -1,6 +1,7 @@ {"name": "github.com/chzyer/logex", "licenceType": "MIT"} {"name": "github.com/grpc-ecosystem/go-grpc-middleware/v2", "licenceType": "Apache-2.0"} {"name": "github.com/hashicorp/vault/api/auth/approle", "licenceType": "MPL-2.0"} +{"name": "github.com/ironcore-dev/gnmi-test-server", "licenceType": "Apache-2.0"} {"name": "github.com/jpillora/longestcommon", "licenceType": "MIT"} {"name": "github.com/logrusorgru/aurora", "licenceType": "Unlicense"} {"name": "github.com/mattn/go-localereader", "licenceType": "MIT"} diff --git a/test/e2e/testutil/doc.go b/test/e2e/testutil/doc.go new file mode 100644 index 000000000..9098b9793 --- /dev/null +++ b/test/e2e/testutil/doc.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +// Package testutil provides test infrastructure for e2e tests. +// +// It supports two test modes selected by build tags: +// +// - Envtest (build tag: envtest): Uses controller-runtime's envtest.Environment +// with an in-process gNMI test server. Fast (~10s) but doesn't test deployment. +// +// - Cluster (default): Uses a real Kubernetes cluster (typically Kind) with a +// deployed operator and gnmi-test-server pod. Slower (~2-5min) but tests full stack. +// +// The concrete types ClusterEnvironment and EnvtestEnvironment provide the same +// methods, allowing test logic to work with either mode via build tag selection. +package testutil diff --git a/test/e2e/testutil/helpers.go b/test/e2e/testutil/helpers.go new file mode 100644 index 000000000..5d38e28fb --- /dev/null +++ b/test/e2e/testutil/helpers.go @@ -0,0 +1,547 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "reflect" + "sort" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +const ( + prometheusURL = "https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.82.2/bundle.yaml" + certmanagerURL = "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml" +) + +// warnError writes a warning to the provided writer. +func warnError(w io.Writer, err error) { + _, _ = fmt.Fprintf(w, "warning: %v\n", err) +} + +// Run executes the provided command within this context. +// It writes the command to the provided writer for logging. +func Run(cmd *exec.Cmd, w io.Writer) (string, error) { + dir, err := GetProjectDir() + if err != nil { + return "", fmt.Errorf("failed to get project directory: %w", err) + } + + cmd.Dir = dir + if err = os.Chdir(cmd.Dir); err != nil { + _, _ = fmt.Fprintf(w, "chdir dir: %s\n", err) + } + + command := strings.Join(cmd.Args, " ") + // #nosec G705 + _, _ = fmt.Fprintf(w, "running: %s\n", command) + + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%s failed with error: (%w) %s", command, err, string(output)) + } + + return string(output), nil +} + +// Apply takes a raw YAML resource and applies it to the cluster by +// creating a temporary file and running 'kubectl apply -f'. +func Apply(ctx context.Context, resource string, w io.Writer) error { + file, err := os.CreateTemp("", "resource-*.yaml") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + // #nosec G703 + defer func() { _ = os.Remove(file.Name()) }() + if _, err = file.WriteString(resource); err != nil { + return fmt.Errorf("failed to write to temp file: %w", err) + } + if err = file.Close(); err != nil { + return fmt.Errorf("failed to close temp file: %w", err) + } + // #nosec G204 G702 + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", file.Name()) + if _, err = Run(cmd, w); err != nil { + return fmt.Errorf("failed to apply resource: %w", err) + } + return nil +} + +// ExtractResourceIdentifier parses YAML and returns "kind/name" for use with kubectl wait. +// The kind is lowercased to match kubectl's resource type format. +func ExtractResourceIdentifier(resourceYAML string) (string, error) { + var obj unstructured.Unstructured + if err := yaml.Unmarshal([]byte(resourceYAML), &obj); err != nil { + return "", fmt.Errorf("failed to unmarshal YAML: %w", err) + } + + kind := strings.ToLower(obj.GetKind()) + name := obj.GetName() + if kind == "" || name == "" { + return "", errors.New("YAML missing kind or metadata.name") + } + + return kind + "/" + name, nil +} + +// CompareJSON compares two JSON strings and returns an error if they are not equal. +// For comparison, it unmarshals both into interface{} and uses reflect.DeepEqual +// after sorting any arrays and removing empty arrays/objects to ignore ordering +// and cleanup artifacts. +func CompareJSON(got, want string) error { + var gotObj, wantObj any + if err := json.Unmarshal([]byte(got), &gotObj); err != nil { + return fmt.Errorf("failed to unmarshal got JSON: %w", err) + } + if err := json.Unmarshal([]byte(want), &wantObj); err != nil { + return fmt.Errorf("failed to unmarshal want JSON: %w", err) + } + + // Normalize both objects (sort arrays, remove empty containers) + gotObj = normalizeJSON(gotObj) + wantObj = normalizeJSON(wantObj) + + if !reflect.DeepEqual(gotObj, wantObj) { + // For error message, show original compacted JSON (not normalized) + // so empty objects show as {} not null + var gotBuf, wantBuf bytes.Buffer + _ = json.Compact(&gotBuf, []byte(got)) //nolint:errcheck // already parsed successfully above + _ = json.Compact(&wantBuf, []byte(want)) //nolint:errcheck // already parsed successfully above + return fmt.Errorf("JSON mismatch:\ngot: %s\nwant: %s", gotBuf.String(), wantBuf.String()) + } + return nil +} + +// normalizeJSON recursively sorts arrays and removes empty arrays/objects +// to make comparison order-independent and ignore cleanup artifacts. +func normalizeJSON(v any) any { + switch val := v.(type) { + case map[string]any: + result := make(map[string]any) + for k, v := range val { + normalized := normalizeJSON(v) + // Skip empty maps and empty arrays + if !isEmpty(normalized) { + result[k] = normalized + } + } + if len(result) == 0 { + return nil + } + return result + case []any: + var result []any + for _, elem := range val { + normalized := normalizeJSON(elem) + if !isEmpty(normalized) { + result = append(result, normalized) + } + } + if len(result) == 0 { + return nil + } + // Sort the array by JSON representation + sort.Slice(result, func(i, j int) bool { + bi, _ := json.Marshal(result[i]) //nolint:errcheck // sorting comparison, errors treated as equal + bj, _ := json.Marshal(result[j]) //nolint:errcheck // sorting comparison, errors treated as equal + return string(bi) < string(bj) + }) + return result + default: + return v + } +} + +// isEmpty checks if a value is an empty map, empty array, or nil. +func isEmpty(v any) bool { + if v == nil { + return true + } + switch val := v.(type) { + case map[string]any: + return len(val) == 0 + case []any: + return len(val) == 0 + } + return false +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator(ctx context.Context, w io.Writer) error { + cmd := exec.CommandContext(ctx, "kubectl", "create", "-f", prometheusURL) + _, err := Run(cmd, w) + return err +} + +// UninstallPrometheusOperator uninstalls the prometheus +func UninstallPrometheusOperator(ctx context.Context, w io.Writer) { + cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", prometheusURL) + if _, err := Run(cmd, w); err != nil { + warnError(w, err) + } +} + +// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed +// by verifying the existence of key CRDs related to Prometheus. +func IsPrometheusCRDsInstalled(ctx context.Context, w io.Writer) bool { + // List of common Prometheus CRDs + prometheusCRDs := []string{ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "prometheusagents.monitoring.coreos.com", + } + + cmd := exec.CommandContext(ctx, "kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") + output, err := Run(cmd, w) + if err != nil { + return false + } + crdList := GetNonEmptyLines(output) + for _, crd := range prometheusCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager(ctx context.Context, w io.Writer) error { + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", certmanagerURL) + if _, err := Run(cmd, w); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + // was re-installed after uninstalling on a cluster. + cmd = exec.CommandContext( + ctx, "kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + if _, err := Run(cmd, w); err != nil { + return err + } + + // Wait for webhook to be fully operational (TLS cert ready) + // The deployment being Available doesn't mean the webhook TLS is ready + cmd = exec.CommandContext( + ctx, "kubectl", "wait", "certificate/cert-manager-webhook-ca", + "--for", "condition=Ready", + "--namespace", "cert-manager", + "--timeout", "2m", + ) + _, _ = Run(cmd, w) //nolint:errcheck // cert may not exist in older versions + + // Give the webhook a moment to pick up the cert + time.Sleep(5 * time.Second) + return nil +} + +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager(ctx context.Context, w io.Writer) { + cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", certmanagerURL) + if _, err := Run(cmd, w); err != nil { + warnError(w, err) + } +} + +// WaitForCertManagerWebhook waits for the cert-manager webhook to be fully operational. +// This should be called before deploying resources that use cert-manager certificates. +func WaitForCertManagerWebhook(ctx context.Context, w io.Writer) error { + // Wait for deployment to be available + cmd := exec.CommandContext( + ctx, "kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "2m", + ) + if _, err := Run(cmd, w); err != nil { + return err + } + + // Wait for the CA injector to inject the CA bundle into the webhook + cmd = exec.CommandContext( + ctx, "kubectl", "wait", "deployment.apps/cert-manager-cainjector", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "2m", + ) + if _, err := Run(cmd, w); err != nil { + return err + } + + // Wait for the cainjector to inject the CA bundle into the webhook configuration + // This is what actually makes the webhook work - the API server needs the CA to verify the webhook's TLS cert + cmd = exec.CommandContext( + ctx, "kubectl", "wait", "validatingwebhookconfiguration/cert-manager-webhook", + "--for", "jsonpath={.webhooks[0].clientConfig.caBundle}", + "--timeout", "2m", + ) + if _, err := Run(cmd, w); err != nil { + return fmt.Errorf("cert-manager webhook CA bundle not injected: %w", err) + } + + return nil +} + +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled(ctx context.Context, w io.Writer) bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.CommandContext(ctx, "kubectl", "get", "crds") + output, err := Run(cmd, w) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(ctx context.Context, name string, w io.Writer) error { + cluster := "kind" + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + // See: https://kind.sigs.k8s.io/docs/user/rootless/#creating-a-kind-cluster-with-rootless-nerdctl + prov, ok := os.LookupEnv("KIND_EXPERIMENTAL_PROVIDER") + if ok && prov != "docker" { + // If kind is configured to not use the docker runtime (e.g. when using podman or nerctl), + // we need to create a temp file to store the image archive and load it as a tarball. + // See: https://github.com/kubernetes-sigs/kind/issues/2760 + file, err := os.CreateTemp("", "operator-image-") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + _ = file.Close() + // #nosec G703 + defer func() { _ = os.Remove(file.Name()) }() + + // https://github.com/containerd/nerdctl/blob/main/docs/command-reference.md#whale-nerdctl-save + // https://docs.podman.io/en/v5.3.0/markdown/podman-save.1.html + // #nosec G702 + cmd := exec.CommandContext(ctx, prov, "save", name, "--output", file.Name()) + if _, err = Run(cmd, w); err != nil { + return fmt.Errorf("failed to save image: %w", err) + } + + cmd = exec.CommandContext(ctx, "kind", "load", "image-archive", file.Name(), "--name", cluster) //nolint:gosec + _, err = Run(cmd, w) + return err + } + cmd := exec.CommandContext(ctx, "kind", "load", "docker-image", name, "--name", cluster) + _, err := Run(cmd, w) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + for element := range strings.SplitSeq(output, "\n") { + if element != "" { + res = append(res, element) + } + } + return res +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, err + } + wd = strings.ReplaceAll(wd, "/test/e2e", "") + return wd, nil +} + +// PatchResourceYAML takes a raw YAML resource and patches its namespace and deviceRef. +// This allows txtar test files to have placeholder values that get replaced at runtime. +// It returns the patched YAML string ready for kubectl apply. +func PatchResourceYAML(resourceYAML, namespace, deviceName string) (string, error) { + // Parse YAML into unstructured map + var obj map[string]any + if err := yaml.Unmarshal([]byte(resourceYAML), &obj); err != nil { + return "", fmt.Errorf("failed to unmarshal YAML: %w", err) + } + + // Patch metadata.namespace + metadata, ok := obj["metadata"].(map[string]any) + if !ok { + metadata = make(map[string]any) + obj["metadata"] = metadata + } + metadata["namespace"] = namespace + + // Ensure labels map exists and add e2e test label + labels, ok := metadata["labels"].(map[string]any) + if !ok { + labels = make(map[string]any) + metadata["labels"] = labels + } + labels[E2ETestLabel] = "" + + // Patch spec.deviceRef.name if it exists + if spec, ok := obj["spec"].(map[string]any); ok { + if deviceRef, ok := spec["deviceRef"].(map[string]any); ok { + deviceRef["name"] = deviceName + } + } + + // Patch the device label if it exists + if _, hasDeviceLabel := labels["networking.metal.ironcore.dev/device"]; hasDeviceLabel { + labels["networking.metal.ironcore.dev/device"] = deviceName + } + + // Marshal back to YAML + out, err := yaml.Marshal(obj) + if err != nil { + return "", fmt.Errorf("failed to marshal patched YAML: %w", err) + } + return string(out), nil +} + +// ApplyWithPatch applies a YAML resource after patching its namespace and deviceRef. +// This is the cluster-mode equivalent of envtest's createResourceFromTxtar. +func ApplyWithPatch(ctx context.Context, resourceYAML, namespace, deviceName string, w io.Writer) error { + patched, err := PatchResourceYAML(resourceYAML, namespace, deviceName) + if err != nil { + return err + } + return Apply(ctx, patched, w) +} + +// WaitForCondition waits for a resource to have a condition set to True. +// It tries "Configured" first, falls back to "Ready" if Configured doesn't exist. +// Skips config-only resources that don't have status conditions. +func WaitForCondition(ctx context.Context, resourceName, namespace string, w io.Writer) error { + // Config-only resources don't have status conditions - skip them + // resourceName format is "kind/name" e.g. "bgpconfig/evpn-settings" + kind := strings.Split(resourceName, "/")[0] + switch strings.ToLower(kind) { + case "interfaceconfig", "lldpconfig", "bgpconfig", "nveconfig", "managementaccessconfig": + return nil // No conditions to wait for + } + + // Try Configured first using jsonpath (more reliable than --for condition=X with multiple conditions) + cmd := exec.CommandContext( + ctx, "kubectl", "wait", resourceName, + "--for", `jsonpath={.status.conditions[?(@.type=="Configured")].status}=True`, + "--namespace", namespace, + "--timeout", "10s", + ) + if _, err := Run(cmd, w); err == nil { + return nil + } + + // Fallback to Ready using jsonpath (condition=Ready doesn't work reliably with custom resources) + cmd = exec.CommandContext( + ctx, "kubectl", "wait", resourceName, + "--for", `jsonpath={.status.conditions[?(@.type=="Ready")].status}=True`, + "--namespace", namespace, + "--timeout", "2m", + ) + _, err := Run(cmd, w) + return err +} + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + content, err := os.ReadFile(filename) + if err != nil { + return err + } + + before, after, ok := bytes.Cut(content, []byte(target)) + if !ok { + if bytes.Contains(content, []byte(target)[len(prefix):]) { + return nil // already uncommented + } + + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + if _, err = out.Write(before); err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return err + } + } + + if _, err = out.Write(after); err != nil { + return err + } + + return os.WriteFile(filename, out.Bytes(), 0o644) +} + +// ExtractConditions extracts status conditions from an unstructured object +// into a typed []metav1.Condition slice for use with apimeta helpers. +func ExtractConditions(obj *unstructured.Unstructured) ([]metav1.Condition, error) { + raw, _, err := unstructured.NestedSlice(obj.Object, "status", "conditions") + if err != nil { + return nil, err + } + data, err := json.Marshal(raw) + if err != nil { + return nil, err + } + var conditions []metav1.Condition + return conditions, json.Unmarshal(data, &conditions) +} diff --git a/test/e2e/testutil/provider.go b/test/e2e/testutil/provider.go new file mode 100644 index 000000000..986747139 --- /dev/null +++ b/test/e2e/testutil/provider.go @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + nxv1alpha1 "github.com/ironcore-dev/network-operator/api/cisco/nx/v1alpha1" + "github.com/ironcore-dev/network-operator/api/core/v1alpha1" + "github.com/ironcore-dev/network-operator/internal/provider" + "github.com/ironcore-dev/network-operator/internal/provider/cisco/iosxr" + "github.com/ironcore-dev/network-operator/internal/provider/cisco/nxos" +) + +// E2ETestLabel is the label key applied to resources created by e2e tests for cleanup tracking. +const E2ETestLabel = "networking.metal.ironcore.dev/e2e-test" + +// ProviderType represents the network device provider to test against. +type ProviderType string + +// ProviderFactory creates a new provider instance. +type ProviderFactory = func() provider.Provider + +// Provider names must match the registered provider names in internal/provider/*/provider.go +const ( + ProviderNXOS ProviderType = "cisco-nxos-gnmi" + ProviderIOSXR ProviderType = "cisco-iosxr-gnmi" +) + +// ProviderConfig holds the configuration for a provider test. +type ProviderConfig struct { + Name ProviderType + NewProvider ProviderFactory +} + +// SupportedProviders lists all providers to test. +var SupportedProviders = []ProviderConfig{ + {Name: ProviderNXOS, NewProvider: func() provider.Provider { return nxos.NewProvider() }}, + {Name: ProviderIOSXR, NewProvider: func() provider.Provider { return iosxr.NewProvider() }}, +} + +// CoreResources are the main API resources with finalizers. +// During cleanup, these are deleted FIRST so their finalizers can +// complete while Device and config resources still exist. +var CoreResources = []schema.GroupVersionKind{ + v1alpha1.GroupVersion.WithKind("Interface"), + v1alpha1.GroupVersion.WithKind("VLAN"), + v1alpha1.GroupVersion.WithKind("VRF"), + v1alpha1.GroupVersion.WithKind("NTP"), + v1alpha1.GroupVersion.WithKind("DNS"), + v1alpha1.GroupVersion.WithKind("LLDP"), + v1alpha1.GroupVersion.WithKind("Banner"), + v1alpha1.GroupVersion.WithKind("OSPF"), + v1alpha1.GroupVersion.WithKind("PIM"), + v1alpha1.GroupVersion.WithKind("NetworkVirtualizationEdge"), + v1alpha1.GroupVersion.WithKind("EVPNInstance"), + v1alpha1.GroupVersion.WithKind("RoutingPolicy"), + v1alpha1.GroupVersion.WithKind("PrefixSet"), + v1alpha1.GroupVersion.WithKind("BGP"), + v1alpha1.GroupVersion.WithKind("BGPPeer"), + v1alpha1.GroupVersion.WithKind("Syslog"), + v1alpha1.GroupVersion.WithKind("SNMP"), + v1alpha1.GroupVersion.WithKind("ManagementAccess"), + v1alpha1.GroupVersion.WithKind("AccessControlList"), + v1alpha1.GroupVersion.WithKind("DHCPRelay"), + v1alpha1.GroupVersion.WithKind("ISIS"), +} + +// ConfigResources are provider-specific config resources (e.g., NX-OS configs). +// During cleanup, these are deleted AFTER core resources. +var ConfigResources = []schema.GroupVersionKind{ + nxv1alpha1.GroupVersion.WithKind("InterfaceConfig"), + nxv1alpha1.GroupVersion.WithKind("LLDPConfig"), + nxv1alpha1.GroupVersion.WithKind("BGPConfig"), + nxv1alpha1.GroupVersion.WithKind("VPCDomain"), +} + +// ResourcePluralName returns the plural resource name for a GVK. +// These must match the CRD spec.names.plural values (from `kubectl api-resources`). +// We can't use meta.UnsafeGuessKindToResource because CRDs define their own plurals +// which don't always follow standard Kubernetes pluralization rules. +func ResourcePluralName(gvk schema.GroupVersionKind) string { + plurals := map[string]string{ + "Interface": "interfaces", + "VLAN": "vlans", + "VRF": "vrfs", + "NTP": "ntp", + "DNS": "dns", + "LLDP": "lldps", + "Banner": "banners", + "OSPF": "ospf", + "PIM": "pim", + "NetworkVirtualizationEdge": "networkvirtualizationedges", + "EVPNInstance": "evpninstances", + "InterfaceConfig": "interfaceconfigs", + "LLDPConfig": "lldpconfigs", + "VPCDomain": "vpcdomains", + "BGPConfig": "bgpconfigs", + "RoutingPolicy": "routingpolicies", + "PrefixSet": "prefixsets", + "BGP": "bgp", + "BGPPeer": "bgppeers", + "Syslog": "syslogs", + "SNMP": "snmp", + "ManagementAccess": "managementaccesses", + "AccessControlList": "accesscontrollists", + "DHCPRelay": "dhcprelays", + "ISIS": "isis", + "Device": "devices", + } + if plural, ok := plurals[gvk.Kind]; ok { + return plural + } + // Fallback to standard pluralization + plural, _ := meta.UnsafeGuessKindToResource(gvk) + return plural.Resource +} + +// CreateTestDevice creates a Device pointing to the gNMI server with a generated name. +func CreateTestDevice(ctx context.Context, c client.Client, gnmiAddr, namespace string) (*v1alpha1.Device, error) { + device := &v1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-device-", + Namespace: namespace, + }, + Spec: v1alpha1.DeviceSpec{ + Endpoint: v1alpha1.Endpoint{ + Address: gnmiAddr, + }, + }, + } + if err := c.Create(ctx, device); err != nil { + return nil, err + } + + // Set the device status to Running so that dependent resources can reconcile + device.Status.Phase = v1alpha1.DevicePhaseRunning + if err := c.Status().Update(ctx, device); err != nil { + return nil, err + } + + return device, nil +} + +// CleanupTimeout is the timeout for cleanup operations. +const CleanupTimeout = 30 * time.Second + +// CleanupInterval is the polling interval for cleanup operations. +const CleanupInterval = 100 * time.Millisecond diff --git a/test/e2e/testutil/timeouts.go b/test/e2e/testutil/timeouts.go new file mode 100644 index 000000000..4c8e8ef31 --- /dev/null +++ b/test/e2e/testutil/timeouts.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import "time" + +const ( + // DefaultTimeout is used for standard resource reconciliation + DefaultTimeout = 30 * time.Second + + // LongTimeout is used for operations that may take longer (deployments, pod starts) + LongTimeout = 2 * time.Minute + + // VeryLongTimeout is used for end-to-end scenarios with multiple dependencies + VeryLongTimeout = 5 * time.Minute +) From 1c42e65e51634417c1f85316946cff857a73e63d Mon Sep 17 00:00:00 2001 From: Pujol Date: Wed, 17 Jun 2026 19:49:04 +0200 Subject: [PATCH 03/10] feat: add cluster-based e2e testing infrastructure Add cluster mode for running gNMI controller tests against a Kind cluster. Tests are isolated via //go:build cluster tag and run with: make test-e2e-cluster PROVIDER=cisco-nxos-gnmi New files: - cluster_suite_test.go: Ginkgo suite with SynchronizedBeforeSuite for parallel execution - cluster_test.go: Manager setup tests and reconciliation tests - testutil/cluster.go: ClusterEnvironment helper for kubectl operations - testdata/cisco-nxos-gnmi/interfaces.txt: Test fixture for Interface reconciliation Makefile additions: - test-e2e-cluster: Run cluster tests with -tags=cluster - test-e2e-envtest: (placeholder) Run envtest tests with -tags=envtest - PROVIDER variable for selecting test provider Signed-off-by: Pujol --- Makefile | 40 +- config/develop/manager_patch.yaml | 1 + test/e2e/cluster_suite_test.go | 99 +++ test/e2e/cluster_test.go | 577 ++++++++++++++++++ test/e2e/e2e_suite_test.go | 2 + test/e2e/e2e_test.go | 2 + .../testdata/cisco-nxos-gnmi/interfaces.txt | 358 +++++++++++ test/e2e/testutil/cluster.go | 339 ++++++++++ test/e2e/util_test.go | 2 + 9 files changed, 1414 insertions(+), 6 deletions(-) create mode 100644 test/e2e/cluster_suite_test.go create mode 100644 test/e2e/cluster_test.go create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/interfaces.txt create mode 100644 test/e2e/testutil/cluster.go diff --git a/Makefile b/Makefile index 1c13ac2b4..f24dc12e1 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,9 @@ $(LOCALBIN): install-gofumpt: FORCE @if ! hash gofumpt 2>/dev/null; then printf "\e[1;36m>> Installing gofumpt...\e[0m\n"; go install mvdan.cc/gofumpt@latest; fi +install-ginkgo: FORCE + @if ! hash ginkgo 2>/dev/null; then printf "\e[1;36m>> Installing ginkgo...\e[0m\n"; go install github.com/onsi/ginkgo/v2/ginkgo@latest; fi + install-kubebuilder: FORCE @set -eou pipefail; if ! hash kubebuilder 2>/dev/null; then printf "\e[1;36m>> Installing kubebuilder...\e[0m\n"; if command -v curl >/dev/null 2>&1; then GET="curl -sLo"; elif command -v wget >/dev/null 2>&1; then GET="wget -O"; else echo "Didn't find curl or wget to download kubebuilder"; exit 2; fi; BIN=$$(go env GOBIN); if [[ -z $$BIN ]]; then BIN=$$(go env GOPATH)/bin; fi; $$GET "$$BIN/kubebuilder" "https://go.kubebuilder.io/dl/latest/$$(go env GOOS)/$$(go env GOARCH)"; chmod +x "$$BIN/kubebuilder"; fi @@ -68,11 +71,15 @@ lint: FORCE bin/golangci-lint-custom ## Run golangci-lint linter @bin/golangci-lint-custom config verify @bin/golangci-lint-custom run +# PROVIDER defines which provider to test (cisco-nxos-gnmi, cisco-iosxr-gnmi, openconfig). +# Used by test-e2e-cluster and test-e2e-envtest to filter tests. +PROVIDER ?= cisco-nxos-gnmi + fmt: FORCE install-gofumpt @printf "\e[1;36m>> gofumpt -l -w .\e[0m\n" @gofumpt -l -w $(shell git ls-files '*.go' | grep -v '^internal/provider/openconfig') -# Run the e2e tests against a k8s cluster. +# Run the scaffolded e2e tests (unchanged from Kubebuilder). test-e2e: FORCE @command -v kind >/dev/null 2>&1 || { \ echo "Kind is not installed. Please install Kind manually."; \ @@ -85,6 +92,26 @@ test-e2e: FORCE @printf "\e[1;36m>> go test ./test/e2e/ -v -ginkgo.v\e[0m\n" @KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v +# Run gNMI controller tests in cluster mode (requires Kind cluster). +# Uses ginkgo for parallel execution. +GINKGO_PROCS ?= 4 +test-e2e-cluster: FORCE install-ginkgo + @command -v kind >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @kind get clusters | grep -q $(KIND_CLUSTER) || { \ + echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ + exit 1; \ + } + @printf "\e[1;36m>> ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=15m -v ./test/e2e/ (PROVIDER=$(PROVIDER))\e[0m\n" + @KIND_CLUSTER=$(KIND_CLUSTER) E2E_PROVIDER=$(PROVIDER) ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=15m -v ./test/e2e/ + +# Run gNMI controller tests in envtest mode (no cluster required). +test-e2e-envtest: FORCE install-setup-envtest + @printf "\e[1;36m>> go test ./test/e2e/ -tags=envtest -v -ginkgo.v (PROVIDER=$(PROVIDER))\e[0m\n" + @KUBEBUILDER_ASSETS=$$(setup-envtest use 1.32 -p path) E2E_PROVIDER=$(PROVIDER) go test ./test/e2e/ -tags=envtest -v -ginkgo.v + docker-build: FORCE @printf "\e[1;36m>> $(CONTAINER_TOOL) build --tag=$(IMG) .\e[0m\n" @$(CONTAINER_TOOL) build --build-arg=BININFO_BUILD_DATE=$(BININFO_BUILD_DATE) --build-arg=BININFO_COMMIT_HASH=$(BININFO_COMMIT_HASH) --build-arg=BININFO_VERSION=$(BININFO_VERSION) --tag=$(IMG) . @@ -98,15 +125,16 @@ build-installer: FORCE generate install-kustomize @printf "\e[1;36m>> kustomize build config/default > dist/install.yaml\e[0m\n" @mkdir -p dist; kustomize build config/default > dist/install.yaml -# Deploy controller to the k8s cluster +# Deploy controller to the k8s cluster. +# Use PROVIDER to set the provider (default: cisco-nxos-gnmi). deploy: FORCE generate install-kustomize - @printf "\e[1;36m>> kustomize build config/default | kubectl apply -f -\e[0m\n" - @kustomize build config/default | kubectl apply -f - + @printf "\e[1;36m>> deploying controller-manager (PROVIDER=$(PROVIDER))\e[0m\n" + @kustomize build config/develop | sed 's/--provider=openconfig/--provider=$(PROVIDER)/' | kubectl apply -f - # Undeploy controller from the k8s cluster undeploy: FORCE install-kustomize - @printf "\e[1;36m>> kustomize build config/default | kubectl delete -f -\e[0m\n" - @kustomize build config/default | kubectl delete --ignore-not-found=true -f - + @printf "\e[1;36m>> undeploying controller-manager\e[0m\n" + @kustomize build config/develop | kubectl delete --ignore-not-found=true -f - # Install CRDs into the k8s cluster deploy-crds: FORCE generate install-kustomize diff --git a/config/develop/manager_patch.yaml b/config/develop/manager_patch.yaml index e07e73a48..162d5e9de 100644 --- a/config/develop/manager_patch.yaml +++ b/config/develop/manager_patch.yaml @@ -1,6 +1,7 @@ - op: replace path: /spec/template/spec/containers/0/args value: + - --metrics-bind-address=:8443 - --leader-elect=false - --health-probe-bind-address=:8081 - --provider=openconfig diff --git a/test/e2e/cluster_suite_test.go b/test/e2e/cluster_suite_test.go new file mode 100644 index 000000000..1782e905e --- /dev/null +++ b/test/e2e/cluster_suite_test.go @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build cluster + +package e2e + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/ironcore-dev/network-operator/test/e2e/testutil" +) + +// TestCluster runs the e2e test suite in cluster mode. +// Named differently from TestE2E to avoid conflict with scaffolded e2e_suite_test.go. +func TestCluster(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting network-operator tests in CLUSTER mode\n") + RunSpecs(t, "e2e suite (cluster)") +} + +// SynchronizedBeforeSuite enables parallel test execution: +// - Process 1: Builds images, installs Prometheus/CertManager, deploys manager (runs first, alone) +// - All processes: Create ClusterEnvironment connection (runs after process 1 completes) +var _ = SynchronizedBeforeSuite( + // First function: runs ONLY on process 1, before other processes start + func(ctx SpecContext) []byte { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + // Note: Timeout is set in the second function for all processes + + By("Ensure that Prometheus is enabled") + cwd, err := testutil.GetProjectDir() + Expect(err).NotTo(HaveOccurred(), "Failed to get project directory") + + err = testutil.UncommentCode(cwd+"/config/default/kustomization.yaml", "#- ../prometheus", "#") + Expect(err).NotTo(HaveOccurred(), "Failed to enable Prometheus") + + // Build and load images to Kind (only process 1) + buildAndLoadImages(ctx) + + // Setup Prometheus and CertManager (only process 1) + setupClusterDependencies(ctx) + + // Deploy controller-manager (includes CRDs via make deploy) + By("deploying controller-manager") + tmpEnv := testutil.NewClusterEnvironment() + Expect(tmpEnv.Setup(ctx)).To(Succeed()) + Expect(tmpEnv.DeployManager(ctx)).To(Succeed()) + + return nil // No data to pass to other processes + }, + // Second function: runs on ALL processes after the first function completes + func(ctx SpecContext, _ []byte) { + SetDefaultEventuallyTimeout(testutil.DefaultTimeout) + SetDefaultEventuallyPollingInterval(time.Second) + + // All processes create their own ClusterEnvironment connection + By("initializing cluster environment") + testEnv = testutil.NewClusterEnvironment() + Expect(testEnv.Setup(ctx)).To(Succeed()) + }, +) + +// SynchronizedAfterSuite enables parallel test cleanup: +// - All processes: Local cleanup (runs on all processes) +// - Process 1: Uninstall shared dependencies (runs last, alone) +var _ = SynchronizedAfterSuite( + // First function: runs on ALL processes + func(ctx SpecContext) { + // Perform local cleanup (will run only once even if called from signal handler) + performCleanup() + + // Wait for all test namespaces to be fully deleted before returning. + // This ensures DeferCleanup hooks have finished deleting resources and their + // finalizers have been processed by the controller. Without this, the second + // function (UndeployManager) may delete the CRDs while resources still exist, + // causing finalizers to be stuck forever. + if testEnv != nil { + _ = testEnv.WaitForTestNamespacesGone(ctx) //nolint:errcheck // best-effort cleanup + } + }, + // Second function: runs ONLY on process 1, after all other processes complete + func(ctx SpecContext) { + // Undeploy the controller-manager + tmpEnv := testutil.NewClusterEnvironment() + _ = tmpEnv.Setup(ctx) //nolint:errcheck // best-effort cleanup + _ = tmpEnv.UndeployManager(ctx) //nolint:errcheck // best-effort cleanup + + // Uninstall Prometheus and CertManager + cleanupClusterDependencies(ctx) + }, +) diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go new file mode 100644 index 000000000..4483974c9 --- /dev/null +++ b/test/e2e/cluster_test.go @@ -0,0 +1,577 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build cluster + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/tools/txtar" + + "github.com/ironcore-dev/network-operator/test/e2e/testutil" +) + +// namespace where the project is deployed in +// tests create resources in separate namespaces +const namespace = "network-operator-system" + +// serviceAccountName created for the project +const serviceAccountName = "network-operator-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "network-operator-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "network-operator-metrics-binding" + +// image is the name of the image which will be build and loaded +// with the code source changes to be tested. +const image = "ghcr.io/ironcore-dev/network-operator:latest" + +// serverImage is the name of the image which will be built and loaded +// with the gNMI test server. +const serverImage = "ghcr.io/ironcore-dev/gnmi-test-server:latest" + +var ( + // Optional Environment Variables: + // - PROMETHEUS_INSTALL_SKIP=true: Skips Prometheus Operator installation during test setup. + // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. + // These variables are useful if Prometheus or CertManager is already installed, avoiding re-installation and conflicts. + skipPrometheusInstall = os.Getenv("PROMETHEUS_INSTALL_SKIP") == "true" + skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" + // isPrometheusOperatorAlreadyInstalled will be set true when prometheus CRDs be found on the cluster + isPrometheusOperatorAlreadyInstalled = false + // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster + isCertManagerAlreadyInstalled = false +) + +var ( + cleanupOnce sync.Once + // testEnv is the cluster test environment. + testEnv *testutil.ClusterEnvironment +) + +func init() { + _, _ = fmt.Fprintf(GinkgoWriter, "Starting network-operator tests in CLUSTER mode\n") +} + +// Manager Setup tests run serially on a single Ginkgo process. +// These tests deploy and verify the controller-manager before reconciliation tests run in parallel. +var _ = Describe("Manager Setup", Serial, Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func(ctx SpecContext) { + By("creating manager namespace") + cmd := exec.CommandContext(ctx, "kubectl", "create", "ns", namespace, "--dry-run=client", "-o", "yaml") + nsYaml, err := testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to generate namespace YAML") + cmd = exec.CommandContext(ctx, "kubectl", "apply", "-f", "-") + cmd.Stdin = bytes.NewBufferString(nsYaml) + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.CommandContext(ctx, "kubectl", "label", "--overwrite", "ns", namespace, "pod-security.kubernetes.io/enforce=restricted") + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.CommandContext(ctx, "make", "deploy-crds") + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.CommandContext(ctx, "make", "deploy") + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + }) + + // After all setup tests complete, clean up the manager. + // Note: CRDs are left installed for the parallel reconciliation tests. + AfterAll(func(ctx SpecContext) { + By("cleaning up the ClusterRoleBinding of the service account to allow access to metrics") + cmd := exec.CommandContext(ctx, "kubectl", "delete", "clusterrolebinding", metricsRoleBindingName, "--ignore-not-found") + _, err := testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to delete ClusterRoleBinding") + + By("cleaning up the curl pod for metrics") + cmd = exec.CommandContext(ctx, "kubectl", "delete", "pod", "curl-metrics", "-n", namespace, "--ignore-not-found") + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to delete curl-metrics pod") + }) + + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func(ctx SpecContext) { + if specReport := CurrentSpecReport(); specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.CommandContext(ctx, "kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := testutil.Run(cmd, GinkgoWriter) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + + By("Fetching Kubernetes events") + cmd = exec.CommandContext(ctx, "kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := testutil.Run(cmd, GinkgoWriter) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } + + By("Fetching curl-metrics logs") + cmd = exec.CommandContext(ctx, "kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := testutil.Run(cmd, GinkgoWriter) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } + + By("Fetching controller manager pod description") + cmd = exec.CommandContext(ctx, "kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := testutil.Run(cmd, GinkgoWriter) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) + + BeforeEach(func() { + SetDefaultEventuallyTimeout(testutil.LongTimeout) + SetDefaultEventuallyPollingInterval(time.Second) + }) + + It("should run successfully", func(ctx SpecContext) { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + cmd := exec.CommandContext( + ctx, "kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + + podOutput, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := testutil.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + controllerPodName = podNames[0] + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) + + // Validate the pod's status + cmd = exec.CommandContext(ctx, "kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace) + output, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func(ctx SpecContext) { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + // #nosec G204 + cmd := exec.CommandContext(ctx, "kubectl", "create", "clusterrolebinding", metricsRoleBindingName, "--clusterrole=network-operator-metrics-reader", fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName)) + _, err := testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.CommandContext(ctx, "kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.CommandContext(ctx, "kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + kcmd := exec.CommandContext(ctx, "kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, kErr := testutil.Run(kcmd, GinkgoWriter) + g.Expect(kErr).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager has started") + verifyManagerStarted := func(g Gomega) { + kcmd := exec.CommandContext(ctx, "kubectl", "logs", controllerPodName, "-n", namespace) + output, kErr := testutil.Run(kcmd, GinkgoWriter) + g.Expect(kErr).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("starting manager"), "Manager not yet started") + } + Eventually(verifyManagerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + // #nosec G204 + cmd = exec.CommandContext(ctx, "kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": ["curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccount": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.CommandContext(ctx, "kubectl", "get", "pods", "curl-metrics", "-o", "jsonpath={.status.phase}", "-n", namespace) + output, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, testutil.VeryLongTimeout).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput(ctx) + Expect(metricsOutput).To(ContainSubstring("controller_runtime_webhook_panics_total")) + }) + + It("should provisioned cert-manager", func(ctx SpecContext) { + By("validating that cert-manager has the certificate Secret") + verifyCertManager := func(g Gomega) { + cmd := exec.CommandContext(ctx, "kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) + _, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred()) + } + Eventually(verifyCertManager).Should(Succeed()) + }) + + It("should have CA injection for validating webhooks", func(ctx SpecContext) { + By("checking CA injection for validating webhooks") + verifyCAInjection := func(g Gomega) { + cmd := exec.CommandContext(ctx, "kubectl", "get", + "validatingwebhookconfigurations.admissionregistration.k8s.io", + "network-operator-validating-webhook-configuration", + "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") + vwhOutput, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(vwhOutput)).To(BeNumerically(">", 10)) + } + Eventually(verifyCAInjection).Should(Succeed()) + }) + + // +kubebuilder:scaffold:e2e-webhooks-checks +}) + +// Reconciliation tests run in parallel across multiple Ginkgo processes. +// Each test creates its own namespace and gnmi-test-server instance for isolation. +var _ = Describe("Reconciliation", func() { + projectDir, err := testutil.GetProjectDir() + if err != nil { + Fail(fmt.Sprintf("Failed to get project directory: %v", err)) + } + + // Get provider filter from environment (set by Makefile) + providerFilter := os.Getenv("E2E_PROVIDER") + + testdataRoot := filepath.Join(projectDir, "test", "e2e", "testdata") + providerDirs, err := os.ReadDir(testdataRoot) + if err != nil { + Fail(fmt.Sprintf("Failed to read testdata directory: %v", err)) + } + + var testFiles []string + var providerName string + for _, providerDir := range providerDirs { + if !providerDir.IsDir() { + continue + } + providerName = providerDir.Name() + + if providerFilter != "" && providerName != providerFilter { + continue + } + + providerTestdataDir := filepath.Join(testdataRoot, providerName) + + testFiles, err = filepath.Glob(filepath.Join(providerTestdataDir, "*.txt")) + if err != nil { + Fail(fmt.Sprintf("Failed to glob testdata: %v", err)) + } + break + } + + for _, testFile := range testFiles { + testName := filepath.Base(testFile) + testName = testName[:len(testName)-4] // remove .txt + + It(fmt.Sprintf("should reconcile %s/%s", providerName, testName), func(ctx SpecContext) { + By("parsing testdata file") + a, err := txtar.ParseFile(testFile) + Expect(err).NotTo(HaveOccurred(), "Failed to parse test file: %s", testFile) + + var state, preload []byte + var resources []txtar.File + for _, f := range a.Files { + switch f.Name { + case "state/expect": + state = f.Data + case "state/preload": + preload = f.Data + default: + resources = append(resources, f) + } + } + Expect(state).NotTo(BeEmpty(), "Expected '-- state/expect --' section in testdata") + Expect(resources).NotTo(BeEmpty(), "Expected at least one resource in testdata") + + By("creating test namespace") + testNamespace := fmt.Sprintf("test-%s-%s-%s", providerName, strings.ReplaceAll(testName, "_", "-"), time.Now().Format("20060102150405")) + // Truncate to 63 chars max (K8s namespace limit) + if len(testNamespace) > 63 { + testNamespace = testNamespace[:63] + } + Expect(testEnv.CreateNamespace(ctx, testNamespace)).NotTo(HaveOccurred(), "Failed to create test namespace") + + DeferCleanup(func(_ SpecContext) { + // Use a fresh context with generous timeout for cleanup + // The SpecContext may be nearly exhausted after test timeout + cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Clean up test resources before deleting the gnmi-test-server pod to avoid issues with finalizers that require API access. + By("deleting test resources") + _ = testEnv.DeleteCustomResources(cleanupCtx, testNamespace) //nolint:errcheck // best-effort cleanup + By("deleting test namespace") + _ = testEnv.DeleteNamespace(cleanupCtx, testNamespace) //nolint:errcheck // best-effort cleanup + }) + + deviceName := fmt.Sprintf("test-device-%d", time.Now().UnixNano()) + + By("deploying a gnmi-test-server instance for this test") + gnmiAddr, err := testEnv.DeployGNMIServer(ctx, testNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy gnmi-test-server") + Expect(gnmiAddr).ToNot(BeNil()) + + By("preloading gNMI state if specified") + if len(preload) > 0 { + err = testEnv.PreloadGNMIState(ctx, testNamespace, preload) + Expect(err).NotTo(HaveOccurred(), "Failed to preload gNMI state") + } + + By("creating a test device") + device := fmt.Sprintf(` +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Device +metadata: + name: %s + namespace: %s + labels: + %s: "" +spec: + endpoint: + address: "%s"`, deviceName, testNamespace, testutil.E2ETestLabel, gnmiAddr.String()) + err = testutil.Apply(ctx, device, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to apply Device") + + By("applying resources from testdata") + _, _ = fmt.Fprintf(GinkgoWriter, "DEBUG: Found %d resources to apply\n", len(resources)) + for _, res := range resources { + _, _ = fmt.Fprintf(GinkgoWriter, "DEBUG: Applying resource: %s\n", res.Name) + err = testutil.ApplyWithPatch(ctx, string(res.Data), testNamespace, deviceName, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to apply resource: %s", res.Name) + } + + By("waiting for resources to be configured") + for _, res := range resources { + // Extract actual kind/name from YAML since section name may differ from metadata.name + resourceID, err := testutil.ExtractResourceIdentifier(string(res.Data)) + Expect(err).NotTo(HaveOccurred(), "Failed to extract resource identifier from: %s", res.Name) + err = testutil.WaitForCondition(ctx, resourceID, testNamespace, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Resource not configured: %s", resourceID) + } + + By("verifying gNMI state matches expected JSON") + gnmiState, err := testEnv.GetGNMIState(ctx, testNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to get gNMI state") + + err = testutil.CompareJSON(string(gnmiState), string(state)) + Expect(err).NotTo(HaveOccurred(), "gNMI state does not match expected JSON") + }) + } +}) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken(ctx context.Context) (string, error) { + // #nosec G101 + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := serviceAccountName + "-token-request" + tokenRequestFile := filepath.Join(os.TempDir(), secretName) + if err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)); err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + // #nosec G204 + cmd := exec.CommandContext(ctx, "kubectl", "create", "--raw", fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s/token", namespace, serviceAccountName), "-f", tokenRequestFile) + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, nil +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput(ctx context.Context) string { + By("getting the curl-metrics logs") + cmd := exec.CommandContext(ctx, "kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} + +// performCleanup ensures testEnv.Teardown is called exactly once +func performCleanup() { + cleanupOnce.Do(func() { + if testEnv != nil { + fmt.Fprintf(os.Stderr, "Tearing down test environment...\n") + ctx, cancel := context.WithTimeout(context.Background(), testutil.DefaultTimeout) + defer cancel() + if err := testEnv.Teardown(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to teardown test environment: %v\n", err) + } + } + }) +} + +// setupClusterDependencies installs Prometheus and CertManager if needed. +// Called by cluster_suite_test.go. +func setupClusterDependencies(ctx SpecContext) { + if !skipPrometheusInstall { + By("checking if prometheus is installed already") + isPrometheusOperatorAlreadyInstalled = testutil.IsPrometheusCRDsInstalled(ctx, GinkgoWriter) + if !isPrometheusOperatorAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing Prometheus Operator...\n") + Expect(testutil.InstallPrometheusOperator(ctx, GinkgoWriter)).To(Succeed(), "Failed to install Prometheus Operator") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: Prometheus Operator is already installed. Skipping installation...\n") + } + } + if !skipCertManagerInstall { + By("checking if cert manager is installed already") + isCertManagerAlreadyInstalled = testutil.IsCertManagerCRDsInstalled(ctx, GinkgoWriter) + if !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") + Expect(testutil.InstallCertManager(ctx, GinkgoWriter)).To(Succeed(), "Failed to install CertManager") + // Fresh install - need to wait for webhook to be ready (can take up to 90s) + By("waiting for cert-manager webhook to be ready (fresh install)") + Expect(testutil.WaitForCertManagerWebhook(ctx, GinkgoWriter)).To(Succeed(), "Cert-manager webhook not ready") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") + // Already installed - webhook should be ready, but verify quickly + } + } +} + +// cleanupClusterDependencies uninstalls Prometheus and CertManager if we installed them. +// Called by cluster_suite_test.go. +func cleanupClusterDependencies(ctx SpecContext) { + if !skipPrometheusInstall && !isPrometheusOperatorAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling Prometheus Operator...\n") + testutil.UninstallPrometheusOperator(ctx, GinkgoWriter) + } + if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") + testutil.UninstallCertManager(ctx, GinkgoWriter) + } +} + +// buildAndLoadImages builds and loads Docker images to Kind. +// Called by cluster_suite_test.go. +func buildAndLoadImages(ctx SpecContext) { + By("building the manager(Operator) image") + cmd := exec.CommandContext(ctx, "make", "docker-build", "IMG="+image) + _, err := testutil.Run(cmd, GinkgoWriter) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") + + By("loading the manager(Operator) image on Kind") + err = testutil.LoadImageToKindClusterWithName(ctx, image, GinkgoWriter) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") + + By("building the gnmi-test-server image") + cmd = exec.CommandContext(ctx, "make", "docker-build-test-gnmi-server", "TEST_SERVER_IMG="+serverImage) + _, err = testutil.Run(cmd, GinkgoWriter) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the gnmi-test-server image") + + By("loading the gnmi-test-server image on Kind") + err = testutil.LoadImageToKindClusterWithName(ctx, serverImage, GinkgoWriter) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the gnmi-test-server image into Kind") +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 3702bb802..f59c18c69 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,3 +1,5 @@ +//go:build !cluster && !envtest + // SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors // SPDX-License-Identifier: Apache-2.0 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e583ec9cb..2dabadeff 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,3 +1,5 @@ +//go:build !cluster && !envtest + // SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors // SPDX-License-Identifier: Apache-2.0 diff --git a/test/e2e/testdata/cisco-nxos-gnmi/interfaces.txt b/test/e2e/testdata/cisco-nxos-gnmi/interfaces.txt new file mode 100644 index 000000000..d2c520759 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/interfaces.txt @@ -0,0 +1,358 @@ +# Integration test for Interface resources +# +# Tests interface types and NX-OS specific InterfaceConfig: +# loopback-vtep -> Loopback with IPv4 address +# uplink-spine1 -> Physical L3 with unnumbered IPv4 + BFD +# edge-port -> Physical L2 with InterfaceConfig (STP edge, BufferBoost disabled) +# host-pc -> Aggregate L2 with vPC, LACP, InterfaceConfig (STP network, LACP options) + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interfaceconfig/edge-port-config -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: InterfaceConfig +metadata: + name: interface-nxconfig-edge + namespace: default +spec: + bufferBoost: + enabled: false + spanningTree: + portType: Edge + bpduGuard: true + +-- interfaceconfig/host-pc-config -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: InterfaceConfig +metadata: + name: interface-nxconfig-po + namespace: default +spec: + lacp: + vpcConvergence: true + suspendIndividual: false + spanningTree: + portType: Network + bpduFilter: true + +-- interface/loopback-vtep -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + labels: + networking.metal.ironcore.dev/device-name: device + name: loopback-vtep + namespace: default +spec: + deviceRef: + name: device + name: lo0 + description: NVE/VTEP Leaf1 + adminState: Up + type: Loopback + ipv4: + addresses: + - 10.0.0.10/32 + +-- interface/uplink-spine1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + labels: + networking.metal.ironcore.dev/device-name: device + name: uplink-spine1 + namespace: default +spec: + deviceRef: + name: device + name: eth1/1 + description: Leaf1 to Spine1 + adminState: Up + type: Physical + mtu: 9216 + ipv4: + unnumbered: + interfaceRef: + name: loopback-vtep + bfd: + enabled: true + desiredMinimumTxInterval: 300ms + requiredMinimumReceive: 300ms + detectionMultiplier: 3 + +-- interface/edge-port -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: edge-port + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/2 + type: Physical + adminState: Up + mtu: 1500 + description: "Edge port with STP and BufferBoost config" + providerConfigRef: + apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 + kind: InterfaceConfig + name: interface-nxconfig-edge + switchport: + mode: Trunk + nativeVlan: 1 + allowedVlans: + - 10 + +-- interface/host-pc -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: host-pc + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: po10 + type: Aggregate + adminState: Up + mtu: 1500 + description: "vPC to Host with STP and LACP config" + providerConfigRef: + apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 + kind: InterfaceConfig + name: interface-nxconfig-po + switchport: + mode: Trunk + nativeVlan: 1 + allowedVlans: + - 10 + aggregation: + controlProtocol: + mode: Active + memberInterfaceRefs: + - name: edge-port + multichassis: + enabled: true + id: 2 + +-- state/expect -- +{ + "System": { + "bfd-items": { + "inst-items": { + "if-items": { + "If-list": [ + { + "adminSt": "enabled", + "id": "eth1/1", + "ifka-items": { + "detectMult": 3, + "minRxIntvl": 300, + "minTxIntvl": 300 + } + } + ] + } + } + }, + "fm-items": { + "bfd-items": { + "adminSt": "enabled" + }, + "lacp-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "ctrl": "port-unreachable,redirect", + "id": "lo0" + }, + { + "ctrl": "port-unreachable", + "id": "eth1/1" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "aggr-items": { + "AggrIf-list": [ + { + "accessVlan": "vlan-1", + "adminSt": "up", + "aggrExtd-items": { + "bufferBoost": "enable" + }, + "descr": "vPC to Host with STP and LACP config", + "id": "po10", + "lacpVpcConvergence": "enable", + "layer": "Layer2", + "medium": "broadcast", + "mode": "trunk", + "mtu": 1500, + "nativeVlan": "vlan-1", + "pcMode": "active", + "rsmbrIfs-items": { + "RsMbrIfs-list": [ + { + "tDn": "/System/intf-items/phys-items/PhysIf-list[id='eth1/2']" + } + ] + }, + "suspIndividual": "disable", + "trunkVlans": "10", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state" + } + ] + }, + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "descr": "NVE/VTEP Leaf1", + "id": "lo0", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + }, + "phys-items": { + "PhysIf-list": [ + { + "FECMode": "auto", + "accessVlan": "unknown", + "adminSt": "up", + "descr": "Leaf1 to Spine1", + "id": "eth1/1", + "layer": "Layer3", + "medium": "p2p", + "mode": "access", + "mtu": 9216, + "nativeVlan": "unknown", + "physExtd-items": { + "bufferBoost": "enable" + }, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state" + }, + { + "accessVlan": "vlan-1", + "adminSt": "up", + "descr": "Edge port with STP and BufferBoost config", + "FECMode": "auto", + "id": "eth1/2", + "layer": "Layer2", + "medium": "broadcast", + "mode": "trunk", + "mtu": 1500, + "nativeVlan": "vlan-1", + "trunkVlans": "10", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state", + "physExtd-items": { + "bufferBoost": "disable" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "addr-items": { + "Addr-list": [ + { + "addr": "10.0.0.10/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + }, + "id": "lo0" + }, + { + "id": "eth1/1", + "unnumbered": "lo0" + } + ] + } + } + ] + } + } + }, + "stp-items": { + "inst-items": { + "if-items": { + "If-list": [ + { + "id": "eth1/2", + "mode": "edge", + "bpdufilter": "default", + "bpduguard": "enable" + }, + { + "id": "po10", + "mode": "network", + "bpdufilter": "enable", + "bpduguard": "default" + } + ] + } + } + }, + "vpc-items": { + "inst-items": { + "dom-items": { + "if-items": { + "If-list": [ + { + "id": 2, + "rsvpcConf-items": { + "tDn": "/System/intf-items/aggr-items/AggrIf-list[id='po10']" + } + } + ] + } + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testutil/cluster.go b/test/e2e/testutil/cluster.go new file mode 100644 index 000000000..f34990ec9 --- /dev/null +++ b/test/e2e/testutil/cluster.go @@ -0,0 +1,339 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "bytes" + "context" + "fmt" + "net/netip" + "os" + "os/exec" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + nxv1alpha1 "github.com/ironcore-dev/network-operator/api/cisco/nx/v1alpha1" + "github.com/ironcore-dev/network-operator/api/core/v1alpha1" +) + +var ( + // gnmiPort is the port on which the gnmi-test-server listens for gNMI requests. + gnmiPort uint16 = 9339 + // serverImage is the container image for the gnmi-test-server. + // This must match the image built by the Makefile. + serverImage = "ghcr.io/ironcore-dev/gnmi-test-server:latest" +) + +// ClusterEnvironment enables end-to-end tests to run against a real Kubernetes cluster (e.g., Kind). +// TODO: use native library instead of kubectl (follow up) +type ClusterEnvironment struct { + restConfig *rest.Config + k8sClient client.Client +} + +// NewClusterEnvironment creates a new cluster-based test environment. +func NewClusterEnvironment() *ClusterEnvironment { + return &ClusterEnvironment{} +} + +// Setup connects to the existing cluster (CRDs should already be installed). +func (c *ClusterEnvironment) Setup(ctx context.Context) error { + // Register schemes + if err := corev1.AddToScheme(scheme.Scheme); err != nil { + return err + } + if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + if err := nxv1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + + // Get REST config from kubeconfig + var err error + c.restConfig = ctrl.GetConfigOrDie() + + c.k8sClient, err = client.New(c.restConfig, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return err + } + + return nil +} + +// InstallCRDs installs CRDs into the cluster. Should only be called once (from process 1). +// Uses server-side apply to handle existing CRDs gracefully. +func (c *ClusterEnvironment) InstallCRDs(ctx context.Context) error { + if err := c.runKubectl(ctx, "apply", "-k", "config/crd", "--server-side", "--force-conflicts"); err != nil { + return fmt.Errorf("failed to install CRDs: %w", err) + } + return nil +} + +// DeployManager deploys the controller-manager and waits for it to be ready. +// Should only be called once (from process 1). +// Respects E2E_PROVIDER env var for provider selection. +func (c *ClusterEnvironment) DeployManager(ctx context.Context) error { + dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback + env := os.Environ() + if provider := os.Getenv("E2E_PROVIDER"); provider != "" { + env = append(env, "PROVIDER="+provider) + } + + // First deploy CRDs explicitly (make deploy also does this, but let's be sure) + cmd := exec.CommandContext(ctx, "make", "deploy-crds") + cmd.Dir = dir + cmd.Env = env + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to deploy CRDs: %s: %w", string(output), err) + } + + // Then deploy the manager + cmd = exec.CommandContext(ctx, "make", "deploy") + cmd.Dir = dir + cmd.Env = env + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to deploy manager: %s: %w", string(output), err) + } + + if err := c.runKubectl(ctx, "wait", "deployment/network-operator-controller-manager", + "-n", "network-operator-system", + "--for", "condition=Available", + "--timeout", "2m"); err != nil { + return fmt.Errorf("manager not ready: %w", err) + } + return nil +} + +// UndeployManager undeploys the controller-manager. +func (c *ClusterEnvironment) UndeployManager(ctx context.Context) error { + cmd := exec.CommandContext(ctx, "make", "undeploy") + dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback + cmd.Dir = dir + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to undeploy manager: %s: %w", string(output), err) + } + return nil +} + +// Teardown cleans up CRDs. +func (c *ClusterEnvironment) Teardown(ctx context.Context) error { + _ = c.runKubectl(ctx, "delete", "-k", "config/crd", "--ignore-not-found") //nolint:errcheck // best-effort cleanup + return nil +} + +// Client returns the Kubernetes client. +func (c *ClusterEnvironment) Client() client.Client { + return c.k8sClient +} + +// RESTConfig returns the REST config. +func (c *ClusterEnvironment) RESTConfig() *rest.Config { + return c.restConfig +} + +// DeployGNMIServer deploys the gnmi-test-server pod and returns its gNMI address. +func (c *ClusterEnvironment) DeployGNMIServer(ctx context.Context, namespace string) (netip.AddrPort, error) { + if err := c.runKubectl( + ctx, + "run", "gnmi-test-server", + "--image", serverImage, + "--image-pull-policy", "Never", + "--namespace", namespace, + "--restart", "Never", + "--port", "8000", + "--port", strconv.FormatUint(uint64(gnmiPort), 10), + ); err != nil { + return netip.AddrPort{}, fmt.Errorf("failed to deploy gnmi-test-server: %w", err) + } + + if err := c.runKubectl( + ctx, + "wait", "pods/gnmi-test-server", + "--for", "condition=Ready", + "--namespace", namespace, + "--timeout", "1m", + ); err != nil { + return netip.AddrPort{}, fmt.Errorf("gnmi-test-server pod not ready: %w", err) + } + + out, err := c.runKubectlOutput( + ctx, + "get", "pod", "gnmi-test-server", + "--output", "jsonpath={.status.podIP}", + "--namespace", namespace, + ) + if err != nil { + return netip.AddrPort{}, fmt.Errorf("failed to get gnmi-test-server IP: %w", err) + } + var s netip.Addr + if s, err = netip.ParseAddr(strings.TrimSpace(out)); err != nil { + return netip.AddrPort{}, fmt.Errorf("invalid IP address from gnmi-test-server pod: %w", err) + } + + return netip.AddrPortFrom(s, gnmiPort), nil +} + +// GetGNMIState fetches state via kubectl exec. +func (c *ClusterEnvironment) GetGNMIState(ctx context.Context, namespace string) ([]byte, error) { + out, err := c.runKubectlOutput( + ctx, + "exec", "gnmi-test-server", + "--namespace", namespace, + "--", + "wget", "-qO-", "http://localhost:8000/v1/state", + ) + if err != nil { + return nil, fmt.Errorf("failed to get gNMI state: %w", err) + } + return []byte(out), nil +} + +// ClearGNMIState clears state. +func (c *ClusterEnvironment) ClearGNMIState(ctx context.Context, namespace string) error { + _, err := c.runKubectlOutput( + ctx, + "exec", "gnmi-test-server", + "--namespace", namespace, + "--", + "wget", "-qO-", "--post-data=", "http://localhost:8000/v1/clear", + ) + return err +} + +// PreloadGNMIState preloads nested JSON into the gnmi-test-server state. +// This allows tests to set up paths like System/procsys-items/bootTime +// before the Device controller reconciles. +func (c *ClusterEnvironment) PreloadGNMIState(ctx context.Context, namespace string, jsonData []byte) error { + _, err := c.runKubectlOutput( + ctx, + "exec", "gnmi-test-server", + "--namespace", namespace, + "--", + "wget", "-qO-", "--post-data="+string(jsonData), "http://localhost:8000/v1/state", + ) + return err +} + +// runKubectl runs a kubectl command and returns an error if it fails. +func (c *ClusterEnvironment) runKubectl(ctx context.Context, args ...string) error { + cmd := exec.CommandContext(ctx, "kubectl", args...) + dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback + cmd.Dir = dir + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %w", string(output), err) + } + return nil +} + +// runKubectlOutput runs a kubectl command and returns its output. +func (c *ClusterEnvironment) runKubectlOutput(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "kubectl", args...) + dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback + cmd.Dir = dir + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("%s: %w", stderr.String(), err) + } + return stdout.String(), nil +} + +// DeleteNamespace deletes the given namespace. +func (c *ClusterEnvironment) DeleteNamespace(ctx context.Context, namespace string) error { + if err := c.runKubectl(ctx, "delete", "namespace", namespace, "--ignore-not-found"); err != nil { + return fmt.Errorf("failed to delete namespace %s: %w", namespace, err) + } + return nil +} + +// CreateNamespace creates a new namespace with the given name and labels it for cleanup tracking. +func (c *ClusterEnvironment) CreateNamespace(ctx context.Context, namespace string) error { + if err := c.runKubectl(ctx, "create", "namespace", namespace); err != nil { + return fmt.Errorf("failed to create namespace %s: %w", namespace, err) + } + // Label the namespace for cleanup tracking across parallel test processes + if err := c.runKubectl(ctx, "label", "namespace", namespace, E2ETestLabel+"="); err != nil { + return fmt.Errorf("failed to label namespace %s: %w", namespace, err) + } + return nil +} + +// WaitForTestNamespacesGone waits for all labeled test namespaces to be fully deleted. +// This ensures DeferCleanup hooks have completed before the manager is undeployed. +// Without this, the manager (and CRDs) may be deleted while resources with finalizers still exist, +// leaving them stuck forever because the controller can no longer process the finalizers. +func (c *ClusterEnvironment) WaitForTestNamespacesGone(ctx context.Context) error { + // Get all namespaces with our e2e test label + out, err := c.runKubectlOutput( + ctx, + "get", "namespaces", + "-l", E2ETestLabel, + "-o", "jsonpath={.items[*].metadata.name}", + ) + if err != nil { + return fmt.Errorf("failed to list test namespaces: %w", err) + } + + namespaces := strings.Fields(out) + if len(namespaces) == 0 { + return nil + } + + // Wait for each test namespace to be deleted (with timeout) + for _, ns := range namespaces { + _ = c.runKubectl(ctx, "wait", "namespace", ns, //nolint:errcheck // best-effort cleanup + "--for=delete", + "--timeout=120s") + } + + return nil +} + +// DeleteCustomResources deletes all custom resources in the given namespace. +// Deletion order: CoreResources first (have finalizers), then ConfigResources, then Device. +// This allows finalizers to complete while their dependencies still exist. +func (c *ClusterEnvironment) DeleteCustomResources(ctx context.Context, namespace string) error { + deleteResources := func(gvks []schema.GroupVersionKind) { + for _, gvk := range gvks { + _ = c.runKubectl(ctx, "delete", //nolint:errcheck // best-effort cleanup + ResourcePluralName(gvk), "--all", + "--namespace", namespace, + "--ignore-not-found") + } + // Wait for resources to be fully gone (finalizers completed) + for _, gvk := range gvks { + _ = c.runKubectl(ctx, "wait", //nolint:errcheck // best-effort cleanup + ResourcePluralName(gvk), + "--for=delete", "--all", + "--namespace", namespace, + "--timeout=60s") + } + } + + // Delete core resources first (have finalizers that need Device + configs) + deleteResources(CoreResources) + // Then delete config resources + deleteResources(ConfigResources) + + // Delete Device LAST - after all other resources and their finalizers are done + _ = c.runKubectl(ctx, "delete", "devices", "--all", //nolint:errcheck // best-effort cleanup + "--namespace", namespace, + "--ignore-not-found") + _ = c.runKubectl(ctx, "wait", "devices", //nolint:errcheck // best-effort cleanup + "--for=delete", "--all", + "--namespace", namespace, + "--timeout=60s") + + return nil +} diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go index 1281d18db..1492aa773 100644 --- a/test/e2e/util_test.go +++ b/test/e2e/util_test.go @@ -1,3 +1,5 @@ +//go:build !cluster && !envtest + // SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors // SPDX-License-Identifier: Apache-2.0 From e0403cc45d78ecd9d804a4a3eacfa01f66948944 Mon Sep 17 00:00:00 2001 From: Pujol Date: Thu, 18 Jun 2026 10:03:45 +0200 Subject: [PATCH 04/10] build: add test-e2e-cluster target and update deploy to use config/develop - Add install-ginkgo target for ginkgo CLI - Add PROVIDER variable for provider selection - Add GINKGO_PROCS variable for parallel test execution - Add test-e2e-cluster target for cluster-based e2e tests - Update deploy/undeploy to use config/develop instead of config/default Signed-off-by: Pujol --- Makefile | 17 ++++++----------- Makefile.maker.yaml | 35 +++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index f24dc12e1..73ed4b966 100644 --- a/Makefile +++ b/Makefile @@ -71,14 +71,14 @@ lint: FORCE bin/golangci-lint-custom ## Run golangci-lint linter @bin/golangci-lint-custom config verify @bin/golangci-lint-custom run -# PROVIDER defines which provider to test (cisco-nxos-gnmi, cisco-iosxr-gnmi, openconfig). -# Used by test-e2e-cluster and test-e2e-envtest to filter tests. -PROVIDER ?= cisco-nxos-gnmi - fmt: FORCE install-gofumpt @printf "\e[1;36m>> gofumpt -l -w .\e[0m\n" @gofumpt -l -w $(shell git ls-files '*.go' | grep -v '^internal/provider/openconfig') +# PROVIDER defines which provider to test (cisco-nxos-gnmi, cisco-iosxr-gnmi, openconfig). +# Used by test-e2e-cluster and test-e2e-envtest to filter tests. +PROVIDER ?= cisco-nxos-gnmi + # Run the scaffolded e2e tests (unchanged from Kubebuilder). test-e2e: FORCE @command -v kind >/dev/null 2>&1 || { \ @@ -104,13 +104,8 @@ test-e2e-cluster: FORCE install-ginkgo echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ exit 1; \ } - @printf "\e[1;36m>> ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=15m -v ./test/e2e/ (PROVIDER=$(PROVIDER))\e[0m\n" - @KIND_CLUSTER=$(KIND_CLUSTER) E2E_PROVIDER=$(PROVIDER) ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=15m -v ./test/e2e/ - -# Run gNMI controller tests in envtest mode (no cluster required). -test-e2e-envtest: FORCE install-setup-envtest - @printf "\e[1;36m>> go test ./test/e2e/ -tags=envtest -v -ginkgo.v (PROVIDER=$(PROVIDER))\e[0m\n" - @KUBEBUILDER_ASSETS=$$(setup-envtest use 1.32 -p path) E2E_PROVIDER=$(PROVIDER) go test ./test/e2e/ -tags=envtest -v -ginkgo.v + @printf "\e[1;36m>> ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ (PROVIDER=$(PROVIDER))\e[0m\n" + @KIND_CLUSTER=$(KIND_CLUSTER) E2E_PROVIDER=$(PROVIDER) ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ docker-build: FORCE @printf "\e[1;36m>> $(CONTAINER_TOOL) build --tag=$(IMG) .\e[0m\n" diff --git a/Makefile.maker.yaml b/Makefile.maker.yaml index 53143db03..fe8639dd5 100644 --- a/Makefile.maker.yaml +++ b/Makefile.maker.yaml @@ -92,6 +92,9 @@ verbatim: | install-gofumpt: FORCE @if ! hash gofumpt 2>/dev/null; then printf "\e[1;36m>> Installing gofumpt...\e[0m\n"; go install mvdan.cc/gofumpt@latest; fi + install-ginkgo: FORCE + @if ! hash ginkgo 2>/dev/null; then printf "\e[1;36m>> Installing ginkgo...\e[0m\n"; go install github.com/onsi/ginkgo/v2/ginkgo@latest; fi + install-kubebuilder: FORCE @set -eou pipefail; if ! hash kubebuilder 2>/dev/null; then printf "\e[1;36m>> Installing kubebuilder...\e[0m\n"; if command -v curl >/dev/null 2>&1; then GET="curl -sLo"; elif command -v wget >/dev/null 2>&1; then GET="wget -O"; else echo "Didn't find curl or wget to download kubebuilder"; exit 2; fi; BIN=$$(go env GOBIN); if [[ -z $$BIN ]]; then BIN=$$(go env GOPATH)/bin; fi; $$GET "$$BIN/kubebuilder" "https://go.kubebuilder.io/dl/latest/$$(go env GOOS)/$$(go env GOARCH)"; chmod +x "$$BIN/kubebuilder"; fi @@ -114,7 +117,11 @@ verbatim: | @printf "\e[1;36m>> gofumpt -l -w .\e[0m\n" @gofumpt -l -w $(shell git ls-files '*.go' | grep -v '^internal/provider/openconfig') - # Run the e2e tests against a k8s cluster. + # PROVIDER defines which provider to test (cisco-nxos-gnmi, cisco-iosxr-gnmi, openconfig). + # Used by test-e2e-cluster and test-e2e-envtest to filter tests. + PROVIDER ?= cisco-nxos-gnmi + + # Run the scaffolded e2e tests (unchanged from Kubebuilder). test-e2e: FORCE @command -v kind >/dev/null 2>&1 || { \ echo "Kind is not installed. Please install Kind manually."; \ @@ -127,6 +134,21 @@ verbatim: | @printf "\e[1;36m>> go test ./test/e2e/ -v -ginkgo.v\e[0m\n" @KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v + # Run gNMI controller tests in cluster mode (requires Kind cluster). + # Uses ginkgo for parallel execution. + GINKGO_PROCS ?= 4 + test-e2e-cluster: FORCE install-ginkgo + @command -v kind >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @kind get clusters | grep -q $(KIND_CLUSTER) || { \ + echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ + exit 1; \ + } + @printf "\e[1;36m>> ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ (PROVIDER=$(PROVIDER))\e[0m\n" + @KIND_CLUSTER=$(KIND_CLUSTER) E2E_PROVIDER=$(PROVIDER) ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ + docker-build: FORCE @printf "\e[1;36m>> $(CONTAINER_TOOL) build --tag=$(IMG) .\e[0m\n" @$(CONTAINER_TOOL) build --build-arg=BININFO_BUILD_DATE=$(BININFO_BUILD_DATE) --build-arg=BININFO_COMMIT_HASH=$(BININFO_COMMIT_HASH) --build-arg=BININFO_VERSION=$(BININFO_VERSION) --tag=$(IMG) . @@ -140,15 +162,16 @@ verbatim: | @printf "\e[1;36m>> kustomize build config/default > dist/install.yaml\e[0m\n" @mkdir -p dist; kustomize build config/default > dist/install.yaml - # Deploy controller to the k8s cluster + # Deploy controller to the k8s cluster. + # Use PROVIDER to set the provider (default: cisco-nxos-gnmi). deploy: FORCE generate install-kustomize - @printf "\e[1;36m>> kustomize build config/default | kubectl apply -f -\e[0m\n" - @kustomize build config/default | kubectl apply -f - + @printf "\e[1;36m>> deploying controller-manager (PROVIDER=$(PROVIDER))\e[0m\n" + @kustomize build config/develop | sed 's/--provider=openconfig/--provider=$(PROVIDER)/' | kubectl apply -f - # Undeploy controller from the k8s cluster undeploy: FORCE install-kustomize - @printf "\e[1;36m>> kustomize build config/default | kubectl delete -f -\e[0m\n" - @kustomize build config/default | kubectl delete --ignore-not-found=true -f - + @printf "\e[1;36m>> undeploying controller-manager\e[0m\n" + @kustomize build config/develop | kubectl delete --ignore-not-found=true -f - # Install CRDs into the k8s cluster deploy-crds: FORCE generate install-kustomize From f1f1f6e823fd3afe8bf154744b5b4c400c071788 Mon Sep 17 00:00:00 2001 From: Pujol Date: Wed, 17 Jun 2026 19:53:58 +0200 Subject: [PATCH 05/10] feat: add envtest mode for fast in-process e2e testing Add envtest mode for fast in-process e2e testing. This alternative testing mode runs the same test cases as cluster mode but executes them in-process using controller-runtime's envtest framework. Run with: make test-e2e-envtest PROVIDER=cisco-nxos-gnmi New files: - envtest_suite_test.go: Ginkgo suite with //go:build envtest tag - envtest_test.go: Provider-based test runner - testutil/envtest.go: EnvtestEnvironment with in-process gNMI server Signed-off-by: Pujol --- go.mod | 2 + go.sum | 2 + test/e2e/envtest_suite_test.go | 42 +++ test/e2e/envtest_test.go | 634 +++++++++++++++++++++++++++++++++ test/e2e/testutil/envtest.go | 186 ++++++++++ 5 files changed, 866 insertions(+) create mode 100644 test/e2e/envtest_suite_test.go create mode 100644 test/e2e/envtest_test.go create mode 100644 test/e2e/testutil/envtest.go diff --git a/go.mod b/go.mod index 7e41b4f98..22aa5b0a9 100644 --- a/go.mod +++ b/go.mod @@ -72,6 +72,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/ironcore-dev/gnmi-test-server v0.0.0-00010101000000-000000000000 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -90,6 +91,7 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect diff --git a/go.sum b/go.sum index 4ae727d67..cf17074ff 100644 --- a/go.sum +++ b/go.sum @@ -191,10 +191,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= diff --git a/test/e2e/envtest_suite_test.go b/test/e2e/envtest_suite_test.go new file mode 100644 index 000000000..c8cfa522c --- /dev/null +++ b/test/e2e/envtest_suite_test.go @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build envtest + +package e2e + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/ironcore-dev/network-operator/test/e2e/testutil" +) + +// TestE2E runs the e2e test suite in envtest mode. +func TestEnvtest(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting network-operator tests in ENVTEST mode\n") + RunSpecs(t, "e2e suite (envtest)") +} + +// BeforeSuite initializes the envtest environment. +// Envtest runs in-process, so no special parallel handling is needed. +var _ = BeforeSuite(func(ctx SpecContext) { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + SetDefaultEventuallyTimeout(testutil.DefaultTimeout) + SetDefaultEventuallyPollingInterval(time.Second) + + initTestEnv(ctx) +}) + +// AfterSuite cleans up the envtest environment. +var _ = AfterSuite(func(ctx SpecContext) { + fmt.Fprintf(GinkgoWriter, "Tearing down test environment...\n") + cleanupTestEnv(ctx) +}) diff --git a/test/e2e/envtest_test.go b/test/e2e/envtest_test.go new file mode 100644 index 000000000..852089b6e --- /dev/null +++ b/test/e2e/envtest_test.go @@ -0,0 +1,634 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build envtest + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/tools/txtar" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + "github.com/ironcore-dev/network-operator/api/core/v1alpha1" + nx "github.com/ironcore-dev/network-operator/internal/controller/cisco/nx" + "github.com/ironcore-dev/network-operator/internal/controller/core" + "github.com/ironcore-dev/network-operator/internal/resourcelock" + "github.com/ironcore-dev/network-operator/test/e2e/testutil" +) + +// reconcileTestNamespacePrefix is used with GenerateName to create unique test namespaces. +// This isolates tests from other resources in the cluster (e.g., from the deployed operator). +const reconcileTestNamespacePrefix = "reconcile-test-" + +// testEnv is the envtest test environment. +var testEnv *testutil.EnvtestEnvironment + +func init() { + _, _ = fmt.Fprintf(GinkgoWriter, "Starting network-operator tests in ENVTEST mode\n") +} + +// initTestEnv initializes the envtest environment. +func initTestEnv(ctx SpecContext) { + By("initializing envtest environment") + testEnv = testutil.NewEnvtestEnvironment() + Expect(testEnv.Setup(ctx)).To(Succeed()) +} + +// cleanupTestEnv performs envtest-specific cleanup (none needed). +func cleanupTestEnv(_ SpecContext) { + // No additional cleanup needed for envtest +} + +// ============================================================================ +// Provider test helpers +// ============================================================================ + +// ProviderTestContext holds the context for a provider-specific test run. +type ProviderTestContext struct { + Provider testutil.ProviderType + Manager ctrl.Manager + Locker *resourcelock.ResourceLocker + Namespace string + CancelFunc context.CancelFunc +} + +// SetupProviderTest creates a new manager with controllers for the given provider. +// The manager only watches the specified namespace to avoid conflicts with other controllers. +// Call the returned cleanup function in AfterEach/AfterAll. +func SetupProviderTest(providerCfg testutil.ProviderConfig, k8sClient client.Client, restCfg *rest.Config, namespace string) *ProviderTestContext { + GinkgoHelper() + + providerCtx, providerCancel := context.WithCancel(context.Background()) //nolint:gosec // cancel stored in ProviderTestContext + + mgr, err := ctrl.NewManager(restCfg, ctrl.Options{ + Scheme: k8sClient.Scheme(), + Logger: GinkgoLogr, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + namespace: {}, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + // Ignore events during tests + recorder := events.NewFakeRecorder(0) + go func() { + for range recorder.Events { //nolint:revive // intentionally drain events + } + }() + + locker, err := resourcelock.NewResourceLocker(mgr.GetClient(), namespace, 15*time.Second, 10*time.Second) + Expect(err).NotTo(HaveOccurred()) + + err = mgr.Add(locker) + Expect(err).NotTo(HaveOccurred()) + + providerFunc := providerCfg.NewProvider + + // Register all controllers + registerControllers(providerCtx, mgr, recorder, providerFunc, locker) + + go func() { + defer GinkgoRecover() + err = mgr.Start(providerCtx) + if providerCtx.Err() == nil { + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + } + }() + + return &ProviderTestContext{ + Provider: providerCfg.Name, + Manager: mgr, + Locker: locker, + Namespace: namespace, + CancelFunc: providerCancel, + } +} + +// TeardownProviderTest stops the manager for a provider test. +func TeardownProviderTest(ptc *ProviderTestContext) { + if ptc != nil && ptc.CancelFunc != nil { + ptc.CancelFunc() + } +} + +// registerControllers registers all controllers with the manager. +func registerControllers(ctx context.Context, mgr ctrl.Manager, recorder *events.FakeRecorder, providerFunc testutil.ProviderFactory, locker *resourcelock.ResourceLocker) { + var err error + + err = (&core.PrefixSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.RoutingPolicyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.InterfaceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.VLANReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.VRFReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.NTPReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.DNSReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.LLDPReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.BannerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.OSPFReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.PIMReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.NetworkVirtualizationEdgeReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.EVPNInstanceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + // NX-OS specific controllers + err = (&nx.VPCDomainReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.BGPReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.BGPPeerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.SyslogReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.SNMPReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.ManagementAccessReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.AccessControlListReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.DHCPRelayReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.ISISReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) +} + +// CreateTestDevice creates a Device pointing to the gNMI server with a generated name. +func CreateTestDevice(ctx context.Context, c client.Client, gnmiAddr, namespace string) (*v1alpha1.Device, error) { + device := &v1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-device-", + Namespace: namespace, + }, + Spec: v1alpha1.DeviceSpec{ + Endpoint: v1alpha1.Endpoint{ + Address: gnmiAddr, + }, + }, + } + if err := c.Create(ctx, device); err != nil { + return nil, err + } + + // Set the device status to Running so that dependent resources can reconcile + device.Status.Phase = v1alpha1.DevicePhaseRunning + if err := c.Status().Update(ctx, device); err != nil { + return nil, err + } + + return device, nil +} + +// ============================================================================ +// Reconciliation tests +// ============================================================================ + +var _ = Describe("gNMI requests tests", func() { + // Resolve provider during tree construction so we can generate individual It nodes + projectDir, err := testutil.GetProjectDir() + if err != nil { + Fail(fmt.Sprintf("Failed to get project directory: %v", err)) + } + + provider := os.Getenv("E2E_PROVIDER") + providerNames := make([]string, len(testutil.SupportedProviders)) + for i, cfg := range testutil.SupportedProviders { + providerNames[i] = string(cfg.Name) + } + + // If provider is invalid, create a failing test with clear message + if provider == "" { + It("requires E2E_PROVIDER to be set", func() { + Fail(fmt.Sprintf("E2E_PROVIDER not set. Please set E2E_PROVIDER to one of: %s", strings.Join(providerNames, ", "))) + }) + return + } + + providerIdx := slices.IndexFunc(testutil.SupportedProviders, func(cfg testutil.ProviderConfig) bool { + return string(cfg.Name) == provider + }) + if providerIdx < 0 { + It("requires valid E2E_PROVIDER", func() { + Fail(fmt.Sprintf("E2E_PROVIDER=%q is not a supported provider. Valid values: %s", provider, strings.Join(providerNames, ", "))) + }) + return + } + + providerCfg := testutil.SupportedProviders[providerIdx] + testdataDir := filepath.Join(projectDir, "test", "e2e", "testdata", string(providerCfg.Name)) + + if _, err := os.Stat(testdataDir); os.IsNotExist(err) { + It("requires testdata directory", func() { + Fail(fmt.Sprintf("Testdata directory does not exist for provider %q: %s", provider, testdataDir)) + }) + return + } + + // Discover test files during tree construction + testFiles, err := filepath.Glob(filepath.Join(testdataDir, "*.txt")) + if err != nil || len(testFiles) == 0 { + It("requires test files", func() { + if err != nil { + Fail(fmt.Sprintf("Failed to glob testdata: %v", err)) + } + Fail(fmt.Sprintf("No test files found in %s", testdataDir)) + }) + return + } + + Describe(fmt.Sprintf("Provider: %s", providerCfg.Name), Ordered, func() { + var ptc *ProviderTestContext + var device *v1alpha1.Device + var testNamespace string + + BeforeAll(func(ctx SpecContext) { + By("creating dedicated test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: reconcileTestNamespacePrefix, + }, + } + Expect(testEnv.Client().Create(ctx, ns)).To(Succeed()) + testNamespace = ns.Name + + By(fmt.Sprintf("setting up %s provider", providerCfg.Name)) + ptc = SetupProviderTest(providerCfg, testEnv.Client(), testEnv.RESTConfig(), testNamespace) + }) + + AfterAll(func(ctx SpecContext) { + By(fmt.Sprintf("tearing down %s provider manager", providerCfg.Name)) + TeardownProviderTest(ptc) + + By("deleting test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + } + _ = testEnv.Client().Delete(ctx, ns) + }) + + AfterEach(func(ctx SpecContext) { + By("cleaning up resources") + cleanupAllResources(testEnv.Client(), testNamespace) + + if device == nil { + return + } + + By("deleting test device") + Expect(client.IgnoreNotFound(testEnv.Client().Delete(ctx, device))).To(Succeed()) + Eventually(func(g Gomega) { + err := testEnv.Client().Get(ctx, client.ObjectKeyFromObject(device), &v1alpha1.Device{}) + g.Expect(client.IgnoreNotFound(err)).To(Succeed()) + g.Expect(err).To(HaveOccurred(), "Device should be deleted") + }).Should(Succeed()) + device = nil + + By("clearing gNMI state for next test") + Expect(testEnv.ClearGNMIState(ctx)).To(Succeed()) + }) + + // Generate individual It nodes for each test file + for _, testFile := range testFiles { + testFile := testFile // capture for closure + testName := filepath.Base(testFile) + testName = testName[:len(testName)-4] // remove .txt + + It("should reconcile "+testName, func(ctx SpecContext) { + By("parsing testdata file") + a, err := txtar.ParseFile(testFile) + Expect(err).NotTo(HaveOccurred(), "Failed to parse test file: %s", testFile) + Expect(len(a.Files)).To(BeNumerically(">=", 2), "Expected at least 2 files (resource(s) and state)") + + var state, preload []byte + var resources []txtar.File + for _, f := range a.Files { + switch f.Name { + case "state/expect": + state = f.Data + case "state/preload": + preload = f.Data + default: + resources = append(resources, f) + } + } + Expect(state).NotTo(BeEmpty(), "Expected '-- state/expect --' section in testdata") + Expect(resources).NotTo(BeEmpty(), "Expected at least one resource in testdata") + + // Preload gNMI state BEFORE creating Device (e.g., bootTime for Device controller) + if len(preload) > 0 { + By("preloading gNMI state") + Expect(testEnv.PreloadGNMIState(ctx, preload)).To(Succeed(), "Failed to preload gNMI state") + } + + By("creating test device") + device, err = testutil.CreateTestDevice(ctx, testEnv.Client(), testEnv.GNMIAddress(), testNamespace) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("creating %d resource(s) from testdata", len(resources))) + for _, res := range resources { + obj := createResourceFromTxtar(ctx, testEnv.Client(), res, device.Name, testNamespace) + waitForResource(ctx, testEnv.Client(), obj) + } + + By("verifying gNMI state matches expected JSON") + gnmiState, err := testEnv.GetGNMIState(ctx) + Expect(err).NotTo(HaveOccurred()) + + err = testutil.CompareJSON(string(gnmiState), string(state)) + Expect(err).NotTo(HaveOccurred(), "gNMI state does not match expected JSON") + }) + } + }) +}) + +// createResourceFromTxtar creates a K8s resource from txtar file data. +// The file name format is "kind/name" (e.g., "prefixset/my-prefixset"). +// It substitutes "device" in deviceRef.name with the actual device name. +func createResourceFromTxtar(ctx SpecContext, c client.Client, res txtar.File, deviceName, namespace string) client.Object { + obj := &unstructured.Unstructured{} + Expect(yaml.Unmarshal(res.Data, obj)).To(Succeed(), "Failed to unmarshal %s", res.Name) + + // Set the namespace + obj.SetNamespace(namespace) + + // Update deviceRef.name to use the actual device name + if spec, ok := obj.Object["spec"].(map[string]any); ok { + if deviceRef, ok := spec["deviceRef"].(map[string]any); ok { + deviceRef["name"] = deviceName + } + } + // Also update the device label + labels := obj.GetLabels() + if labels != nil { + if _, ok := labels[v1alpha1.DeviceLabel]; ok { + labels[v1alpha1.DeviceLabel] = deviceName + obj.SetLabels(labels) + } + } + + Expect(c.Create(ctx, obj)).To(Succeed(), "Failed to create %s", res.Name) + return obj +} + +// waitForResource waits for a resource to be in Ready=True or Configured=True. +// If Configured condition exists, it checks it, otherwise it falls back to Ready condition. +// Skips config-only --controller-less-- resources that don't have status conditions (e.g., InterfaceConfig). +func waitForResource(ctx SpecContext, c client.Client, obj client.Object) { + key := client.ObjectKeyFromObject(obj) + gvk := obj.GetObjectKind().GroupVersionKind() + + // Add as needed. + switch gvk.Kind { + case "InterfaceConfig", "LLDPConfig", "BGPConfig", "NVEConfig", "ManagementAccessConfig": + return + } + + Eventually(func(g Gomega) { + r := &unstructured.Unstructured{} + r.SetGroupVersionKind(gvk) + g.Expect(c.Get(ctx, key, r)).To(Succeed()) + + conditions, err := testutil.ExtractConditions(r) + g.Expect(err).NotTo(HaveOccurred()) + + conditionToCheck := string(v1alpha1.ReadyCondition) + if apimeta.FindStatusCondition(conditions, string(v1alpha1.ConfiguredCondition)) != nil { + conditionToCheck = string(v1alpha1.ConfiguredCondition) + } + + g.Expect(apimeta.IsStatusConditionTrue(conditions, conditionToCheck)).To(BeTrue()) + }).Should(Succeed()) +} + +// cleanupAllResources deletes all test resources and lets the controller handle finalizer cleanup. +// Uses a background context with timeout to ensure cleanup completes even on interrupt. +// Deletion order: CoreResources first (have finalizers), then ConfigResources, then Device. +func cleanupAllResources(c client.Client, namespace string) { + // Use background context with timeout - cleanup must complete even on Ctrl+C + cleanupCtx, cancel := context.WithTimeout(context.Background(), testutil.LongTimeout) + defer cancel() + + deleteResources := func(gvks []schema.GroupVersionKind) { + for _, gvk := range gvks { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind + "List", + }) + + if err := c.List(cleanupCtx, list, client.InNamespace(namespace)); err != nil { + if apimeta.IsNoMatchError(err) { + continue // CRD not installed, skip + } + Expect(err).NotTo(HaveOccurred(), "Failed to list %s", gvk.Kind) + } + + // Delete all resources - controller will handle finalizer removal + for _, item := range list.Items { + Expect(client.IgnoreNotFound(c.Delete(cleanupCtx, &item))).To(Succeed()) + } + } + } + + // Delete core resources first (have finalizers that need Device + configs) + deleteResources(testutil.CoreResources) + // Then delete config resources + deleteResources(testutil.ConfigResources) +} diff --git a/test/e2e/testutil/envtest.go b/test/e2e/testutil/envtest.go new file mode 100644 index 000000000..244840764 --- /dev/null +++ b/test/e2e/testutil/envtest.go @@ -0,0 +1,186 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "context" + "os" + "path/filepath" + "slices" + + gnmitestserver "github.com/ironcore-dev/gnmi-test-server/testserver" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + nxv1alpha1 "github.com/ironcore-dev/network-operator/api/cisco/nx/v1alpha1" + "github.com/ironcore-dev/network-operator/api/core/v1alpha1" +) + +// EnvtestEnvironment implements TestEnvironment using envtest and an in-process gNMI server. +type EnvtestEnvironment struct { + testEnv *envtest.Environment + restConfig *rest.Config + k8sClient client.Client + gnmiServer *gnmitestserver.Server + gnmiAddr string + cancel context.CancelFunc +} + +// NewEnvtestEnvironment creates a new envtest-based test environment. +func NewEnvtestEnvironment() *EnvtestEnvironment { + return &EnvtestEnvironment{} +} + +// Setup initializes envtest and starts the in-process gNMI server. +func (e *EnvtestEnvironment) Setup(ctx context.Context) error { + ctx, e.cancel = context.WithCancel(ctx) + + // Start in-process gNMI test server with NX-OS behavior + var err error + e.gnmiServer, e.gnmiAddr, _, err = gnmitestserver.NewTestServer(ctx, gnmitestserver.WithNXOSBehavior()) + if err != nil { + return err + } + + // Register schemes + if err := corev1.AddToScheme(scheme.Scheme); err != nil { + return err + } + if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + if err := nxv1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + + // Start envtest + e.testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // Detect test binary directory for IDEs + if dir := detectTestBinaryDir(); dir != "" { + e.testEnv.BinaryAssetsDirectory = dir + } + + e.restConfig, err = e.testEnv.Start() + if err != nil { + return err + } + + e.k8sClient, err = client.New(e.restConfig, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return err + } + + // Wait for default namespace to be ready + for { + var ns corev1.Namespace + if err := e.k8sClient.Get(ctx, client.ObjectKey{Name: metav1.NamespaceDefault}, &ns); err == nil { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + } + + return nil +} + +// Teardown stops envtest and the gNMI server. +func (e *EnvtestEnvironment) Teardown(_ context.Context) error { + if e.cancel != nil { + e.cancel() + } + + if e.gnmiServer != nil { + if err := e.gnmiServer.Close(); err != nil { + return err + } + } + + if e.testEnv != nil { + if err := e.testEnv.Stop(); err != nil { + return err + } + } + + return nil +} + +// Client returns the Kubernetes client. +func (e *EnvtestEnvironment) Client() client.Client { + return e.k8sClient +} + +// RESTConfig returns the REST config. +func (e *EnvtestEnvironment) RESTConfig() *rest.Config { + return e.restConfig +} + +// GNMIAddress returns the in-process gNMI server address. +func (e *EnvtestEnvironment) GNMIAddress() string { + return e.gnmiAddr +} + +// GetGNMIState fetches state directly from the in-process server. +func (e *EnvtestEnvironment) GetGNMIState(_ context.Context) ([]byte, error) { + return e.gnmiServer.GetState() +} + +// ClearGNMIState clears state directly on the in-process server. +func (e *EnvtestEnvironment) ClearGNMIState(_ context.Context) error { + e.gnmiServer.ClearState() + return nil +} + +// PreloadGNMIState replaces the in-process gNMI server state with the given JSON. +// This resets the server to a clean state for test isolation. +func (e *EnvtestEnvironment) PreloadGNMIState(_ context.Context, jsonData []byte) error { + if len(jsonData) == 0 { + return nil + } + // Replace entire state by directly setting Buf (thread-safe with Lock) + e.gnmiServer.State.Lock() + e.gnmiServer.State.Buf = jsonData + e.gnmiServer.State.Unlock() + return nil +} + +// IsEnvtest returns true for envtest mode. +func (e *EnvtestEnvironment) IsEnvtest() bool { + return true +} + +// detectTestBinaryDir locates the envtest binary directory. +// The structure is: bin/k8s/k8s/--/ +// We need to find the version directory that contains etcd/kube-apiserver. +func detectTestBinaryDir() string { + basePath := filepath.Join("..", "..", "bin", "k8s", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + return "" + } + // Find a version directory (e.g., "1.35.0-darwin-arm64") + idx := slices.IndexFunc(entries, func(e os.DirEntry) bool { + if !e.IsDir() { + return false + } + // Check if this directory contains etcd + etcdPath := filepath.Join(basePath, e.Name(), "etcd") + _, err := os.Stat(etcdPath) + return err == nil + }) + if idx >= 0 { + return filepath.Join(basePath, entries[idx].Name()) + } + return "" +} From bce5bb055bf1b42eea4f55cd0a6a1a2579891f3b Mon Sep 17 00:00:00 2001 From: Pujol Date: Thu, 18 Jun 2026 10:04:34 +0200 Subject: [PATCH 06/10] build: add test-e2e-envtest target for envtest mode Add test-e2e-envtest target that runs e2e tests in envtest mode (no cluster required). Uses setup-envtest to provide kubebuilder assets. Signed-off-by: Pujol --- Makefile | 5 +++++ Makefile.maker.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Makefile b/Makefile index 73ed4b966..6a9221c8f 100644 --- a/Makefile +++ b/Makefile @@ -107,6 +107,11 @@ test-e2e-cluster: FORCE install-ginkgo @printf "\e[1;36m>> ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ (PROVIDER=$(PROVIDER))\e[0m\n" @KIND_CLUSTER=$(KIND_CLUSTER) E2E_PROVIDER=$(PROVIDER) ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ +# Run gNMI controller tests in envtest mode (no cluster required). +test-e2e-envtest: FORCE install-setup-envtest + @printf "\e[1;36m>> go test ./test/e2e/ -tags=envtest -v -ginkgo.v (PROVIDER=$(PROVIDER))\e[0m\n" + @KUBEBUILDER_ASSETS=$$(setup-envtest use 1.32 -p path) E2E_PROVIDER=$(PROVIDER) go test ./test/e2e/ -tags=envtest -v -ginkgo.v + docker-build: FORCE @printf "\e[1;36m>> $(CONTAINER_TOOL) build --tag=$(IMG) .\e[0m\n" @$(CONTAINER_TOOL) build --build-arg=BININFO_BUILD_DATE=$(BININFO_BUILD_DATE) --build-arg=BININFO_COMMIT_HASH=$(BININFO_COMMIT_HASH) --build-arg=BININFO_VERSION=$(BININFO_VERSION) --tag=$(IMG) . diff --git a/Makefile.maker.yaml b/Makefile.maker.yaml index fe8639dd5..3ecaefc5b 100644 --- a/Makefile.maker.yaml +++ b/Makefile.maker.yaml @@ -149,6 +149,11 @@ verbatim: | @printf "\e[1;36m>> ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ (PROVIDER=$(PROVIDER))\e[0m\n" @KIND_CLUSTER=$(KIND_CLUSTER) E2E_PROVIDER=$(PROVIDER) ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ + # Run gNMI controller tests in envtest mode (no cluster required). + test-e2e-envtest: FORCE install-setup-envtest + @printf "\e[1;36m>> go test ./test/e2e/ -tags=envtest -v -ginkgo.v (PROVIDER=$(PROVIDER))\e[0m\n" + @KUBEBUILDER_ASSETS=$$(setup-envtest use 1.32 -p path) E2E_PROVIDER=$(PROVIDER) go test ./test/e2e/ -tags=envtest -v -ginkgo.v + docker-build: FORCE @printf "\e[1;36m>> $(CONTAINER_TOOL) build --tag=$(IMG) .\e[0m\n" @$(CONTAINER_TOOL) build --build-arg=BININFO_BUILD_DATE=$(BININFO_BUILD_DATE) --build-arg=BININFO_COMMIT_HASH=$(BININFO_COMMIT_HASH) --build-arg=BININFO_VERSION=$(BININFO_VERSION) --tag=$(IMG) . From 76b47676f9727f9957be1818e4d4a8f24200d789 Mon Sep 17 00:00:00 2001 From: Pujol Date: Wed, 17 Jun 2026 19:54:18 +0200 Subject: [PATCH 07/10] test: add testdata fixtures for all resource types Add txtar-format test fixtures for all supported resource types. Signed-off-by: Pujol --- test/e2e/testdata/cisco-nxos-gnmi/acl.txt | 79 ++++++ test/e2e/testdata/cisco-nxos-gnmi/banner.txt | 41 ++++ .../testdata/cisco-nxos-gnmi/bgp_bgppeer.txt | 226 ++++++++++++++++++ .../testdata/cisco-nxos-gnmi/dhcprelay.txt | 178 ++++++++++++++ test/e2e/testdata/cisco-nxos-gnmi/dns.txt | 63 +++++ .../testdata/cisco-nxos-gnmi/evpninstance.txt | 64 +++++ test/e2e/testdata/cisco-nxos-gnmi/isis.txt | 158 ++++++++++++ test/e2e/testdata/cisco-nxos-gnmi/lldp.txt | 58 +++++ .../cisco-nxos-gnmi/managementaccess.txt | 75 ++++++ test/e2e/testdata/cisco-nxos-gnmi/ntp.txt | 62 +++++ test/e2e/testdata/cisco-nxos-gnmi/nve.txt | 144 +++++++++++ test/e2e/testdata/cisco-nxos-gnmi/ospf.txt | 159 ++++++++++++ test/e2e/testdata/cisco-nxos-gnmi/pim.txt | 142 +++++++++++ .../testdata/cisco-nxos-gnmi/routedvlan.txt | 136 +++++++++++ .../routingpolicy_prefixset.txt | 154 ++++++++++++ test/e2e/testdata/cisco-nxos-gnmi/snmp.txt | 76 ++++++ .../testdata/cisco-nxos-gnmi/subinterface.txt | 142 +++++++++++ test/e2e/testdata/cisco-nxos-gnmi/syslog.txt | 76 ++++++ .../testdata/cisco-nxos-gnmi/vpcdomain.txt | 191 +++++++++++++++ test/e2e/testdata/cisco-nxos-gnmi/vrf.txt | 129 ++++++++++ test/e2e/testdata/openconfig/interface.txt | 61 +++++ 21 files changed, 2414 insertions(+) create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/acl.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/banner.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/bgp_bgppeer.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/dhcprelay.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/dns.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/evpninstance.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/isis.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/lldp.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/managementaccess.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/ntp.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/nve.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/ospf.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/pim.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/routedvlan.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/routingpolicy_prefixset.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/snmp.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/subinterface.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/syslog.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/vpcdomain.txt create mode 100644 test/e2e/testdata/cisco-nxos-gnmi/vrf.txt create mode 100644 test/e2e/testdata/openconfig/interface.txt diff --git a/test/e2e/testdata/cisco-nxos-gnmi/acl.txt b/test/e2e/testdata/cisco-nxos-gnmi/acl.txt new file mode 100644 index 000000000..95e5a8841 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/acl.txt @@ -0,0 +1,79 @@ +# NX-OS Access Control List Integration Test +# +# Tests ACL configuration with permit/deny entries. + + + +-- accesscontrollist/test-acl -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: AccessControlList +metadata: + name: test-acl + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: BLOCK-EXTERNAL + entries: + - sequence: 10 + action: Permit + protocol: IP + sourceAddress: 10.0.0.0/8 + destinationAddress: 0.0.0.0/0 + description: "Allow internal" + - sequence: 20 + action: Deny + protocol: IP + sourceAddress: 0.0.0.0/0 + destinationAddress: 0.0.0.0/0 + description: "Deny all" + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- state/expect -- +{ + "System": { + "acl-items": { + "ipv4-items": { + "name-items": { + "ACL-list": [ + { + "name": "BLOCK-EXTERNAL", + "seq-items": { + "ACE-list": [ + { + "action": "deny", + "dstPrefix": "0.0.0.0", + "protocol": 0, + "seqNum": 20, + "srcPrefix": "0.0.0.0" + }, + { + "action": "permit", + "dstPrefix": "0.0.0.0", + "protocol": 0, + "seqNum": 10, + "srcPrefix": "10.0.0.0", + "srcPrefixLength": 8 + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/banner.txt b/test/e2e/testdata/cisco-nxos-gnmi/banner.txt new file mode 100644 index 000000000..239703a56 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/banner.txt @@ -0,0 +1,41 @@ +# NX-OS Banner Integration Test + + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- banner/prelogin -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Banner +metadata: + name: prelogin + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + type: PreLogin + message: + inline: "Authorized users only. All activity is monitored." + +-- state/expect -- +{ + "System": { + "userext-items": { + "preloginbanner-items": { + "delimiter": "^", + "message": "Authorized users only. All activity is monitored." + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/bgp_bgppeer.txt b/test/e2e/testdata/cisco-nxos-gnmi/bgp_bgppeer.txt new file mode 100644 index 000000000..990761e8f --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/bgp_bgppeer.txt @@ -0,0 +1,226 @@ +# NX-OS BGP and BGPPeer Integration Test +# +# Resource dependency chain: +# loopback-vtep -> Loopback interface (spec.name: lo0) for BGP router-id/update-source +# evpn-settings -> BGPConfig (NX-OS provider config) for advertise-pip +# evpn-bgp -> BGP instance with L2VPN EVPN, references BGPConfig +# spine1 -> BGPPeer referencing BGP instance and loopback for localAddress + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/loopback-vtep -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: loopback-vtep + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: lo0 + adminState: Up + type: Loopback + ipv4: + addresses: + - 10.0.0.1/32 + +-- bgpconfig/evpn-settings -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: BGPConfig +metadata: + name: evpn-settings + namespace: default +spec: + addressFamilies: + l2vpnEvpn: + advertisePIP: true + +-- bgp/evpn-bgp -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: evpn-bgp + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + asNumber: 65000 + routerId: "10.0.0.1" + providerConfigRef: + apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 + kind: BGPConfig + name: evpn-settings + addressFamilies: + l2vpnEvpn: + enabled: true + routeTargetPolicy: + retainAll: true + +-- bgppeer/spine1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGPPeer +metadata: + name: spine1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + bgpRef: + name: evpn-bgp + address: "10.0.0.2" + asNumber: 65000 + description: "EVPN peering with spine1" + localAddress: + interfaceRef: + name: loopback-vtep + addressFamilies: + l2vpnEvpn: + enabled: true + sendCommunity: Both + routeReflectorClient: true + +-- state/expect -- +{ + "System": { + "bgp-items": { + "inst-items": { + "adminSt": "enabled", + "asn": "65000", + "dom-items": { + "Dom-list": [ + { + "name": "default", + "rtrId": "10.0.0.1", + "rtrIdAuto": "disabled", + "af-items": { + "DomAf-list": [ + { + "advPip": "enabled", + "exportGwIp": "disabled", + "maxExtEcmp": 1, + "maxExtIntEcmp": 1, + "retainRttAll": "enabled", + "type": "l2vpn-evpn" + } + ] + }, + "peer-items": { + "Peer-list": [ + { + "addr": "10.0.0.2", + "adminSt": "enabled", + "asn": "65000", + "asnType": "none", + "name": "EVPN peering with spine1", + "srcIf": "lo0", + "af-items": { + "PeerAf-list": [ + { + "ctrl": "rr-client", + "sendComExt": "enabled", + "sendComStd": "enabled", + "type": "l2vpn-evpn" + } + ] + } + } + ] + }, + "peercont-items": { + "PeerCont-list": [ + { + "name": "__operator-managed--default__" + } + ] + } + } + ] + } + } + }, + "fm-items": { + "bgp-items": { + "adminSt": "enabled" + }, + "evpn-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "ctrl": "port-unreachable,redirect", + "id": "lo0" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "id": "lo0", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "addr-items": { + "Addr-list": [ + { + "addr": "10.0.0.1/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + }, + "id": "lo0" + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/dhcprelay.txt b/test/e2e/testdata/cisco-nxos-gnmi/dhcprelay.txt new file mode 100644 index 000000000..232f45866 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/dhcprelay.txt @@ -0,0 +1,178 @@ +# NX-OS DHCP Relay Integration Test +# +# Resource dependency chain: +# vlan100 -> VLAN (id: 100, name: DHCP-VLAN) +# svi100 -> RoutedVLAN Interface referencing vlan100 (spec.name: Vlan100) +# relay-config -> DHCPRelay with server and interface reference + + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- vlan/vlan100 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VLAN +metadata: + name: vlan100 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + id: 100 + name: DHCP-VLAN + +-- interface/svi100 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: svi100 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: Vlan100 + adminState: Up + type: RoutedVLAN + vlanRef: + name: vlan100 + ipv4: + addresses: + - 192.168.100.1/24 + +-- dhcprelay/relay-config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: DHCPRelay +metadata: + name: relay-config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + servers: + - "10.0.0.10" + interfaceRefs: + - name: svi100 + +-- state/expect -- +{ + "System": { + "bd-items": { + "bd-items": { + "BD-list": [ + { + "BdState": "active", + "adminSt": "active", + "fabEncap": "vlan-100", + "name": "DHCP-VLAN" + } + ] + } + }, + "dhcp-items": { + "inst-items": { + "relayif-items": { + "RelayIf-list": [ + { + "addr-items": { + "RelayAddr-list": [ + { + "address": "10.0.0.10", + "vrf": "!unspecified" + } + ] + }, + "id": "vlan100" + } + ] + } + } + }, + "fm-items": { + "dhcp-items": { + "adminSt": "enabled" + }, + "ifvlan-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "if-items": { + "If-list": [ + { + "ctrl": "port-unreachable", + "id": "vlan100" + } + ] + }, + "name": "default" + } + ] + } + } + }, + "intf-items": { + "svi-items": { + "If-list": [ + { + "adminSt": "up", + "descr": "", + "id": "vlan100", + "medium": "bcast", + "mtu": 1500, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + }, + "vlanId": 100 + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "if-items": { + "If-list": [ + { + "addr-items": { + "Addr-list": [ + { + "addr": "192.168.100.1/24", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + }, + "id": "vlan100" + } + ] + }, + "name": "default" + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/dns.txt b/test/e2e/testdata/cisco-nxos-gnmi/dns.txt new file mode 100644 index 000000000..68c640c3c --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/dns.txt @@ -0,0 +1,63 @@ +# NX-OS DNS Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- dns/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: DNS +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + domain: example.com + servers: + - address: 10.10.10.10 + vrfName: management + +-- state/expect -- +{ + "System": { + "dns-items": { + "adminSt": "enabled", + "prof-items": { + "Prof-list": [ + { + "name": "default", + "dom-items": { + "name": "example.com" + }, + "vrf-items": { + "Vrf-list": [ + { + "name": "management", + "prov-items": { + "Provider-list": [ + { + "addr": "10.10.10.10" + } + ] + } + } + ] + } + } + ] + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/evpninstance.txt b/test/e2e/testdata/cisco-nxos-gnmi/evpninstance.txt new file mode 100644 index 000000000..c54ba30f8 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/evpninstance.txt @@ -0,0 +1,64 @@ +# NX-OS EVPNInstance Integration Test +# +# Tests EVPN Instance (L3VNI) for IP-VRF in VXLAN fabric. +# Using Routed type which doesn't require a VLAN dependency. + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- evpninstance/l3vni -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: EVPNInstance +metadata: + name: l3vni + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + vni: 50000 + type: Routed + routeDistinguisher: "auto" + +-- state/expect -- +{ + "System": { + "eps-items": { + "epId-items": { + "Ep-list": [ + { + "epId": "1", + "nws-items": { + "vni-items": { + "Nw-list": [ + { + "vni": 50000, + "associateVrfFlag": true + } + ] + } + } + } + ] + } + }, + "fm-items": { + "nvo-items": { + "adminSt": "enabled" + }, + "vnsegment-items": { + "adminSt": "enabled" + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/isis.txt b/test/e2e/testdata/cisco-nxos-gnmi/isis.txt new file mode 100644 index 000000000..36a8361d8 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/isis.txt @@ -0,0 +1,158 @@ +# NX-OS ISIS Integration Test +# +# Resource dependency chain: +# ethernet1 -> Physical Interface (spec.name: Ethernet1/1 -> eth1/1) +# fabric-isis -> ISIS instance referencing ethernet1 + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/ethernet1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: ethernet1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: Ethernet1/1 + adminState: Up + type: Physical + ipv4: + addresses: + - 10.0.1.1/30 + +-- isis/fabric-isis -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: ISIS +metadata: + name: fabric-isis + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + instance: FABRIC + networkEntityTitle: "49.0001.0000.0000.0001.00" + type: Level2 + addressFamilies: + - IPv4Unicast + interfaceRefs: + - name: ethernet1 + +-- state/expect -- +{ + "System": { + "fm-items": { + "isis-items": { + "adminSt": "enabled" + } + }, + "intf-items": { + "phys-items": { + "PhysIf-list": [ + { + "FECMode": "auto", + "accessVlan": "unknown", + "adminSt": "up", + "id": "eth1/1", + "layer": "Layer3", + "medium": "broadcast", + "mode": "access", + "mtu": 1500, + "nativeVlan": "unknown", + "physExtd-items": { + "bufferBoost": "enable" + }, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_state" + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "if-items": { + "If-list": [ + { + "addr-items": { + "Addr-list": [ + { + "addr": "10.0.1.1/30", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + }, + "id": "eth1/1" + } + ] + }, + "name": "default" + } + ] + } + } + }, + "isis-items": { + "inst-items": { + "Inst-list": [ + { + "adminSt": "enabled", + "dom-items": { + "Dom-list": [ + { + "af-items": { + "DomAf-list": [ + { + "type": "v4" + } + ] + }, + "if-items": { + "If-list": [ + { + "id": "eth1/1", + "networkTypeP2P": "on", + "v4Bfd": "inheritVrf", + "v4enable": true, + "v6Bfd": "", + "v6enable": false + } + ] + }, + "isType": "l2", + "name": "default", + "net": "49.0001.0000.0000.0001.00", + "passiveDflt": "l2" + } + ] + }, + "name": "FABRIC" + } + ] + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/lldp.txt b/test/e2e/testdata/cisco-nxos-gnmi/lldp.txt new file mode 100644 index 000000000..f3527bad2 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/lldp.txt @@ -0,0 +1,58 @@ +# NX-OS LLDP Integration Test + + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- lldpconfig/lldp-settings -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: LLDPConfig +metadata: + name: lldp-settings + namespace: default +spec: + holdTime: 180 + initDelay: 5 + +-- lldp/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: LLDP +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + providerConfigRef: + apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 + kind: LLDPConfig + name: lldp-settings + +-- state/expect -- +{ + "System": { + "fm-items": { + "lldp-items": { + "adminSt": "enabled" + } + }, + "lldp-items": { + "inst-items": { + "holdTime": 180, + "initDelayTime": 5 + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/managementaccess.txt b/test/e2e/testdata/cisco-nxos-gnmi/managementaccess.txt new file mode 100644 index 000000000..f4932744c --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/managementaccess.txt @@ -0,0 +1,75 @@ +# NX-OS ManagementAccess Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- managementaccess/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: ManagementAccess +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + grpc: + enabled: true + port: 9339 + vrfName: default + gnmi: + maxConcurrentCall: 8 + keepAliveTimeout: 600s + ssh: + enabled: true + timeout: 10m + sessionLimit: 32 + +-- state/expect -- +{ + "System": { + "fm-items": { + "grpc-items": { + "adminSt": "enabled" + }, + "ssh-items": { + "adminSt": "enabled" + } + }, + "grpc-items": { + "gnmi-items": { + "keepAliveTimeout": 600, + "maxCalls": 8 + }, + "port": 9339, + "useVrf": "default" + }, + "terml-items": { + "ln-items": { + "cons-items": { + "execTmeout-items": { + "timeout": 0 + } + }, + "vty-items": { + "execTmeout-items": { + "timeout": 10 + }, + "ssLmt-items": { + "sesLmt": 32 + } + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/ntp.txt b/test/e2e/testdata/cisco-nxos-gnmi/ntp.txt new file mode 100644 index 000000000..f0cbbbcc2 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/ntp.txt @@ -0,0 +1,62 @@ +# NX-OS NTP Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- ntp/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: NTP +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + sourceInterfaceName: mgmt0 + servers: + - address: ntp.example.com + prefer: true + vrfName: management + +-- state/expect -- +{ + "System": { + "fm-items": { + "ntpd-items": { + "adminSt": "enabled" + } + }, + "time-items": { + "adminSt": "enabled", + "logging": "disabled", + "prov-items": { + "NtpProvider-list": [ + { + "keyId": 0, + "maxPoll": 6, + "minPoll": 4, + "name": "ntp.example.com", + "preferred": true, + "provT": "server", + "vrf": "management" + } + ] + }, + "srcIf-items": { + "srcIf": "mgmt0" + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/nve.txt b/test/e2e/testdata/cisco-nxos-gnmi/nve.txt new file mode 100644 index 000000000..1eb311208 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/nve.txt @@ -0,0 +1,144 @@ +# NX-OS NVE (Network Virtualization Edge) Integration Test +# +# NVE requires a loopback interface as source. + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/lo-nve -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: lo-nve + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: lo1 + type: Loopback + adminState: Up + ipv4: + addresses: + - 10.0.0.1/32 + +-- nve/nve1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: NetworkVirtualizationEdge +metadata: + name: nve1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + sourceInterfaceRef: + name: lo-nve + hostReachability: BGP + +-- state/expect -- +{ + "System": { + "eps-items": { + "epId-items": { + "Ep-list": [ + { + "epId": 1, + "adminSt": "enabled", + "advertiseVmac": false, + "holdDownTime": 180, + "hostReach": "bgp", + "sourceInterface": "lo1", + "suppressARP": false + } + ] + } + }, + "fm-items": { + "evpn-items": { + "adminSt": "enabled" + }, + "nvo-items": { + "adminSt": "enabled" + } + }, + "hmm-items": { + "fwdinst-items": { + "adminSt": "disabled", + "amac": "" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo1", + "ctrl": "port-unreachable,redirect" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "id": "lo1", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo1", + "addr-items": { + "Addr-list": [ + { + "addr": "10.0.0.1/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/ospf.txt b/test/e2e/testdata/cisco-nxos-gnmi/ospf.txt new file mode 100644 index 000000000..305bad3f9 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/ospf.txt @@ -0,0 +1,159 @@ +# NX-OS OSPF Integration Test +# +# OSPF requires at least one interface. + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/lo-ospf -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: lo-ospf + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: lo10 + type: Loopback + adminState: Up + ipv4: + addresses: + - 10.255.255.10/32 + +-- ospf/underlay -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: OSPF +metadata: + name: underlay + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + instance: UNDERLAY + routerId: 10.255.255.10 + interfaceRefs: + - name: lo-ospf + area: "0.0.0.0" + +-- state/expect -- +{ + "System": { + "fm-items": { + "ospf-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo10", + "ctrl": "port-unreachable,redirect" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "id": "lo10", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo10", + "addr-items": { + "Addr-list": [ + { + "addr": "10.255.255.10/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "ospf-items": { + "inst-items": { + "Inst-list": [ + { + "name": "UNDERLAY", + "adminSt": "enabled", + "dom-items": { + "Dom-list": [ + { + "name": "default", + "adminSt": "enabled", + "adjChangeLogLevel": "none", + "bwRef": 40000, + "bwRefUnit": "mbps", + "ctrl": "default-passive", + "dist": 110, + "rtrId": "10.255.255.10", + "if-items": { + "If-list": [ + { + "id": "lo10", + "adminSt": "enabled", + "advertiseSecondaries": true, + "area": "0.0.0.0", + "bfdCtrl": "unspecified", + "nwT": "unspecified", + "passiveCtrl": "disabled" + } + ] + } + } + ] + } + } + ] + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/pim.txt b/test/e2e/testdata/cisco-nxos-gnmi/pim.txt new file mode 100644 index 000000000..a1e1e39cf --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/pim.txt @@ -0,0 +1,142 @@ +# NX-OS PIM Integration Test +# +# PIM requires at least one interface. + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/lo-pim -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: lo-pim + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: lo20 + type: Loopback + adminState: Up + ipv4: + addresses: + - 10.255.255.20/32 + +-- pim/multicast -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: PIM +metadata: + name: multicast + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + interfaceRefs: + - name: lo-pim + mode: Sparse + +-- state/expect -- +{ + "System": { + "fm-items": { + "pim-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo20", + "ctrl": "port-unreachable,redirect" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "id": "lo20", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo20", + "addr-items": { + "Addr-list": [ + { + "addr": "10.255.255.20/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "pim-items": { + "adminSt": "enabled", + "inst-items": { + "adminSt": "enabled", + "dom-items": { + "Dom-list": [ + { + "name": "default", + "adminSt": "enabled", + "if-items": { + "If-list": [ + { + "id": "lo20", + "pimSparseMode": true + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/routedvlan.txt b/test/e2e/testdata/cisco-nxos-gnmi/routedvlan.txt new file mode 100644 index 000000000..7568c3628 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/routedvlan.txt @@ -0,0 +1,136 @@ +# NX-OS RoutedVLAN (SVI) Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- vlan/vlan10 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VLAN +metadata: + name: vlan10 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + id: 10 + name: SERVERS + +-- interface/svi-vlan10 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: svi-vlan10 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: vlan10 + type: RoutedVLAN + adminState: Up + mtu: 9000 + description: "SVI for VLAN 10 - Servers" + vlanRef: + name: vlan10 + ipv4: + addresses: + - 10.10.0.1/24 + +-- state/expect -- +{ + "System": { + "bd-items": { + "bd-items": { + "BD-list": [ + { + "BdState": "active", + "adminSt": "active", + "fabEncap": "vlan-10", + "name": "SERVERS" + } + ] + } + }, + "fm-items": { + "ifvlan-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "vlan10", + "ctrl": "port-unreachable" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "svi-items": { + "If-list": [ + { + "adminSt": "up", + "descr": "SVI for VLAN 10 - Servers", + "id": "vlan10", + "medium": "bcast", + "mtu": 9000, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + }, + "vlanId": 10 + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "vlan10", + "addr-items": { + "Addr-list": [ + { + "addr": "10.10.0.1/24", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/routingpolicy_prefixset.txt b/test/e2e/testdata/cisco-nxos-gnmi/routingpolicy_prefixset.txt new file mode 100644 index 000000000..dc3a070a7 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/routingpolicy_prefixset.txt @@ -0,0 +1,154 @@ +# RoutingPolicy with PrefixSet reference + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- prefixset/test-prefixset-v4 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: PrefixSet +metadata: + name: test-prefixset-v4 + namespace: default + labels: + networking.metal.ironcore.dev/device-name: device +spec: + deviceRef: + name: device + name: IPV4-NETWORKS + entries: + - sequence: 10 + prefix: 10.0.0.0/8 +-- prefixset/test-prefixset-v6 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: PrefixSet +metadata: + name: test-prefixset-v6 + namespace: default + labels: + networking.metal.ironcore.dev/device-name: device +spec: + deviceRef: + name: device + name: IPV6-NETWORKS + entries: + - sequence: 10 + prefix: "2001:db8::/32" +-- routingpolicy/rm-import -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: RoutingPolicy +metadata: + name: rm-import + namespace: default + labels: + networking.metal.ironcore.dev/device-name: device +spec: + deviceRef: + name: device + name: RM-IMPORT + statements: + - sequence: 10 + conditions: + matchPrefixSet: + prefixSetRef: + name: test-prefixset-v4 + actions: + routeDisposition: AcceptRoute + - sequence: 20 + conditions: + matchPrefixSet: + prefixSetRef: + name: test-prefixset-v6 + actions: + routeDisposition: AcceptRoute + +-- state/expect -- +{ + "System": { + "rpm-items": { + "pfxlistv4-items": { + "RuleV4-list": [ + { + "name": "IPV4-NETWORKS", + "ent-items": { + "Entry-list": [ + { + "action": "permit", + "criteria": "exact", + "fromPfxLen": 0, + "order": 10, + "pfx": "10.0.0.0/8", + "toPfxLen": 0 + } + ] + } + } + ] + }, + "pfxlistv6-items": { + "RuleV6-list": [ + { + "name": "IPV6-NETWORKS", + "ent-items": { + "Entry-list": [ + { + "action": "permit", + "criteria": "exact", + "fromPfxLen": 0, + "order": 10, + "pfx": "2001:db8::/32", + "toPfxLen": 0 + } + ] + } + } + ] + }, + "rtmap-items": { + "Rule-list": [ + { + "name": "RM-IMPORT", + "ent-items": { + "Entry-list": [ + { + "action": "permit", + "order": 10, + "mrtdst-items": { + "rsrtDstAtt-items": { + "RsRtDstAtt-list": [ + { + "tDn": "/System/rpm-items/pfxlistv4-items/RuleV4-list[name='IPV4-NETWORKS']" + } + ] + } + } + }, + { + "action": "permit", + "order": 20, + "mrtdst-items": { + "rsrtDstAtt-items": { + "RsRtDstAtt-list": [ + { + "tDn": "/System/rpm-items/pfxlistv6-items/RuleV6-list[name='IPV6-NETWORKS']" + } + ] + } + } + } + ] + } + } + ] + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/snmp.txt b/test/e2e/testdata/cisco-nxos-gnmi/snmp.txt new file mode 100644 index 000000000..8e8515bff --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/snmp.txt @@ -0,0 +1,76 @@ +# NX-OS SNMP Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- snmp/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: SNMP +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + contact: "admin@example.com" + location: "DC1-Rack42" + sourceInterfaceName: "mgmt0" + hosts: + - address: "10.0.0.100" + communities: + - name: "public" + group: "network-operator" + +-- state/expect -- +{ + "System": { + "snmp-items": { + "inst-items": { + "community-items": { + "CommSecP-list": [ + { + "commAcess": "unspecified", + "grpName": "network-operator", + "name": "public" + } + ] + }, + "globals-items": { + "srcInterfaceInforms-items": { + "ifname": "mgmt0" + }, + "srcInterfaceTraps-items": { + "ifname": "mgmt0" + } + }, + "host-items": { + "Host-list": [ + { + "hostName": "10.0.0.100", + "notifType": "traps", + "secLevel": "noauth", + "udpPortID": 162, + "version": "v2c" + } + ] + }, + "sysinfo-items": { + "sysContact": "admin@example.com", + "sysLocation": "DC1-Rack42" + }, + "traps-items": {} + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/subinterface.txt b/test/e2e/testdata/cisco-nxos-gnmi/subinterface.txt new file mode 100644 index 000000000..5235e7d77 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/subinterface.txt @@ -0,0 +1,142 @@ +# NX-OS Subinterface Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/parent-eth -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: parent-eth + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/3 + type: Physical + adminState: Up + mtu: 9216 + description: "Parent interface for subinterfaces" + +-- interface/subif-vlan100 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: subif-vlan100 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/3.100 + type: Subinterface + adminState: Up + mtu: 1500 + description: "L3 Subinterface VLAN 100" + parentInterfaceRef: + name: parent-eth + encapsulation: + type: "802.1q" + tag: 100 + ipv4: + addresses: + - 10.100.0.1/24 + +-- state/expect -- +{ + "System": { + "intf-items": { + "phys-items": { + "PhysIf-list": [ + { + "FECMode": "auto", + "accessVlan": "vlan-1", + "adminSt": "up", + "descr": "Parent interface for subinterfaces", + "id": "eth1/3", + "layer": "Layer2", + "medium": "broadcast", + "mode": "access", + "mtu": 9216, + "nativeVlan": "vlan-1", + "physExtd-items": { + "bufferBoost": "enable" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state" + } + ] + }, + "encrtd-items": { + "EncRtdIf-list": [ + { + "adminSt": "up", + "descr": "L3 Subinterface VLAN 100", + "encap": "vlan-100", + "id": "eth1/3.100", + "mediumType": "broadcast", + "mtu": 1500, + "mtuInherit": false, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "eth1/3.100", + "addr-items": { + "Addr-list": [ + { + "addr": "10.100.0.1/24", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "stp-items": { + "inst-items": { + "if-items": { + "If-list": [ + { + "id": "eth1/3", + "mode": "default", + "bpdufilter": "default", + "bpduguard": "default" + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/syslog.txt b/test/e2e/testdata/cisco-nxos-gnmi/syslog.txt new file mode 100644 index 000000000..f490780bd --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/syslog.txt @@ -0,0 +1,76 @@ +# NX-OS Syslog Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- syslog/logging -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Syslog +metadata: + name: logging + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + servers: + - address: "10.0.0.100" + severity: Warning + vrfName: management + facilities: + - name: Local7 + severity: Info + +-- state/expect -- +{ + "System": { + "logging-items": { + "loglevel-items": { + "facility-items": { + "Facility-list": [ + { + "facilityName": "Local7", + "severityLevel": "information" + } + ] + } + } + }, + "syslog-items": { + "logginghistory-items": { + "level": "information", + "size": 500 + }, + "originid-items": { + "idtype": "string", + "idvalue": "logging" + }, + "rdst-items": { + "RemoteDest-list": [ + { + "forwardingFacility": "local7", + "host": "10.0.0.100", + "port": 514, + "severity": "warnings", + "transport": "udp", + "vrfName": "management" + } + ] + }, + "source-items": { + "adminState": "enabled", + "ifName": "mgmt0" + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/vpcdomain.txt b/test/e2e/testdata/cisco-nxos-gnmi/vpcdomain.txt new file mode 100644 index 000000000..22738179d --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/vpcdomain.txt @@ -0,0 +1,191 @@ +# NX-OS VPCDomain Integration Test +# +# Tests vPC (Virtual Port Channel) domain configuration. +# VPCDomain is NX-OS specific (nx.cisco.networking.metal.ironcore.dev/v1alpha1). +# Requires a port-channel interface for the peer-link with member interfaces. +# +# Using this payload on a clean device will fail as features must be already enabled. + + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/eth1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: eth1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/1 + type: Physical + adminState: Up + +-- interface/vpc-peerlink -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: vpc-peerlink + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: po1 + type: Aggregate + adminState: Up + aggregation: + memberInterfaceRefs: + - name: eth1 + controlProtocol: + mode: Active + +-- vpcdomain/vpc1 -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: VPCDomain +metadata: + name: vpc1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + domainId: 100 + adminState: Up + rolePriority: 32667 + systemPriority: 32667 + peer: + adminState: Up + interfaceRef: + name: vpc-peerlink + keepalive: + destination: 10.0.0.2 + source: 10.0.0.1 + vrfName: management + +-- state/expect -- +{ + "System": { + "fm-items": { + "lacp-items": { + "adminSt": "enabled" + }, + "vpc-items": { + "adminSt": "enabled" + } + }, + "intf-items": { + "aggr-items": { + "AggrIf-list": [ + { + "id": "po1", + "adminSt": "up", + "accessVlan": "vlan-1", + "aggrExtd-items": { + "bufferBoost": "enable" + }, + "lacpVpcConvergence": "disable", + "layer": "Layer2", + "medium": "broadcast", + "mode": "access", + "mtu": 1500, + "nativeVlan": "vlan-1", + "pcMode": "active", + "rsmbrIfs-items": { + "RsMbrIfs-list": [ + { + "tDn": "/System/intf-items/phys-items/PhysIf-list[id='eth1/1']" + } + ] + }, + "suspIndividual": "enable", + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_state" + } + ] + }, + "phys-items": { + "PhysIf-list": [ + { + "id": "eth1/1", + "FECMode": "auto", + "accessVlan": "vlan-1", + "adminSt": "up", + "layer": "Layer2", + "medium": "broadcast", + "mode": "access", + "mtu": 1500, + "nativeVlan": "vlan-1", + "physExtd-items": { + "bufferBoost": "enable" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_state" + } + ] + } + }, + "stp-items": { + "inst-items": { + "if-items": { + "If-list": [ + { + "id": "eth1/1", + "bpdufilter": "default", + "bpduguard": "default", + "mode": "default" + }, + { + "id": "po1", + "bpdufilter": "default", + "bpduguard": "default", + "mode": "default" + } + ] + } + } + }, + "vpc-items": { + "inst-items": { + "dom-items": { + "id": 100, + "adminSt": "enabled", + "autoRecovery": "disabled", + "autoRecoveryIntvl": 240, + "delayRestoreSVI": 10, + "delayRestoreVPC": 30, + "fastConvergence": "disabled", + "keepalive-items": { + "destIp": "10.0.0.2", + "srcIp": "10.0.0.1", + "vrf": "management", + "peerlink-items": { + "adminSt": "enabled", + "id": "po1" + } + }, + "l3PeerRouter": "disabled", + "peerGw": "disabled", + "peerSwitch": "disabled", + "rolePrio": 32667, + "sysPrio": 32667 + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/vrf.txt b/test/e2e/testdata/cisco-nxos-gnmi/vrf.txt new file mode 100644 index 000000000..5bec2e965 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/vrf.txt @@ -0,0 +1,129 @@ +# NX-OS VRF Integration Test +# +# Tests VRF creation with L3 interface membership: +# tenant1 -> VRF with description +# vrf-uplink -> Physical L3 interface assigned to the VRF + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- vrf/tenant1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VRF +metadata: + name: k8s-vrf-tenant1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: TENANT-1 + description: "Tenant 1 VRF" + +-- interface/vrf-uplink -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: vrf-uplink + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/10 + type: Physical + adminState: Up + mtu: 9000 + description: "Uplink in TENANT-1 VRF" + vrfRef: + name: k8s-vrf-tenant1 + ipv4: + addresses: + - 10.1.1.1/30 + +-- state/expect -- +{ + "System": { + "inst-items": { + "Inst-list": [ + { + "name": "TENANT-1", + "descr": "Tenant 1 VRF", + "l3vni": false, + "dom-items": { + "Dom-list": [ + { + "name": "TENANT-1" + } + ] + } + } + ] + }, + "intf-items": { + "phys-items": { + "PhysIf-list": [ + { + "FECMode": "auto", + "accessVlan": "unknown", + "adminSt": "up", + "descr": "Uplink in TENANT-1 VRF", + "id": "eth1/10", + "layer": "Layer3", + "medium": "broadcast", + "mode": "access", + "mtu": 9000, + "nativeVlan": "unknown", + "physExtd-items": { + "bufferBoost": "enable" + }, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='TENANT-1']" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state" + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "TENANT-1", + "if-items": { + "If-list": [ + { + "id": "eth1/10", + "addr-items": { + "Addr-list": [ + { + "addr": "10.1.1.1/30", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/openconfig/interface.txt b/test/e2e/testdata/openconfig/interface.txt new file mode 100644 index 000000000..f6055ec7e --- /dev/null +++ b/test/e2e/testdata/openconfig/interface.txt @@ -0,0 +1,61 @@ +# Loopback Interface +-- interfaces/lo0 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: lo0 + namespace: default + labels: + networking.metal.ironcore.dev/device-name: device +spec: + deviceRef: + name: device + name: lo0 + description: Router-ID Leaf1 + adminState: Up + type: Loopback + mtu: 1500 + ipv4: + addresses: + - 10.0.0.10/32 +-- state/expect -- +{ + "interfaces": { + "interface": [ + { + "config": { + "description": "Router-ID Leaf1", + "enabled": true, + "mtu": 1500, + "name": "lo0", + "type": "softwareLoopback" + }, + "name": "lo0", + "subinterfaces": { + "subinterface": [ + { + "config": { + "index": 0 + }, + "index": 0, + "ipv4": { + "addresses": { + "address": [ + { + "config": { + "ip": "10.0.0.10", + "prefix-length": 32, + "type": "PRIMARY" + }, + "ip": "10.0.0.10" + } + ] + } + } + } + ] + } + } + ] + } +} From 096badfdb0c62307bf8ad21bc105eb1ecb0921a6 Mon Sep 17 00:00:00 2001 From: Pujol Date: Thu, 18 Jun 2026 02:01:30 +0200 Subject: [PATCH 08/10] fix: prevent stdin inheritance in e2e test commands Commands executed via exec.CommandContext with CombinedOutput can hang when stdin is not explicitly set to nil. This was causing UndeployManager to block indefinitely during test cleanup. Changes: - Set cmd.Stdin = nil on all exec.Command calls to prevent stdin inheritance - Change UndeployManager to use kubectl delete -k directly instead of make undeploy to avoid shell pipeline stdin issues Signed-off-by: Pujol --- test/e2e/testutil/cluster.go | 12 +++++++----- test/e2e/testutil/helpers.go | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/e2e/testutil/cluster.go b/test/e2e/testutil/cluster.go index f34990ec9..bd22fad9b 100644 --- a/test/e2e/testutil/cluster.go +++ b/test/e2e/testutil/cluster.go @@ -92,6 +92,7 @@ func (c *ClusterEnvironment) DeployManager(ctx context.Context) error { cmd := exec.CommandContext(ctx, "make", "deploy-crds") cmd.Dir = dir cmd.Env = env + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to deploy CRDs: %s: %w", string(output), err) } @@ -100,6 +101,7 @@ func (c *ClusterEnvironment) DeployManager(ctx context.Context) error { cmd = exec.CommandContext(ctx, "make", "deploy") cmd.Dir = dir cmd.Env = env + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to deploy manager: %s: %w", string(output), err) } @@ -114,12 +116,10 @@ func (c *ClusterEnvironment) DeployManager(ctx context.Context) error { } // UndeployManager undeploys the controller-manager. +// Uses kubectl delete -k directly instead of make undeploy to avoid shell pipeline stdin issues. func (c *ClusterEnvironment) UndeployManager(ctx context.Context) error { - cmd := exec.CommandContext(ctx, "make", "undeploy") - dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback - cmd.Dir = dir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to undeploy manager: %s: %w", string(output), err) + if err := c.runKubectl(ctx, "delete", "-k", "config/develop", "--ignore-not-found"); err != nil { + return fmt.Errorf("failed to undeploy manager: %w", err) } return nil } @@ -228,6 +228,7 @@ func (c *ClusterEnvironment) runKubectl(ctx context.Context, args ...string) err cmd := exec.CommandContext(ctx, "kubectl", args...) dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback cmd.Dir = dir + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("%s: %w", string(output), err) @@ -240,6 +241,7 @@ func (c *ClusterEnvironment) runKubectlOutput(ctx context.Context, args ...strin cmd := exec.CommandContext(ctx, "kubectl", args...) dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback cmd.Dir = dir + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr diff --git a/test/e2e/testutil/helpers.go b/test/e2e/testutil/helpers.go index 5d38e28fb..b9888543d 100644 --- a/test/e2e/testutil/helpers.go +++ b/test/e2e/testutil/helpers.go @@ -42,6 +42,7 @@ func Run(cmd *exec.Cmd, w io.Writer) (string, error) { } cmd.Dir = dir + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs if err = os.Chdir(cmd.Dir); err != nil { _, _ = fmt.Fprintf(w, "chdir dir: %s\n", err) } From 315adf2ba01c6d32e82eb881f13d31b9aab84136 Mon Sep 17 00:00:00 2001 From: Pujol Date: Thu, 18 Jun 2026 02:01:34 +0200 Subject: [PATCH 09/10] docs: add AGENTS.md for e2e test documentation Documents the e2e test infrastructure including: - Envtest vs Cluster mode comparison - Test commands and environment variables - Testdata format and test flow - Architecture and gnmi-test-server behavior Signed-off-by: Pujol --- test/e2e/AGENTS.md | 146 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 test/e2e/AGENTS.md diff --git a/test/e2e/AGENTS.md b/test/e2e/AGENTS.md new file mode 100644 index 000000000..86c917bf4 --- /dev/null +++ b/test/e2e/AGENTS.md @@ -0,0 +1,146 @@ +# E2E Tests + +Integration tests that validate the full reconciliation pipeline from Kubernetes CRD to gNMI JSON output. + +## Rationale + +Unit tests verify individual functions but miss the gaps between layers: CRD validation, controller logic, provider field mapping, and gNMI path construction. For example, a controller passing the wrong value to a provider. + +Two test modes via Go build tags (`//go:build envtest` vs `//go:build !envtest`): + +- **Envtest mode** — runs tests sequantially without infrastructure using envtest + in-process gnmi-test-server. Tests complete in seconds. +- **Cluster mode** — runs tests in parallel against Kind with full operator installation. Validates RBAC, webhooks, metrics. +Each reconciliation test is isolated: unique namespace + dedicated gnmi-test-server pod. + +## Commands + +```bash +make test-e2e-envtest PROVIDER=cisco-nxos-gnmi # no cluster required +make test-e2e-kind PROVIDER=cisco-nxos-gnmi # requires Kind cluster "network" +``` + +**Provider options:** `openconfig`, `cisco-nxos-gnmi` + +## Mode Comparison + +| Aspect | Envtest | Cluster | +|--------|---------|---------| +| **K8s API** | In-process (envtest) | Kind cluster | +| **gNMI server** | In-process (gnmi-test-server) | Deployed pod per test | +| **Controllers** | In-process per test | Deployed operator pod | +| **Speed** | ~10s | ~3-4min (parallel) | +| **Dependencies** | None | Docker + Kind | + +| Coverage | Envtest | Cluster | +|----------|:-------:|:-------:| +| Controller reconciliation | ✅ | ✅ | +| Status conditions | ✅ | ✅ | +| gNMI payload generation | ✅ | ✅ | +| RBAC / ServiceAccount | ❌ | ✅ | +| Webhook TLS + cert-manager | ❌ | ✅ | +| Container image build | ❌ | ✅ | +| Metrics endpoint | ❌ | ✅ | + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `PROVIDER` / `E2E_PROVIDER` | Filter tests to specific provider | +| `PROMETHEUS_INSTALL_SKIP=true` | Skip Prometheus installation (cluster mode) | +| `CERT_MANAGER_INSTALL_SKIP=true` | Skip CertManager installation (cluster mode) | + +## Architecture + +Both modes implement `TestEnvironment` interface via `testutil/`: + +| Method | Purpose | +|--------|---------| +| `Setup()` / `Teardown()` | Initialize/cleanup K8s + gNMI server | +| `Client()` / `RESTConfig()` | K8s client for resources and managers | +| `GNMIAddress()` | Endpoint for Device CRDs | +| `GetGNMIState()` / `ClearGNMIState()` | Verify and reset gNMI state | +| `PreloadGNMIState()` | Set initial state before reconciliation | +| `IsEnvtest()` | Detect mode for conditional logic | + +### GNMI test server + +The server accumulates gNMI Set operations and exposes state via `GetState()`. + +When configured with `WithNXOSBehavior()`: +- Strips fields with value `"DME_UNSET_PROPERTY_MARKER"` when storing (the marker means "unset this field", not "store this literal string") +- Returns empty TypedValue for non-existent paths (instead of NOT_FOUND error), matching real NX-OS behavior + + +## Testdata Format + +Location: `test/e2e/testdata//.txt` — auto-discovered, txtar format. + +Each test file contains K8s YAML resources and expected gNMI JSON state: + +``` +-- state/preload -- # OPTIONAL: initial gNMI state +{"System": {"procsys-items": {"bootTime": "1700000000"}}} + +-- / -- # Resource to create +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +spec: + deviceRef: + name: device # substituted at runtime + ... + +-- / -- # Multiple resources supported +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGPPeer +spec: + deviceRef: + name: device + ... + +-- state/expect -- # Expected gNMI JSON (array order ignored) +{"System": {"intf-items": ...}} +``` + +### Test Flow + +1. Parse txtar file → extract resources + expected state +2. Create test namespace +3. Deploy/connect to gnmi-test-server +4. Preload gNMI state if `state/preload` section exists +5. Create Device pointing to gnmi-test-server address +6. Apply resources **sequentially**, waiting for `Configured` condition before next +7. Compare final gNMI state vs `state/expect` using semantic JSON comparison + +### Important Notes + +> **gNMI State Differences:** The JSON stored in gnmi-test-server may differ from actual provider output. For example, NXOS `DME_UNSET_PROPERTY_MARKER` values are filtered out by the test server. + +> **Resource Dependencies:** Resources are created in file order. Use ordering to handle dependencies (e.g., BGPPeer after BGP, RoutingPolicy after PrefixSet). + +## Existing Test Files + +``` +test/e2e/testdata/cisco-nxos-gnmi/ +├── acl.txt +├── banner.txt +├── bgp_bgppeer.txt +├── dhcprelay.txt +├── dns.txt +├── evpninstance.txt +├── interfaceconfig.txt +├── interfaces.txt +├── isis.txt +├── lldp.txt +├── managementaccess.txt +├── ntp.txt +├── nve.txt +├── ospf.txt +├── pim.txt +├── routedvlan.txt +├── routingpolicy_prefixset.txt +├── snmp.txt +├── subinterface.txt +├── syslog.txt +├── vpcdomain.txt +└── vrf.txt +``` From 37fea8a4b60a115c60ad4740b38b85331bef4bd9 Mon Sep 17 00:00:00 2001 From: Pujol Date: Thu, 18 Jun 2026 02:04:15 +0200 Subject: [PATCH 10/10] ci: add separate e2e test workflows for envtest and cluster modes - test-e2e-envtest.yml: Fast in-process tests, no cluster required - test-e2e-cluster.yml: Full cluster tests with Kind, 8 parallel processes Signed-off-by: Pujol --- .github/workflows/test-e2e-cluster.yml | 32 ++++++++++++++++++++++++++ .github/workflows/test-e2e-envtest.yml | 26 +++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 .github/workflows/test-e2e-cluster.yml create mode 100644 .github/workflows/test-e2e-envtest.yml diff --git a/.github/workflows/test-e2e-cluster.yml b/.github/workflows/test-e2e-cluster.yml new file mode 100644 index 000000000..60fe00fe5 --- /dev/null +++ b/.github/workflows/test-e2e-cluster.yml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +# SPDX-License-Identifier: Apache-2.0 + +name: Test E2E (Cluster) + +on: + pull_request: + branches: + - main + paths-ignore: + - 'docs/**' + - '**/*.md' + +jobs: + test-e2e-cluster: + name: Run E2E Tests (Cluster) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + version: v0.31.0 + kubectl_version: v1.35.0 + cluster_name: network + - name: Running E2E Cluster Tests + run: | + go mod download + make test-e2e-cluster GINKGO_PROCS=8 diff --git a/.github/workflows/test-e2e-envtest.yml b/.github/workflows/test-e2e-envtest.yml new file mode 100644 index 000000000..550de5f88 --- /dev/null +++ b/.github/workflows/test-e2e-envtest.yml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +# SPDX-License-Identifier: Apache-2.0 + +name: Test E2E (Envtest) + +on: + pull_request: + branches: + - main + paths-ignore: + - 'docs/**' + - '**/*.md' + +jobs: + test-e2e-envtest: + name: Run E2E Tests (Envtest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + - name: Running E2E Envtest Tests + run: | + go mod download + make test-e2e-envtest