Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -92,5 +92,18 @@ jobs:
go-version: 1.26.3
- name: fv-pullmode
run: make create-cluster-pullmode fv-pullmode
env:
FV: true
FV-NAMESPACE:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: 1.26.3
- name: fv
run: make create-cluster-infra fv-namespace
env:
FV: true
66 changes: 48 additions & 18 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ ARCH ?= $(shell go env GOARCH)
OS ?= $(shell uname -s | tr A-Z a-z)
K8S_LATEST_VER ?= $(shell curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)
export CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME)
TAG ?= v1.10.0
TAG ?= main

## Tool Binaries
CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen
Expand Down Expand Up @@ -186,6 +186,9 @@ NUM_NODES ?= 5
.PHONY: kind-test
kind-test: test create-cluster fv ## Build docker image; start kind cluster; load docker image; install all cluster api components and run fv

.PHONY: kind-test-namespace
kind-test-namespace: test create-cluster-infra fv-namespace ## Build docker image; start kind cluster; deploy Sveltos in a random namespace and run fv

.PHONY: fv
fv: $(GINKGO) ## Run Sveltos Controller tests using existing cluster
cd test/fv; $(GINKGO) -nodes $(NUM_NODES) --label-filter='FV' --v --trace --randomize-all
Expand All @@ -212,13 +215,44 @@ fv-agentless: $(KUBECTL) $(GINKGO) ## Run Sveltos Controller tests using existin
fv-pullmode: $(GINKGO) ## Run Sveltos Controller tests using existing cluster
cd test/fv; $(GINKGO) -nodes $(NUM_NODES) --label-filter='PULLMODE' --v --trace --randomize-all

.PHONY: deploy-crds
deploy-crds: $(KUBECTL) ## Install all required Sveltos CRDs
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_debuggingconfigurations.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_classifiers.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_classifierreports.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_accessrequests.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_rolerequests.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_sveltosclusters.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_configurationgroups.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_configurationbundles.lib.projectsveltos.io.yaml

.PHONY: fv-namespace
fv-namespace: $(GINKGO) $(KUBECTL) $(KUSTOMIZE) $(ENVSUBST) ## Run FV tests with Sveltos deployed in a random namespace
$(MAKE) load-image
$(MAKE) deploy-crds

@echo "Deploying access manager"
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/access-manager/$(TAG)/manifest/manifest.yaml
@echo "wait for cluster to be provisioned"
$(KUBECTL) wait cluster $(WORKLOAD_CLUSTER_NAME) -n default --for=jsonpath='{.status.phase}'=Provisioned --timeout=$(TIMEOUT)

cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
SVELTOS_NS="sveltos-$$(openssl rand -hex 4)"; \
echo "Deploying Sveltos classifier in namespace: $$SVELTOS_NS"; \
$(KUSTOMIZE) build config/default | $(ENVSUBST) | \
sed -E 's/^([[:space:]]+)(name|namespace): projectsveltos$$/\1\2: '"$$SVELTOS_NS"'/' | \
$(KUBECTL) apply -f-; \
echo "Waiting for classifier-manager to be available in namespace $$SVELTOS_NS..."; \
$(KUBECTL) wait --for=condition=Available deployment/classifier-manager -n "$$SVELTOS_NS" --timeout=$(TIMEOUT); \
cd test/fv && SVELTOS_NAMESPACE="$$SVELTOS_NS" $(GINKGO) -nodes $(NUM_NODES) --label-filter='FV' --v --trace --randomize-all


.PHONY: test
test: manifests generate fmt vet $(SETUP_ENVTEST) ## Run uts.
KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test $(shell go list ./... |grep -v test/fv |grep -v test/helpers) $(TEST_ARGS) -coverprofile cover.out

.PHONY: create-cluster
create-cluster: $(KIND) $(CLUSTERCTL) $(KUBECTL) $(ENVSUBST) ## Create a new kind cluster designed for development
.PHONY: create-cluster-infra
create-cluster-infra: $(KIND) $(CLUSTERCTL) $(KUBECTL) $(ENVSUBST) ## Create kind cluster and workload cluster without deploying Sveltos
$(MAKE) create-control-cluster

@echo "sleep allowing webhook to be ready"
Expand All @@ -227,17 +261,12 @@ create-cluster: $(KIND) $(CLUSTERCTL) $(KUBECTL) $(ENVSUBST) ## Create a new kin
@echo "Create a workload cluster"
$(KUBECTL) apply -f $(KIND_CLUSTER_YAML)

@echo "Start projectsveltos classifier"
$(MAKE) deploy-projectsveltos
@echo "sleep allowing control plane to be ready"
sleep 60

@echo "Deploying access manager"
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/access-manager/$(TAG)/manifest/manifest.yaml
@echo "wait for cluster to be provisioned"
$(KUBECTL) wait cluster $(WORKLOAD_CLUSTER_NAME) -n default --for=jsonpath='{.status.phase}'=Provisioned --timeout=$(TIMEOUT)

@echo "sleep allowing control plane to be ready"
sleep 60

@echo "get kubeconfig to access workload cluster"
$(KIND) get kubeconfig --name $(WORKLOAD_CLUSTER_NAME) > test/fv/workload_kubeconfig

Expand All @@ -247,6 +276,11 @@ create-cluster: $(KIND) $(CLUSTERCTL) $(KUBECTL) $(ENVSUBST) ## Create a new kin
@echo wait for calico pod
$(KUBECTL) --kubeconfig=./test/fv/workload_kubeconfig wait --for=condition=Available deployment/calico-kube-controllers -n kube-system --timeout=$(TIMEOUT)

.PHONY: create-cluster
create-cluster: create-cluster-infra ## Create a new kind cluster designed for development
@echo "Start projectsveltos classifier"
$(MAKE) deploy-projectsveltos

create-cluster-pullmode: $(KIND) $(KUBECTL) $(ENVSUBST) $(KUSTOMIZE)
docker network rm $(SVELTOS_NETWORK_NAME) 2>/dev/null || true
docker network create $(SVELTOS_NETWORK_NAME)
Expand Down Expand Up @@ -406,14 +440,10 @@ deploy-projectsveltos: $(KUSTOMIZE)
$(MAKE) load-image

@echo 'Install libsveltos CRDs'
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_debuggingconfigurations.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_classifiers.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_classifierreports.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_accessrequests.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_rolerequests.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_sveltosclusters.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_configurationgroups.lib.projectsveltos.io.yaml
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_configurationbundles.lib.projectsveltos.io.yaml
$(MAKE) deploy-crds

@echo "Deploying access manager"
$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/access-manager/$(TAG)/manifest/manifest.yaml

# Install projectsveltos classifier components
@echo 'Install projectsveltos classifier components'
Expand Down
15 changes: 14 additions & 1 deletion config/default/manager_auth_proxy_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,19 @@ spec:
- "--shard-key="
- --capi-onboard-annotation=
- "--v=5"
- "--version=v1.10.0"
- "--version=main"
- "--registry="
- "--agent-in-mgmt-cluster=false"
env:
- name: TOTAL_MEMORY_LIMIT
valueFrom:
resourceFieldRef:
resource: limits.memory
- name: GOMAXPROCS
valueFrom:
resourceFieldRef:
resource: limits.cpu
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
2 changes: 1 addition & 1 deletion config/default/manager_image_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ spec:
spec:
containers:
# Change the value of image field below to your controller image URL
- image: docker.io/projectsveltos/classifier:v1.10.0
- image: docker.io/projectsveltos/classifier:main
name: manager
2 changes: 0 additions & 2 deletions controllers/classifier_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,6 @@ const (

controlplaneendpoint = "controlplaneendpoint-key"
configurationHash = "configurationHash"

projectsveltos = "projectsveltos"
)

// ClassifierReconciler reconciles a Classifier object
Expand Down
73 changes: 61 additions & 12 deletions controllers/classifier_deployer.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ const (
sveltosApplierOverrideAnnotation = "sveltosapplier.projectsveltos.io/config-override-ref"
)

func getSveltosAgentNamespace() string {
return projectsveltos
func getSveltosAgentNamespace(sveltosNamespace string) string {
return sveltosNamespace
}

func (r *ClassifierReconciler) deployClassifier(ctx context.Context, classifierScope *scope.ClassifierScope,
Expand Down Expand Up @@ -374,10 +374,10 @@ func getKubeconfigFromAccessRequest(ctx context.Context, c client.Client, cluste

func createSecretNamespace(ctx context.Context, c client.Client) error {
ns := &corev1.Namespace{}
err := c.Get(ctx, types.NamespacedName{Name: libsveltosv1beta1.ClassifierSecretNamespace}, ns)
err := c.Get(ctx, types.NamespacedName{Name: getSveltosNamespace()}, ns)
if err != nil {
if apierrors.IsNotFound(err) {
ns.Name = libsveltosv1beta1.ClassifierSecretNamespace
ns.Name = getSveltosNamespace()
return c.Create(ctx, ns)
}
return err
Expand Down Expand Up @@ -406,14 +406,14 @@ func updateSecretWithAccessManagementKubeconfig(ctx context.Context, c client.Cl

secret := &corev1.Secret{}
key := client.ObjectKey{
Namespace: libsveltosv1beta1.ClassifierSecretNamespace,
Namespace: getSveltosNamespace(),
Name: libsveltosv1beta1.ClassifierSecretName,
}

dataKey := "kubeconfig"
err = remoteClient.Get(ctx, key, secret)
if err != nil {
secret.Namespace = libsveltosv1beta1.ClassifierSecretNamespace
secret.Namespace = getSveltosNamespace()
secret.Name = libsveltosv1beta1.ClassifierSecretName
secret.Data = map[string][]byte{
dataKey: kubeconfig,
Expand Down Expand Up @@ -1553,7 +1553,7 @@ func createSveltosAgentNamespaceInManagedCluster(ctx context.Context, c client.C

ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: getSveltosAgentNamespace(),
Name: getSveltosAgentNamespace(getSveltosNamespace()),
},
}

Expand Down Expand Up @@ -1671,6 +1671,40 @@ func deploySveltosAgentInManagementCluster(ctx context.Context, restConfig *rest
restConfig, agentYAML, lbls, patches, false, logger)
}

// updateResourceNamespace sets the namespace on a resource that requires it.
// For namespaced resources the object metadata namespace is updated.
// For ClusterRoleBinding the subjects are also patched: the resource is cluster-scoped so
// GetNamespace() returns "" and the plain SetNamespace call would not reach it, but each
// ServiceAccount subject still carries an explicit namespace that must match the actual
// location of the ServiceAccount.
func updateResourceNamespace(policy *unstructured.Unstructured, namespace string) error {
if policy.GetNamespace() != "" {
policy.SetNamespace(namespace)
}

if policy.GetKind() != "ClusterRoleBinding" {
return nil
}

subjects, found, err := unstructured.NestedSlice(policy.Object, "subjects")
if err != nil || !found {
return err
}

for i := range subjects {
subject, ok := subjects[i].(map[string]interface{})
if !ok {
continue
}
if subject["kind"] == "ServiceAccount" {
subject["namespace"] = namespace
subjects[i] = subject
}
}

return unstructured.SetNestedSlice(policy.Object, subjects, "subjects")
}

func deploySveltosAgentResources(ctx context.Context, clusterNamespace, clusterName, classifierName string,
restConfig *rest.Config, agentYAML string, lbls map[string]string, patches []libsveltosv1beta1.Patch,
isPullMode bool, logger logr.Logger) error {
Expand Down Expand Up @@ -1710,6 +1744,11 @@ func deploySveltosAgentResources(ctx context.Context, clusterNamespace, clusterN
}
}

if err := updateResourceNamespace(policy, getSveltosNamespace()); err != nil {
logger.V(logs.LogInfo).Error(err, "failed to update resource namespace")
return err
}

var referencedUnstructured []*unstructured.Unstructured
if len(patches) > 0 {
logger.V(logs.LogInfo).Info("Patch sveltos-agent resources")
Expand Down Expand Up @@ -1765,6 +1804,11 @@ func deploySveltosApplierResources(ctx context.Context, clusterNamespace, cluste
return err
}

if err := updateResourceNamespace(policy, getSveltosNamespace()); err != nil {
logger.V(logs.LogInfo).Error(err, "failed to update resource namespace")
return err
}

var referencedUnstructured []*unstructured.Unstructured
if len(patches) > 0 {
logger.V(logs.LogDebug).Info("Patch sveltos-applier resources")
Expand Down Expand Up @@ -1851,7 +1895,8 @@ func getSveltosAgentDeploymentName(ctx context.Context, restConfig *rest.Config,
}

// using client and a List would require permission at cluster level. So using clientset instead
deployments, err := clientset.AppsV1().Deployments(getSveltosAgentNamespace()).List(ctx, listOptions)
deployments, err := clientset.AppsV1().Deployments(getSveltosAgentNamespace(getSveltosNamespace())).
List(ctx, listOptions)
if err != nil {
return "", err
}
Expand Down Expand Up @@ -1930,6 +1975,10 @@ func removeSveltosAgentFromManagementCluster(ctx context.Context,
return err
}

if policy.GetNamespace() != "" {
policy.SetNamespace(getSveltosNamespace())
}

dr, err := k8s_utils.GetDynamicResourceInterface(restConfig, policy.GroupVersionKind(), policy.GetNamespace())
if err != nil {
logger.V(logs.LogInfo).Error(err, "failed to get dynamic client")
Expand Down Expand Up @@ -2028,7 +2077,7 @@ func getSveltosAgentPatchesOld(ctx context.Context, c client.Client,
configMap := &corev1.ConfigMap{}
if configMapName != "" {
err := c.Get(ctx,
types.NamespacedName{Namespace: projectsveltos, Name: configMapName},
types.NamespacedName{Namespace: getSveltosNamespace(), Name: configMapName},
configMap)
if err != nil {
logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get ConfigMap %s: %v",
Expand Down Expand Up @@ -2060,7 +2109,7 @@ func getSveltosApplierPatchesOld(ctx context.Context, c client.Client,
configMap := &corev1.ConfigMap{}
if configMapName != "" {
err := c.Get(ctx,
types.NamespacedName{Namespace: projectsveltos, Name: configMapName},
types.NamespacedName{Namespace: getSveltosNamespace(), Name: configMapName},
configMap)
if err != nil {
logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get ConfigMap %s: %v",
Expand Down Expand Up @@ -2098,7 +2147,7 @@ func getSveltosAgentPatches(ctx context.Context, c client.Client,
}

configMapName := getSveltosAgentConfigMap()
patches, err = getSveltosAgentPatchesNew(ctx, c, projectsveltos, configMapName, logger)
patches, err = getSveltosAgentPatchesNew(ctx, c, getSveltosNamespace(), configMapName, logger)
if err == nil {
return patches, nil
}
Expand All @@ -2120,7 +2169,7 @@ func getSveltosApplierPatches(ctx context.Context, c client.Client,
}

configMapName := getSveltosApplierConfigMap()
patches, err = getSveltosApplierPatchesNew(ctx, c, projectsveltos, configMapName, logger)
patches, err = getSveltosApplierPatchesNew(ctx, c, getSveltosNamespace(), configMapName, logger)
if err == nil {
return patches, nil
}
Expand Down
Loading