re-work eks multicluster target

assets/eks-multicloud-target/kubeconfig.sh

@@ -0,0 +1,34 @@
+SERVICE_ACCOUNT=pxmigration
+NAMESPACE=kube-system
+SERVER=$1
+
+SERVICE_ACCOUNT_TOKEN_COUNT=$(kubectl -n ${NAMESPACE} get secret -o=jsonpath='{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="'${SERVICE_ACCOUNT}'")].metadata.name}' | wc -w)
+if [ ${SERVICE_ACCOUNT_TOKEN_COUNT} -gt 1 ]
+then
+  SERVICE_ACCOUNT_TOKEN_NAME=$(kubectl -n ${NAMESPACE} get secret -o=jsonpath='{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="'${SERVICE_ACCOUNT}'")].metadata.name}' | awk '{for(i=1;i<=NF;i++){ if($i ~ /-token/){print $i} } }' | head -n 1)
+else
+  SERVICE_ACCOUNT_TOKEN_NAME=$(kubectl -n ${NAMESPACE} get secret -o=jsonpath='{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="'${SERVICE_ACCOUNT}'")].metadata.name}')
+fi
+SERVICE_ACCOUNT_TOKEN=$(kubectl -n ${NAMESPACE} get secret ${SERVICE_ACCOUNT_TOKEN_NAME} -o "jsonpath={.data.token}" | base64 --decode)
+SERVICE_ACCOUNT_CERTIFICATE=$(kubectl -n ${NAMESPACE} get secret ${SERVICE_ACCOUNT_TOKEN_NAME} -o "jsonpath={.data['ca\.crt']}")
+
+cat <<END
+apiVersion: v1
+kind: Config
+clusters:
+- name: default-cluster
+  cluster:
+    certificate-authority-data: ${SERVICE_ACCOUNT_CERTIFICATE}
+    server: ${SERVER}
+contexts:
+- name: default-context
+  context:
+    cluster: default-cluster
+    namespace: ${NAMESPACE}
+    user: ${SERVICE_ACCOUNT}
+current-context: default-context
+users:
+- name: ${SERVICE_ACCOUNT}
+  user:
+    token: ${SERVICE_ACCOUNT_TOKEN}
+END

assets/eks-multicloud-target/serviceaccount.yaml

@@ -0,0 +1,27 @@
+ apiVersion: v1
+ kind: ServiceAccount
+ metadata:
+   name: pxmigration
+   namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pxmigration-clusterrolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: pxmigration
+  namespace: kube-system
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: pxmigration
+  namespace: kube-system
+  annotations: 
+    kubernetes.io/service-account.name: pxmigration
+type: kubernetes.io/service-account-token

defaults.yml

@@ -13,14 +13,14 @@ aws_type: t3.large
 aws_ebs: "gp2:50"
 aws_access_key_id: ""
 aws_secret_access_key: "" 
-eks_version: "1.34"
+eks_version: "1.35"
 #aws_ebs: "gp2:20 standard:30"
 
 gcp_region: europe-north1
 gcp_zone: b
 gcp_type: n1-standard-4
 gcp_disks: "pd-standard:50"
-gke_version: "1.34"
+gke_version: "1.35"
 #gcp_disks: "pd-standard:20 pd-ssd:30"
 #gcp_project: "px-deploy"
 
@@ -33,7 +33,7 @@ azure_client_secret: ""
 azure_client_id: ""
 azure_tenant_id: ""
 azure_subscription_id: ""
-aks_version: "1.34"
+aks_version: "1.35"
 
 rancher_version: "2.10.1"
 rancher_k3s_version: "1.30.8+k3s1"

infra/eks-master

@@ -2,7 +2,7 @@ cat <<EOF >>/etc/bashrc
 PS1='[\u@bastion-'$cluster' \W]\$ '
 EOF
 
-curl -sL -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl
+curl -sL -o /usr/bin/kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
 chmod 755 /usr/bin/kubectl
 
 while ! dnf install -y docker; do

scripts/eks-multicloud-target

@@ -1,6 +1,15 @@
+# generate serviceaccount for migration
+mkdir -p /root/drscripts
 
+kubectl apply -f /assets/eks-multicloud-target/serviceaccount.yaml
+clusterapi=$(aws eks describe-cluster --name px-deploy-$name-$cluster | jq -r '.cluster.endpoint')
+bash /assets/eks-multicloud-target/kubeconfig.sh $clusterapi > /root/drscripts/migration_dest_kubeconfig
+
+migration_dest_kubeconfig=$(cat /root/drscripts/migration_dest_kubeconfig | base64 -w0)
+
+storagecluster=$(kubectl get stc -n portworx -o jsonpath='{.items[].metadata.name}')
 kubectl create secret  generic --from-file=/root/.aws/credentials -n portworx aws-creds
-kubectl patch stc/px-deploy-$cluster  --type merge  -n portworx -p '{"spec": {
+kubectl patch stc/$storagecluster  --type merge  -n portworx -p '{"spec": {
   "stork": {
       "volumes": [
        {
@@ -15,60 +24,43 @@ kubectl patch stc/px-deploy-$cluster  --type merge  -n portworx -p '{"spec": {
      }
 }}'
 
-kubectl annotate stc px-deploy-$cluster -n portworx portworx.io/service-type="LoadBalancer" --overwrite
+kubectl annotate stc $storagecluster -n portworx portworx.io/service-type="LoadBalancer" --overwrite
 
-while : ; do
-  token=$(kubectl exec -n portworx -it $(kubectl get pods -n portworx -lname=portworx --field-selector=status.phase=Running | tail -1 | cut -f 1 -d " ") -- /opt/pwx/bin/pxctl cluster token show 2>/dev/null | cut -f 3 -d " ")
-  echo $token | grep -Eq '\w{128}'
-  [ $? -eq 0 ] && break
-  sleep 5
-  echo waiting for portworx
-done
-
-UUID=$(kubectl get stc -n portworx -o jsonpath='{.items[].status.clusterUid}')
 S3_ACCESS_KEY=$(sed -n 's/aws_access_key_id[ =]*//p' /root/.aws/credentials 2>/dev/null | head -1)
 S3_SECRET_KEY=$(sed -n 's/aws_secret_access_key[ =]*//p' /root/.aws/credentials 2>/dev/null | head -1)
 
 S3_BUCKET_REGION=$(aws s3api get-bucket-location --bucket $DR_BUCKET --output text)
 # Region us-east-1 returns "None" instead of the region name
 if [ "$S3_BUCKET_REGION" = "None" ]; then
-  BUCKET_REGION="us-east-1"
+  S3_BUCKET_REGION="us-east-1"
 fi
 echo "Bucket region: $S3_BUCKET_REGION"
 
-
 while : ;do
     host=$(kubectl get svc -n portworx portworx-service -o jsonpath='{.status.loadBalancer.ingress[].hostname}')
     [ "$host" ] && break
-    sleep 1
+    echo "waiting for px service LB hostname"
+    sleep 5
 done
 
-mkdir /root/drscripts
-
-PX_POD=$(kubectl get pods -l name=portworx -n portworx -o jsonpath='{.items[0].metadata.name}')
-
-# run dummy "pxctl credentials list" to get driver ready
-CRED_CMD="pxctl credentials list"
-kubectl exec $PX_POD -n portworx -- /opt/pwx/bin/$CRED_CMD
-
-CRED_CMD="pxctl credentials create --provider s3 --s3-access-key $S3_ACCESS_KEY --s3-secret-key $S3_SECRET_KEY --s3-region $S3_BUCKET_REGION --s3-endpoint s3.$S3_BUCKET_REGION.amazonaws.com --s3-storage-class STANDARD --bucket $DR_BUCKET clusterPair_$UUID"
-
-PX_POD=$(kubectl get pods -l name=portworx -n portworx -o jsonpath='{.items[0].metadata.name}')
-kubectl exec $PX_POD -n portworx -- /opt/pwx/bin/$CRED_CMD
-
-
-storkctl generate clusterpair -n kube-system remotecluster | sed "/insert_storage_options_here/c\    ip: $host\n    token: $token\n" >/root/drscripts/cp.yaml
-
 # create preparation script for source cluster
-cat <<EOF >>/root/drscripts/prepare_migrate_dr_source.sh
+cat <<EOF >/root/drscripts/prepare_migrate_dr_source.sh
+echo "writing dest kubeconfig"
+echo $migration_dest_kubeconfig | base64 -d > /tmp/migration_dest_kubeconfig
 
+echo "writing src kubeconfig"
+kubectl apply -f /assets/eks-multicloud-target/serviceaccount.yaml
+clusterapi=\$(cat /root/.kube/config  | grep server | cut -d: -f2-)
+bash /assets/eks-multicloud-target/kubeconfig.sh \$clusterapi > /tmp/migration_src_kubeconfig
+
+echo "creating PX AWS stork credentials"
 echo "[default]" > ./credentials
 echo "aws_access_key_id = $S3_ACCESS_KEY" >> ./credentials
 echo "aws_secret_access_key = $S3_SECRET_KEY" >> ./credentials
-
-kubectl pxc $CRED_CMD
 kubectl create secret  generic --from-file=./credentials -n portworx aws-creds
-kubectl patch stc/px-deploy-1  --type merge  -n portworx -p '{"spec": {
+storagecluster=\$(kubectl get stc -n portworx -o jsonpath='{.items[].metadata.name}')
+kubectl annotate stc \$storagecluster -n portworx portworx.io/service-type="LoadBalancer" --overwrite
+kubectl patch stc/\$storagecluster --type merge  -n portworx -p '{"spec": {
   "stork": {
       "volumes": [
        {
@@ -83,7 +75,16 @@ kubectl patch stc/px-deploy-1  --type merge  -n portworx -p '{"spec": {
      }
 }}'
 
-kubectl apply -f ./cp.yaml
+while : ;do
+    host=\$(kubectl get svc -n portworx portworx-service -o jsonpath='{.status.loadBalancer.ingress[].ip}')
+    [ "\$host" ] && break
+    echo "waiting for px service LB IP"
+    sleep 5
+done
+
+echo "creating clusterpair"
+storkctl create clusterpair remotecluster -n kube-system --mode migration --bucket $DR_BUCKET --provider s3 --s3-access-key $S3_ACCESS_KEY --s3-secret-key $S3_SECRET_KEY --s3-endpoint s3.amazonaws.com --s3-region $S3_BUCKET_REGION --dest-kube-file /tmp/migration_dest_kubeconfig --src-kube-file /tmp/migration_src_kubeconfig
+
 EOF
 
 chmod +x /root/drscripts/prepare_migrate_dr_source.sh
@@ -93,8 +94,8 @@ cat <<EOF >> /etc/motd
 +================================================+
 Howto setup multi cloud Migrate/Async DR Source
 +================================================+
-- Copy content of /root/drscripts to source master and execute
-- on target system you could use /assets/app-migration.yml to setup migration
+- Copy content of /root/drscripts to source master and execute to create clusterpair
+- on target system you can use /assets/app-migration.yml to setup migration
 +================================================+
 EOF
 

templates/eks-multicloud-target.yml

@@ -3,11 +3,12 @@
 #
 # Requires:
 # working settings for Cloud AWS
-# env variable DR_BUCKET in defaults.yml pointing to existing AWS S3 Bucket
+# env variable DR_BUCKET in defaults.yml pointing to existing AWS S3 Bucket accessible with the provided aws credentials 
 #
-# creates directory /root/drscripts containing clusterpair spec and setup script for source cluster
+# creates directory /root/drscripts setup script for source cluster
 # when 'px-deploy status' is ready copy master node directory /root/drscripts to source px cluster and execute setup script
-# clusterpair should then be ready on source cluster
+# clusterpair should then be ready on source & destination cluster
+# validated with aks source cluster
 #
 # Maintainer: Daniel Paul
 #

terraform/aws/main.tf

@@ -306,6 +306,18 @@ resource "aws_iam_policy" "px-policy" {
           "autoscaling:DescribeAutoScalingGroups"
         ]
         Resource = "*"
+      },
+	  {
+        Effect= "Allow"
+        Action = [
+          "s3:PutObject",
+          "s3:GetObject",
+          "s3:ListAllMyBuckets",
+          "s3:ListBucket",
+          "s3:DeleteObject",
+          "s3:GetBucketLocation"
+        ]
+        Resource = "*"
       },
       {
         Effect = "Allow"

terraform/azure/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "=4.13.0"
+      version = "=4.81.0"
     }
     local = {
       source = "hashicorp/local"