Adding Argo Events - Azure

azure/terraform/main.tf

@@ -9,35 +9,35 @@ terraform {
       version = "3.14.0"
     }
     helm = {
-      source = "hashicorp/helm"
+      source  = "hashicorp/helm"
       version = "2.6.0"
     }
   }
 }
 
 
 data "azurerm_kubernetes_cluster" "default" {
-  depends_on          = [module.infra]  # refresh cluster state before reading
+  depends_on          = [module.infra] # refresh cluster state before reading
   resource_group_name = local.metaflow_resource_group_name
   name                = local.kubernetes_cluster_name
 }
 
 data "azurerm_postgresql_flexible_server" "default" {
-  depends_on          = [module.infra]  # refresh cluster state before reading
+  depends_on          = [module.infra] # refresh cluster state before reading
   resource_group_name = local.metaflow_resource_group_name
   name                = local.database_server_name
 }
 
 data "azurerm_storage_account" "default" {
-  depends_on          = [module.infra]  # refresh cluster state before reading
+  depends_on          = [module.infra] # refresh cluster state before reading
   resource_group_name = local.metaflow_resource_group_name
   name                = local.storage_account_name
-  
+
 }
 
 data "azurerm_storage_container" "default" {
-  depends_on          = [module.infra]  # refresh cluster state before reading
-  name                = local.storage_container_name
+  depends_on           = [module.infra] # refresh cluster state before reading
+  name                 = local.storage_container_name
   storage_account_name = local.storage_account_name
 }
 
@@ -95,15 +95,16 @@ module "services" {
   metaflow_db_user                             = local.metaflow_database_server_admin_login
   metaflow_db_password                         = local.metaflow_db_password
   metaflow_kubernetes_secret_name              = local.metaflow_kubernetes_secret_name
-  azure_storage_credentials                    = {
+  azure_storage_credentials = {
     AZURE_CLIENT_ID     = module.infra.service_principal_client_id
     AZURE_TENANT_ID     = module.infra.service_principal_tenant_id
     AZURE_CLIENT_SECRET = module.infra.service_principal_client_secret
   }
-
-  deploy_airflow                          = var.deploy_airflow
-  deploy_argo                             = var.deploy_argo
-
-  airflow_version =  local.airflow_version
-  airflow_frenet_secret =  local.airflow_frenet_secret
+
+  deploy_airflow     = var.deploy_airflow
+  deploy_argo        = var.deploy_argo
+  deploy_argo_events = var.deploy_argo_events
+
+  airflow_version       = local.airflow_version
+  airflow_frenet_secret = local.airflow_frenet_secret
 }

azure/terraform/services/argo.tf

@@ -5,9 +5,20 @@ resource "kubernetes_namespace" "argo" {
   }
 }
 
+resource "kubernetes_namespace" "argo-events" {
+  count = var.deploy_argo_events ? 1 : 0
+  metadata {
+    name = "argo-events"
+  }
+}
+
 locals {
-  is_windows = substr(pathexpand("~"), 0, 1) == "/" ? false : true
-  _kubectl_cmd = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start-postgres.yaml"
+  is_windows          = substr(pathexpand("~"), 0, 1) == "/" ? false : true
+  _argo_cmd           = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start-postgres.yaml"
+  _argo_events_cmd    = "kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-events/v1.7.3/manifests/install.yaml"
+  _service_accts_cmd  = "kubectl apply -n argo -f ${path.module}/argo_events/service_accounts.yaml"
+  _event_bus_cmd      = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-events/v1.7.3/examples/eventbus/native.yaml"
+  _webhook_source_cmd = "kubectl apply -n argo -f ${path.module}/argo_events/webhook_source.yaml"
 }
 
 # Yes local-exec is unfortunate.
@@ -17,10 +28,58 @@ locals {
 resource "null_resource" "argo-quick-start-installation" {
   count = var.deploy_argo ? 1 : 0
   triggers = {
-    cmd = local._kubectl_cmd
+    cmd = local._argo_cmd
+  }
+  provisioner "local-exec" {
+    interpreter = local.is_windows ? ["PowerShell"] : null
+    command     = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._argo_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._argo_cmd}"
+  }
+}
+
+resource "null_resource" "argo-events-quick-start" {
+  count      = var.deploy_argo_events ? 1 : 0
+  depends_on = [null_resource.argo-quick-start-installation]
+  triggers = {
+    cmd = local._argo_events_cmd
+  }
+  provisioner "local-exec" {
+    interpreter = local.is_windows ? ["PowerShell"] : null
+    command     = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._argo_events_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._argo_events_cmd}"
+  }
+}
+
+resource "null_resource" "argo-events-service-accounts" {
+  count      = var.deploy_argo_events ? 1 : 0
+  depends_on = [null_resource.argo-events-quick-start]
+  triggers = {
+    cmd = local._service_accts_cmd
+  }
+  provisioner "local-exec" {
+    interpreter = local.is_windows ? ["PowerShell"] : null
+    command     = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._service_accts_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._service_accts_cmd}"
+  }
+}
+
+resource "null_resource" "argo-events-event-bus" {
+  count      = var.deploy_argo_events ? 1 : 0
+  depends_on = [null_resource.argo-events-quick-start]
+  triggers = {
+    cmd = local._event_bus_cmd
+  }
+  provisioner "local-exec" {
+    interpreter = local.is_windows ? ["PowerShell"] : null
+    command     = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._event_bus_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._event_bus_cmd}"
+  }
+}
+
+resource "null_resource" "argo-events-webhook-source" {
+  count      = var.deploy_argo_events ? 1 : 0
+  depends_on = [null_resource.argo-events-event-bus]
+  triggers = {
+    cmd = local._webhook_source_cmd
   }
   provisioner "local-exec" {
     interpreter = local.is_windows ? ["PowerShell"] : null
-    command = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._kubectl_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._kubectl_cmd}"
+    command     = local.is_windows ? "$env:KUBECONFIG='${local_file.kubeconfig.filename}'; ${local._webhook_source_cmd}" : "KUBECONFIG=${local_file.kubeconfig.filename} ${local._webhook_source_cmd}"
   }
 }

azure/terraform/services/argo_events/service_accounts.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: operate-workflow-sa
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: operate-workflow-role
+rules:
+  - apiGroups:
+      - argoproj.io
+    verbs:
+      - "*"
+    resources:
+      - workflows
+      - workflowtemplates
+      - cronworkflows
+      - clusterworkflowtemplates
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: operate-workflow-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: operate-workflow-role
+subjects:
+  - kind: ServiceAccount
+    name: operate-workflow-sa

azure/terraform/services/argo_events/webhook_source.yaml

@@ -0,0 +1,15 @@
+apiVersion: argoproj.io/v1alpha1
+kind: EventSource
+metadata:
+  name: metaflow-webhook
+  namespace: argo
+spec:
+  service:
+    ports:
+    - port: 12000
+      targetPort: 12000
+  webhook:
+    event:
+      port: "12000"
+      endpoint: /event
+      method: POST

azure/terraform/services/variables.tf

@@ -42,10 +42,10 @@ variable "metaflow_azure_storage_blob_service_endpoint" {
 }
 
 variable "azure_storage_credentials" {
-  type = map
+  type = map(any)
 }
 
-variable "metaflow_kubernetes_secret_name"{
+variable "metaflow_kubernetes_secret_name" {
   type = string
 }
 
@@ -61,6 +61,10 @@ variable "deploy_argo" {
   type = bool
 }
 
+variable "deploy_argo_events" {
+  type = bool
+}
+
 variable "deploy_airflow" {
   type = bool
 }

azure/terraform/variables.tf

@@ -3,35 +3,35 @@ locals {
 
 
 
-  storage_container_name = "metaflow-storage-container"
+  storage_container_name           = "metaflow-storage-container"
   metaflow_datastore_sysroot_azure = "${local.storage_container_name}/tf-full-stack-sysroot"
-  location = "westus"
-  metaflow_resource_group_name = "rg-metaflow-${terraform.workspace}-${local.location}"
+  location                         = "westus"
+  metaflow_resource_group_name     = "rg-metaflow-${terraform.workspace}-${local.location}"
   # MUST be globally unique (entire Azure). Would recommend user to add a meaningful prefix.
   kubernetes_cluster_name = "aks-${var.org_prefix}-metaflow-${terraform.workspace}"
   # This MUST be globally unique (entire Azure). Pick a meaningful and unique value for org_prefix
   database_server_name = "psql-${var.org_prefix}-metaflow-${terraform.workspace}"
   # This MUST be globally unique (entire Azure). Pick a meaningful and unique value for org_prefix
-  storage_account_name = "st${var.org_prefix}metaflow${terraform.workspace}"
+  storage_account_name           = "st${var.org_prefix}metaflow${terraform.workspace}"
   storage_service_principal_name = "Metaflow storage service principal (${terraform.workspace})"
-  virtual_network_name = "vnet-${var.org_prefix}-metaflow-${local.location}-${terraform.workspace}"
-  db_subnet_name = "snet-${var.org_prefix}-metaflow-db-${local.location}-${terraform.workspace}"
-  k8s_subnet_name = "snet-${var.org_prefix}-metaflow-k8s-${local.location}-${terraform.workspace}"
+  virtual_network_name           = "vnet-${var.org_prefix}-metaflow-${local.location}-${terraform.workspace}"
+  db_subnet_name                 = "snet-${var.org_prefix}-metaflow-db-${local.location}-${terraform.workspace}"
+  k8s_subnet_name                = "snet-${var.org_prefix}-metaflow-k8s-${local.location}-${terraform.workspace}"
 
   # Changeable after initial "terraform apply" (e.g. image upgrades, secret rotation)
-  metadata_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3"
-  metaflow_ui_static_service_image = "public.ecr.aws/outerbounds/metaflow_ui:v1.1.4"
+  metadata_service_image            = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3"
+  metaflow_ui_static_service_image  = "public.ecr.aws/outerbounds/metaflow_ui:v1.1.4"
   metaflow_ui_backend_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3"
-  metaflow_kubernetes_secret_name = "metaflow-azure-storage-credentials"
+  metaflow_kubernetes_secret_name   = "metaflow-azure-storage-credentials"
 
   # Forever constants
   metaflow_database_server_admin_login = "metaflow"
-  metaflow_db_name = "metaflow"
-  metaflow_db_password = "metaflow" # DB is private, accessible only within vnet.
-  metaflow_db_port = 5432
+  metaflow_db_name                     = "metaflow"
+  metaflow_db_password                 = "metaflow" # DB is private, accessible only within vnet.
+  metaflow_db_port                     = 5432
 
   # Airflow Related Options
-  airflow_version = "2.3.3"
+  airflow_version       = "2.3.3"
   airflow_frenet_secret = "myverysecretvalue"
 }
 
@@ -45,11 +45,16 @@ variable "org_prefix" {
 }
 
 variable "deploy_argo" {
-  type = bool
+  type    = bool
+  default = true
+}
+
+variable "deploy_argo_events" {
+  type    = bool
   default = true
 }
 
 variable "deploy_airflow" {
-  type = bool
+  type    = bool
   default = false
-}
+}