fix(deploy): organise deployed jobs into folders

deployment/clouddeploy/gke-workers/base/core/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+- exporter.yaml
+- gitter.yaml
+- importer.yaml
+- importer-deleter.yaml
+- importer-reconciler.yaml
+- record-checker.yaml
+- recoverer.yaml
+- scaler.yaml
+- workers.yaml

deployment/clouddeploy/gke-workers/base/extra/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+- backup.yaml
+- custommetrics.yaml
+- generate-sitemap.yaml
+- relations.yaml
+- vanir-signatures.yaml
+- workers-cves.yaml
+- workers-reimport.yaml

deployment/clouddeploy/gke-workers/base/feeds/kustomization.yaml

@@ -0,0 +1,11 @@
+resources:
+- alpine-cve-convert.yaml
+- combine-to-osv.yaml
+- cpe-repo-gen.yaml
+- cve5-to-osv.yaml
+- debian-convert.yaml
+- debian-copyright-mirror.yaml
+- debian-cve-convert.yaml
+- debian-first-version.yaml
+- nvd-cve-osv.yaml
+- nvd-mirror.yaml

deployment/clouddeploy/gke-workers/base/ksm/kustomization.yaml

@@ -0,0 +1,9 @@
+resources:
+- ksm_cluster_pod_monitoring.yaml
+- ksm_cluster_role.yaml
+- ksm_cluster_role_binding.yaml
+- ksm_horizontal_pod_autoscaler.yaml
+- ksm_pod_monitoring.yaml
+- ksm_service.yaml
+- ksm_service_account.yaml
+- ksm_stateful_set.yaml

deployment/clouddeploy/gke-workers/base/kustomization.yaml

@@ -1,36 +1,5 @@
 resources:
-- workers.yaml
-- scaler.yaml
-- workers-cves.yaml
-- workers-reimport.yaml
-- importer.yaml
-- importer-deleter.yaml
-- importer-reconciler.yaml
-- exporter.yaml
-- alpine-cve-convert.yaml
-- debian-cve-convert.yaml
-- combine-to-osv.yaml
-- debian-convert.yaml
-- debian-first-version.yaml
-- debian-copyright-mirror.yaml
-- cpe-repo-gen.yaml
-- gitter.yaml
-- nvd-cve-osv.yaml
-- relations.yaml
-- nvd-mirror.yaml
-- backup.yaml
-- generate-sitemap.yaml
-- ksm_cluster_pod_monitoring.yaml
-- ksm_cluster_role_binding.yaml
-- ksm_cluster_role.yaml
-- ksm_horizontal_pod_autoscaler.yaml
-- ksm_pod_monitoring.yaml
-- ksm_service_account.yaml
-- ksm_service.yaml
-- ksm_stateful_set.yaml
-- recoverer.yaml
-- record-checker.yaml
-- cve5-to-osv.yaml
-- custommetrics.yaml
-- vanir-signatures.yaml
-
+- core
+- feeds
+- extra
+- ksm

deployment/clouddeploy/gke-workers/environments/private/kustomization.yaml

@@ -2,16 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-# - ../../base/backup.yaml  # TODO(michaelkedar): figure out how to backup specific datastore IDs
-- ../../base/exporter.yaml
-- ../../base/gitter.yaml
-- ../../base/importer.yaml
-- ../../base/importer-deleter.yaml
-- ../../base/importer-reconciler.yaml
-- ../../base/record-checker.yaml
-- ../../base/recoverer.yaml
-- ../../base/workers.yaml
-- ../../base/scaler.yaml
+- ../../base/core
+# TODO(michaelkedar): figure out how to backup specific datastore database IDs
 
 patches:
 # - path: backup.yaml

deployment/terraform/environments/oss-vdb-test/main.tf

@@ -6,7 +6,7 @@ locals {
 
   all_resources = concat(
     [for resource in local.env_kustomization.resources : "${local.env_kustomization_path}/${resource}" if can(regex("\\.yaml$", resource))],
-    [for resource in local.base_kustomization.resources : "${local.base_kustomization_path}/${resource}" if can(regex("\\.yaml$", resource))],
+    [for f in fileset(local.base_kustomization_path, "**/*.yaml") : "${local.base_kustomization_path}/${f}" if !endswith(f, "kustomization.yaml")],
   )
 
   # Iterate of each yaml configuration and create a key based on kind and name in the yaml file.

deployment/terraform/environments/oss-vdb/main.tf

@@ -6,7 +6,7 @@ locals {
 
   all_resources = concat(
     [for resource in local.env_kustomization.resources : "${local.env_kustomization_path}/${resource}" if can(regex("\\.yaml$", resource))],
-    [for resource in local.base_kustomization.resources : "${local.base_kustomization_path}/${resource}" if can(regex("\\.yaml$", resource))],
+    [for f in fileset(local.base_kustomization_path, "**/*.yaml") : "${local.base_kustomization_path}/${f}" if !endswith(f, "kustomization.yaml")],
   )
 
   # Iterate of each yaml configuration and create a key based on kind and name in the yaml file.

docs/data.md

@@ -117,7 +117,7 @@ The quality of the data in OSV.dev [is very important to us](https://google.gith
 
 ## Data dumps
 
-For convenience, these sources are aggregated and [continuously](https://github.com/google/osv.dev/blob/master/deployment/clouddeploy/gke-workers/base/exporter.yaml) 
+For convenience, these sources are aggregated and [continuously](https://github.com/google/osv.dev/blob/master/deployment/clouddeploy/gke-workers/base/core/exporter.yaml) 
 exported to a GCS bucket maintained by OSV:
 [`gs://osv-vulnerabilities`](https://storage.googleapis.com/osv-vulnerabilities/index.html)
 

vulnfeeds/cmd/combine-to-osv/README.md

@@ -18,7 +18,7 @@ See [`run_combine_to_osv_convert.sh`](run_combine_to_osv_convert.sh):
   * What gets written can be overridden by OSV records in [`gs://cve-osv-conversion/osv-output-overrides`](https://storage.googleapis.com/cve-osv-conversion/index.html?prefix=osv-output-overrides/)
 ## Operational matters
 
-* Runs every hour (on the half hour) as a [Kubernetes CronJob](https://github.com/google/osv.dev/blob/master/deployment/clouddeploy/gke-workers/base/combine-to-osv.yaml)
+* Runs every hour (on the half hour) as a [Kubernetes CronJob](https://github.com/google/osv.dev/blob/master/deployment/clouddeploy/gke-workers/base/feeds/combine-to-osv.yaml)
 
 ## Usage