From c9e061ff0363aa1b3618b6f791230bfa9b1f6a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 16:41:42 +0200 Subject: [PATCH 01/41] Add nomad installation --- ansible/deploy-nomad.yml | 10 + ansible/roles/nomad/defaults/main.yml | 0 ansible/roles/nomad/handlers/main.yml | 27 +++ ansible/roles/nomad/tasks/main.yml | 47 ++++ .../roles/nomad/templates/02-spool-dir.conf | 19 ++ .../roles/nomad/templates/api-uploader.conf | 10 + ansible/roles/nomad/templates/fastpath.conf | 21 ++ .../nomad/templates/ooni-api-uploader.service | 11 + .../nomad/templates/ooni-api-uploader.timer | 12 + .../nomad/templates/ooni_api_uploader.py | 226 ++++++++++++++++++ 10 files changed, 383 insertions(+) create mode 100644 ansible/deploy-nomad.yml create mode 100644 ansible/roles/nomad/defaults/main.yml create mode 100644 ansible/roles/nomad/handlers/main.yml create mode 100644 ansible/roles/nomad/tasks/main.yml create mode 100644 ansible/roles/nomad/templates/02-spool-dir.conf create mode 100644 ansible/roles/nomad/templates/api-uploader.conf create mode 100644 ansible/roles/nomad/templates/fastpath.conf create mode 100644 ansible/roles/nomad/templates/ooni-api-uploader.service create mode 100644 ansible/roles/nomad/templates/ooni-api-uploader.timer create mode 100644 ansible/roles/nomad/templates/ooni_api_uploader.py diff --git a/ansible/deploy-nomad.yml b/ansible/deploy-nomad.yml new file mode 100644 index 00000000..9127fb74 --- /dev/null +++ b/ansible/deploy-nomad.yml @@ -0,0 +1,10 @@ +--- +# This will be in dev mode for the moment, just as a POC +- name: Deploy Nomad agent + hosts: anonc.dev.ooni.io # Use this host in the meanwhile + become: true + vars: + ssl_domains: + - "{{ inventory_hostname }}" + roles: + - nomad diff --git a/ansible/roles/nomad/defaults/main.yml b/ansible/roles/nomad/defaults/main.yml new file mode 100644 index 00000000..e69de29b diff --git a/ansible/roles/nomad/handlers/main.yml b/ansible/roles/nomad/handlers/main.yml new file mode 100644 index 00000000..594ebfe9 --- /dev/null +++ b/ansible/roles/nomad/handlers/main.yml @@ -0,0 +1,27 @@ +- name: test nginx config + command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: + - restart nginx + - reload nginx + +- name: restart nginx + service: + name: nginx + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: restart docker + tags: docker + ansible.builtin.systemd_service: + name: docker + state: restarted \ No newline at end of file diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml new file mode 100644 index 00000000..6e3341c8 --- /dev/null +++ b/ansible/roles/nomad/tasks/main.yml @@ -0,0 +1,47 @@ +--- +# For prometheus scrape requests +- name: Flush all handlers + meta: flush_handlers + +- name: Install prerequisite packages + ansible.builtin.apt: + name: + - gpg + - coreutils + state: present + +- name: Create keyring directory + ansible.builtin.file: + path: /usr/share/keyrings + state: directory + mode: '0755' + +- name: Download HashiCorp GPG key + ansible.builtin.get_url: + url: https://apt.releases.hashicorp.com/gpg + dest: /tmp/hashicorp.asc + mode: '0644' + +- name: Dearmor HashiCorp GPG key + ansible.builtin.command: + cmd: gpg --dearmor -y -o /usr/share/keyrings/hashicorp-archive-keyring.gpg /tmp/hashicorp.asc + args: + creates: /usr/share/keyrings/hashicorp-archive-keyring.gpg + +- name: Add HashiCorp apt repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com {{ ansible_distribution_release }} main" + filename: hashicorp + state: present + +- name: Install Nomad + ansible.builtin.apt: + name: nomad + state: present + update_cache: yes + +- name: Ensure Nomad service is enabled and started + ansible.builtin.service: + name: nomad + state: started + enabled: yes diff --git a/ansible/roles/nomad/templates/02-spool-dir.conf b/ansible/roles/nomad/templates/02-spool-dir.conf new file mode 100644 index 00000000..a333a70e --- /dev/null +++ b/ansible/roles/nomad/templates/02-spool-dir.conf @@ -0,0 +1,19 @@ +# This nginx config will serve spool dir files. +# This is intended for the oonimeasurements component, +# which will try to find jsonl measurements in this host +# if it can't find them in S3 + +server { + listen 8475; + server_name fastpath.{{env}}.ooni.io fastpath2.{{env}}.ooni.io; + + location /measurement_spool/ { + alias /var/lib/ooniapi/measurements/incoming/; + autoindex off; + sendfile on; + tcp_nopush on; + if_modified_since off; + expires off; + etag off; + } +} \ No newline at end of file diff --git a/ansible/roles/nomad/templates/api-uploader.conf b/ansible/roles/nomad/templates/api-uploader.conf new file mode 100644 index 00000000..82b8c26e --- /dev/null +++ b/ansible/roles/nomad/templates/api-uploader.conf @@ -0,0 +1,10 @@ +# OONI API measurement uploader - Python ini format +# Deployed by ansible, see roles/ooni-backend/templates/api-uploader.conf +[DEFAULT] +# arn:aws:iam::676739448697:user/ooni-pipeline, AWS: OONI Open Data +aws_access_key_id = AKIAJURD7T4DTN5JMJ5Q +aws_secret_access_key = {{ s3_ooni_open_data_access_key }} +bucket_name = {{ bucket_name }} +msmt_spool_dir = /var/lib/ooniapi/measurements +collector_id = {{ collector_id }} +db_uri = {{ clickhouse_url }} \ No newline at end of file diff --git a/ansible/roles/nomad/templates/fastpath.conf b/ansible/roles/nomad/templates/fastpath.conf new file mode 100644 index 00000000..ac997db3 --- /dev/null +++ b/ansible/roles/nomad/templates/fastpath.conf @@ -0,0 +1,21 @@ +[DEFAULT] +# Collector hostnames, comma separated +collectors = localhost + + +{% if psql_uri is defined %} +# The password is already made public +db_uri = {{ psql_uri }} +{% else %} +db_uri = +{% endif %} + +# S3 access credentials +# Currently unused +s3_access_key = +s3_secret_key = + + +clickhouse_url = {{clickhouse_url}} + +msmt_spool_dir = /var/lib/ooniapi/measurements \ No newline at end of file diff --git a/ansible/roles/nomad/templates/ooni-api-uploader.service b/ansible/roles/nomad/templates/ooni-api-uploader.service new file mode 100644 index 00000000..3aff1e53 --- /dev/null +++ b/ansible/roles/nomad/templates/ooni-api-uploader.service @@ -0,0 +1,11 @@ +[Unit] +Description=Upload raw OONI measurements to S3 +Wants=ooni-api-uploader.timer + +[Service] +Type=oneshot +ExecStart=/usr/bin/ooni_api_uploader.py + +[Install] +WantedBy=multi-user.target + diff --git a/ansible/roles/nomad/templates/ooni-api-uploader.timer b/ansible/roles/nomad/templates/ooni-api-uploader.timer new file mode 100644 index 00000000..b0946c58 --- /dev/null +++ b/ansible/roles/nomad/templates/ooni-api-uploader.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Upload raw OONI measurements to S3 +Requires=ooni-api-uploader.service + +[Timer] +Unit=ooni-api-uploader.service +# run every hour at 10 minutes past the hour +OnCalendar=*-*-* *:10 +#OnCalendar=*-*-* *:00/5 + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/nomad/templates/ooni_api_uploader.py b/ansible/roles/nomad/templates/ooni_api_uploader.py new file mode 100644 index 00000000..8c9a5255 --- /dev/null +++ b/ansible/roles/nomad/templates/ooni_api_uploader.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +Uploads OONI API measurements to S3 +Reads /etc/ooni/api.conf +""" + +from configparser import ConfigParser +from pathlib import Path +from pathlib import PosixPath as PP +from datetime import datetime, timedelta +from typing import List, Dict +import gzip +import logging +import tarfile +import yaml + +import ujson +import boto3 +import statsd # debdeps: python3-statsd + +# debdeps: python3-clickhouse-driver +from clickhouse_driver import Client as Clickhouse + + +metrics = statsd.StatsClient("127.0.0.1", 8125, prefix="ooni_api_uploader") +log = logging.getLogger("ooni_api_uploader") + +try: + from systemd.journal import JournalHandler # debdeps: python3-systemd + + log.addHandler(JournalHandler(SYSLOG_IDENTIFIER="ooni_api_uploader")) +except ImportError: + pass + +log.setLevel(logging.DEBUG) + + +def create_s3_client(conf): + session = boto3.Session( + aws_access_key_id=conf.get("aws_access_key_id"), + aws_secret_access_key=conf.get("aws_secret_access_key"), + ) + return session.resource("s3") + + +def read_conf(): + cf = "/etc/ooni/api-uploader.conf" + log.info(f"Starting. Reading {cf}") + conf = ConfigParser() + conf.read(cf) + return conf["DEFAULT"] + + +def connect_to_db(conf): + default = "clickhouse://api:api@localhost/default" + uri = conf.get("db_uri", default) + log.debug(f"Connecting to database on {uri}") + conn = Clickhouse.from_url(uri) + return conn + + +@metrics.timer("update_db_table") +def update_db_table(conn, lookup_list, jsonl_s3path): + for d in lookup_list: + d["s3path"] = jsonl_s3path + + q = "INSERT INTO jsonl (report_id, input, s3path, linenum, measurement_uid) VALUES" + log.info(f"Writing {len(lookup_list)} rows to DB") + conn.execute(q, lookup_list) + + +@metrics.timer("upload_measurement") +def upload_to_s3(s3, bucket_name: str, tarf: PP, s3path: str) -> None: + obj = s3.Object(bucket_name, s3path) + log.info(f"Uploading {s3path}") + obj.put(Body=tarf.read_bytes()) + + +@metrics.timer("fill_postcan") +def fill_postcan(hourdir: PP, postcanf: PP) -> List[PP]: + msmt_files = sorted(f for f in hourdir.iterdir() if f.suffix == ".post") + if not msmt_files: + log.info(f"Nothing to fill {postcanf.name}") + return [] + log.info(f"Filling {postcanf.name}") + measurements = [] + postcan_byte_thresh = 20 * 1000 * 1000 + # Open postcan + with tarfile.open(str(postcanf), "w:gz") as tar: + for msmt_f in msmt_files: + # Add a msmt and delete the msmt file + metrics.incr("msmt_count") + tar.add(str(msmt_f)) + measurements.append(msmt_f) + tarsize = postcanf.stat().st_size + if tarsize > postcan_byte_thresh: + log.info(f"Reached {tarsize} bytes") + return measurements + + return measurements + + +@metrics.timer("fill_jsonl") +def fill_jsonl(measurements: List[PP], jsonlf: PP) -> List[Dict]: + log.info(f"Filling {jsonlf.name}") + # report_id, input, 2020092119_IT_tor.n0.0.jsonl.gz + lookup_list = [] + with gzip.open(jsonlf, "w") as jf: + for linenum, msmt_f in enumerate(measurements): + try: + post = ujson.load(msmt_f.open()) + except Exception: + log.error("Unable to parse measurement") + jf.write(b"{}\n") + continue + + fmt = post.get("format", "").lower() + msm = None + if fmt == "json": + msm = post.get("content", {}) + elif fmt == "yaml": + try: + msm = yaml.load(msm, Loader=yaml.CLoader) + except Exception: + pass + + if msm is None: + log.error("Unable to parse measurement") + jf.write(b"{}\n") + continue + + jf.write(ujson.dumps(msm).encode()) + jf.write(b"\n") + + rid = msm.get("report_id") or "" + input = msm.get("input") or "" + msmt_uid = msmt_f.name[:-5] + d = dict( + report_id=rid, input=input, measurement_uid=msmt_uid, linenum=linenum + ) + lookup_list.append(d) + + return lookup_list + + +def delete_msmt_posts(measurements: List[PP]) -> None: + log.info(f"Deleting {len(measurements)} measurements") + for msmt_f in measurements: + msmt_f.unlink() + + +@metrics.timer("total_run_time") +def main(): + conf = read_conf() + bucket_name = conf.get("bucket_name") + spooldir = Path(conf.get("msmt_spool_dir")) + format_char = "n" + collector_id = conf.get("collector_id") + identity = f"{format_char}{collector_id}" + log.info(f"Uploader {collector_id} starting") + assert collector_id, "collector_id is not set" + log.info(f"Using bucket {bucket_name} and spool {spooldir}") + + s3 = create_s3_client(conf) + + db_conn = connect_to_db(conf) + + # Scan spool directories, by age + idir = spooldir / "incoming" + threshold = datetime.utcnow() - timedelta(hours=1) + for hourdir in sorted(idir.iterdir()): + if not hourdir.is_dir() or hourdir.suffix == ".tmp": + continue + try: + tstamp, cc, testname = hourdir.name.split("_") + except Exception: + continue + if len(tstamp) != 10: + continue + hourdir_time = datetime.strptime(tstamp, "%Y%m%d%H") + if hourdir_time > threshold: + log.info(f"Stopping before {hourdir_time}") + break + + log.info(f"Processing {hourdir}") + # Split msmts across multiple postcans and jsonl files + can_cnt = 0 + while True: + # Compress raw POSTs into a tar.gz postcan + postcanf = hourdir.with_suffix(f".{identity}.{can_cnt}.tar.gz") + jsonlf = hourdir.with_suffix(f".{identity}.{can_cnt}.jsonl.gz") + msmfiles = fill_postcan(hourdir, postcanf) + if len(msmfiles) == 0: + break + # Also create jsonl file and delete msmt POSTs + lookup_list = fill_jsonl(msmfiles, jsonlf) + delete_msmt_posts(msmfiles) + + # Upload current postcan to S3 + postcan_s3path = ( + f"raw/{tstamp[:8]}/{tstamp[8:10]}/{cc}/{testname}/{postcanf.name}" + ) + jsonl_s3path = ( + f"raw/{tstamp[:8]}/{tstamp[8:10]}/{cc}/{testname}/{jsonlf.name}" + ) + + upload_to_s3(s3, bucket_name, postcanf, postcan_s3path) + upload_to_s3(s3, bucket_name, jsonlf, jsonl_s3path) + update_db_table(db_conn, lookup_list, jsonl_s3path) + + postcanf.unlink() + jsonlf.unlink() + + can_cnt += 1 + metrics.incr("postcan_count") + + # Delete whole hourly directory + for f in sorted(hourdir.iterdir()): + f.unlink() + hourdir.rmdir() + + log.info("Exiting") + + +if __name__ == "__main__": + main() From 430f31f518eacc159500d3a03b9c675053e0d790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 16:52:16 +0200 Subject: [PATCH 02/41] Add nomad config file --- ansible/deploy-nomad.yml | 1 + ansible/roles/nomad/handlers/main.yml | 5 +++++ ansible/roles/nomad/tasks/main.yml | 31 +++++++++++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/ansible/deploy-nomad.yml b/ansible/deploy-nomad.yml index 9127fb74..496f1414 100644 --- a/ansible/deploy-nomad.yml +++ b/ansible/deploy-nomad.yml @@ -7,4 +7,5 @@ ssl_domains: - "{{ inventory_hostname }}" roles: + - nginx - nomad diff --git a/ansible/roles/nomad/handlers/main.yml b/ansible/roles/nomad/handlers/main.yml index 594ebfe9..94777c6a 100644 --- a/ansible/roles/nomad/handlers/main.yml +++ b/ansible/roles/nomad/handlers/main.yml @@ -24,4 +24,9 @@ tags: docker ansible.builtin.systemd_service: name: docker + state: restarted + +- name: restart nomad + ansible.builtin.systemd_service: + name: nomad state: restarted \ No newline at end of file diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 6e3341c8..b78b3e57 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -40,6 +40,37 @@ state: present update_cache: yes +- name: Create Nomad configuration directory + ansible.builtin.file: + path: /etc/nomad.d + state: directory + mode: '0755' + owner: root + group: root + +- name: Setup Nomad in dev mod + ansible.builtin.copy: + dest: /etc/nomad.d/nomad.hcl # Main config file + content: | + data_dir = "/opt/nomad" + + server { + enabled = true + bootstrap_expect = 1 + } + + client { + enabled = true + } + + acl { + enabled = true + } + mode: '0644' + owner: root + group: root + notify: restart nomad + - name: Ensure Nomad service is enabled and started ansible.builtin.service: name: nomad From c3bc013b8d27c60c128843478ebeeb0c45d6ba3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 17:05:44 +0200 Subject: [PATCH 03/41] Install docker in nomad server --- ansible/deploy-nomad.yml | 1 - ansible/roles/nomad/tasks/main.yml | 43 +++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ansible/deploy-nomad.yml b/ansible/deploy-nomad.yml index 496f1414..9127fb74 100644 --- a/ansible/deploy-nomad.yml +++ b/ansible/deploy-nomad.yml @@ -7,5 +7,4 @@ ssl_domains: - "{{ inventory_hostname }}" roles: - - nginx - nomad diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index b78b3e57..65e76de9 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -3,9 +3,11 @@ - name: Flush all handlers meta: flush_handlers -- name: Install prerequisite packages +- name: Install Docker prerequisites ansible.builtin.apt: name: + - ca-certificates + - curl - gpg - coreutils state: present @@ -16,6 +18,39 @@ state: directory mode: '0755' +- name: Download Docker GPG key + ansible.builtin.get_url: + url: https://download.docker.com/linux/debian/gpg + dest: /tmp/docker.asc + mode: '0644' + +- name: Dearmor Docker GPG key + ansible.builtin.command: + cmd: gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg /tmp/docker.asc + args: + creates: /usr/share/keyrings/docker-archive-keyring.gpg + +- name: Add Docker apt repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + filename: docker + state: present + +- name: Install Docker + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + state: present + update_cache: yes + +- name: Ensure Docker service is enabled and started + ansible.builtin.service: + name: docker + state: started + enabled: yes + - name: Download HashiCorp GPG key ansible.builtin.get_url: url: https://apt.releases.hashicorp.com/gpg @@ -24,7 +59,7 @@ - name: Dearmor HashiCorp GPG key ansible.builtin.command: - cmd: gpg --dearmor -y -o /usr/share/keyrings/hashicorp-archive-keyring.gpg /tmp/hashicorp.asc + cmd: gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg /tmp/hashicorp.asc args: creates: /usr/share/keyrings/hashicorp-archive-keyring.gpg @@ -52,7 +87,8 @@ ansible.builtin.copy: dest: /etc/nomad.d/nomad.hcl # Main config file content: | - data_dir = "/opt/nomad" + bind_addr = "0.0.0.0" + data_dir = "/opt/nomad/data" server { enabled = true @@ -61,6 +97,7 @@ client { enabled = true + servers = ["127.0.0.1"] } acl { From 15204614db12cea359f74870673657460e3a594c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 17:17:14 +0200 Subject: [PATCH 04/41] Add relevant tags to every service --- ansible/deploy-nomad.yml | 1 + ansible/roles/nomad/tasks/main.yml | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ansible/deploy-nomad.yml b/ansible/deploy-nomad.yml index 9127fb74..d065f8ec 100644 --- a/ansible/deploy-nomad.yml +++ b/ansible/deploy-nomad.yml @@ -7,4 +7,5 @@ ssl_domains: - "{{ inventory_hostname }}" roles: + - bootstrap - nomad diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 65e76de9..46152598 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -4,6 +4,7 @@ meta: flush_handlers - name: Install Docker prerequisites + tags: docker ansible.builtin.apt: name: - ca-certificates @@ -13,30 +14,35 @@ state: present - name: Create keyring directory + tags: [docker, nomad] ansible.builtin.file: path: /usr/share/keyrings state: directory mode: '0755' - name: Download Docker GPG key + tags: docker ansible.builtin.get_url: url: https://download.docker.com/linux/debian/gpg dest: /tmp/docker.asc mode: '0644' - name: Dearmor Docker GPG key + tags: docker ansible.builtin.command: cmd: gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg /tmp/docker.asc args: creates: /usr/share/keyrings/docker-archive-keyring.gpg - name: Add Docker apt repository + tags: docker ansible.builtin.apt_repository: - repo: "deb [signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + repo: "deb [signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} stable" filename: docker state: present - name: Install Docker + tags: docker ansible.builtin.apt: name: - docker-ce @@ -46,36 +52,42 @@ update_cache: yes - name: Ensure Docker service is enabled and started + tags: docker ansible.builtin.service: name: docker state: started enabled: yes - name: Download HashiCorp GPG key + tags: nomad ansible.builtin.get_url: url: https://apt.releases.hashicorp.com/gpg dest: /tmp/hashicorp.asc mode: '0644' - name: Dearmor HashiCorp GPG key + tags: nomad ansible.builtin.command: cmd: gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg /tmp/hashicorp.asc args: creates: /usr/share/keyrings/hashicorp-archive-keyring.gpg - name: Add HashiCorp apt repository + tags: nomad ansible.builtin.apt_repository: repo: "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com {{ ansible_distribution_release }} main" filename: hashicorp state: present - name: Install Nomad + tags: nomad ansible.builtin.apt: name: nomad state: present update_cache: yes - name: Create Nomad configuration directory + tags: nomad ansible.builtin.file: path: /etc/nomad.d state: directory @@ -84,6 +96,7 @@ group: root - name: Setup Nomad in dev mod + tags: nomad ansible.builtin.copy: dest: /etc/nomad.d/nomad.hcl # Main config file content: | @@ -109,6 +122,7 @@ notify: restart nomad - name: Ensure Nomad service is enabled and started + tags: nomad ansible.builtin.service: name: nomad state: started From be8c6b8d3337b5d60b937adff5c47725d5a5e5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 17:29:18 +0200 Subject: [PATCH 05/41] Add dehydrated and nginx for https --- ansible/deploy-nomad.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/deploy-nomad.yml b/ansible/deploy-nomad.yml index d065f8ec..2d35e43f 100644 --- a/ansible/deploy-nomad.yml +++ b/ansible/deploy-nomad.yml @@ -7,5 +7,7 @@ ssl_domains: - "{{ inventory_hostname }}" roles: - - bootstrap + # - bootstrap + - nginx + - dehydrated - nomad From 9d367ce06ea47d5d86c8ad235ccbf68434a76737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 17:36:40 +0200 Subject: [PATCH 06/41] Open nomad ports --- ansible/roles/bootstrap/tasks/main.yml | 2 +- ansible/roles/nomad/tasks/main.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index 2b65c611..e680663b 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -68,7 +68,7 @@ - name: install systemd-resolved tags: resolved - ignore_errors: yes # some ubuntu hosts do not have the systemd-resolved package + ignore_errors: true # some ubuntu hosts do not have the systemd-resolved package ansible.builtin.apt: install_recommends: no cache_valid_time: 86400 diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 46152598..eae23c9e 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -3,6 +3,32 @@ - name: Flush all handlers meta: flush_handlers +- name: Allow Nomad HTTP traffic + become: true + tags: + - nomad-ports + - nftables + blockinfile: + path: /etc/ooni/nftables/tcp/4646.nft + create: true + block: | + add rule inet filter input tcp dport 4646 counter accept comment "nomad http api and web ui" + notify: + - Reload nftables + +- name: Allow Nomad RPC traffic + become: true + tags: + - nomad-ports + - nftables + blockinfile: + path: /etc/ooni/nftables/tcp/4647.nft + create: true + block: | + add rule inet filter input tcp dport 4647 counter accept comment "nomad rpc" + notify: + - Reload nftables + - name: Install Docker prerequisites tags: docker ansible.builtin.apt: From 540168718996724f8b9260a003cfffc6fd10c458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 17:46:12 +0200 Subject: [PATCH 07/41] Add nomad webui port --- tf/environments/dev/main.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 85474480..1d9951c1 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1422,6 +1422,11 @@ module "ooni_anonc" { protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { + from_port = 4646, # for the nomad webui + to_port = 4646, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { from_port = 9100, # for node exporter metrics to_port = 9100, protocol = "tcp" From 9b2ba524a4db5abb5ce719b54357450a5d4e167a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 18:02:17 +0200 Subject: [PATCH 08/41] Add tls to nomad webui --- ansible/roles/nomad/tasks/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index eae23c9e..5ae372c3 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -142,6 +142,15 @@ acl { enabled = true } + + tls { + http = true + rpc = true + + ca_file = "/var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem" + cert_file = "/var/lib/dehydrated/certs/{{ inventory_hostname }}/cert.pem" + key_file = "/var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem" + } mode: '0644' owner: root group: root From 9b9ccdfb24f5839d53f0f9f5fe2353818c1a0c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 18:17:55 +0200 Subject: [PATCH 09/41] Add basic auth to nomad --- ansible/roles/nomad/defaults/main.yml | 4 + ansible/roles/nomad/tasks/main.yml | 47 ++-- .../roles/nomad/templates/02-spool-dir.conf | 19 -- .../roles/nomad/templates/api-uploader.conf | 10 - ansible/roles/nomad/templates/fastpath.conf | 21 -- .../nomad/templates/ooni-api-uploader.service | 11 - .../nomad/templates/ooni-api-uploader.timer | 12 - .../nomad/templates/ooni_api_uploader.py | 226 ------------------ tf/environments/dev/main.tf | 4 +- 9 files changed, 32 insertions(+), 322 deletions(-) delete mode 100644 ansible/roles/nomad/templates/02-spool-dir.conf delete mode 100644 ansible/roles/nomad/templates/api-uploader.conf delete mode 100644 ansible/roles/nomad/templates/fastpath.conf delete mode 100644 ansible/roles/nomad/templates/ooni-api-uploader.service delete mode 100644 ansible/roles/nomad/templates/ooni-api-uploader.timer delete mode 100644 ansible/roles/nomad/templates/ooni_api_uploader.py diff --git a/ansible/roles/nomad/defaults/main.yml b/ansible/roles/nomad/defaults/main.yml index e69de29b..2c5f42c2 100644 --- a/ansible/roles/nomad/defaults/main.yml +++ b/ansible/roles/nomad/defaults/main.yml @@ -0,0 +1,4 @@ +--- +nomad_webui_username: "admin" +nomad_webui_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/nomad_webui_password', profile='oonidevops_user_prod') }}" +nomad_htpasswd_path: "/etc/nginx/nomad.htpasswd" diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 5ae372c3..b0afff82 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -3,29 +3,14 @@ - name: Flush all handlers meta: flush_handlers -- name: Allow Nomad HTTP traffic +- name: Allow Nginx HTTPS traffic for Nomad (Port 4645) become: true - tags: - - nomad-ports - - nftables - blockinfile: - path: /etc/ooni/nftables/tcp/4646.nft - create: true - block: | - add rule inet filter input tcp dport 4646 counter accept comment "nomad http api and web ui" - notify: - - Reload nftables - -- name: Allow Nomad RPC traffic - become: true - tags: - - nomad-ports - - nftables + tags: nomad blockinfile: - path: /etc/ooni/nftables/tcp/4647.nft + path: /etc/ooni/nftables/tcp/4645.nft create: true block: | - add rule inet filter input tcp dport 4647 counter accept comment "nomad rpc" + add rule inet filter input tcp dport 4645 counter accept comment "nomad web ui reverse proxy" notify: - Reload nftables @@ -126,7 +111,7 @@ ansible.builtin.copy: dest: /etc/nomad.d/nomad.hcl # Main config file content: | - bind_addr = "0.0.0.0" + bind_addr = "127.0.0.1" data_dir = "/opt/nomad/data" server { @@ -144,7 +129,7 @@ } tls { - http = true + http = false rpc = true ca_file = "/var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem" @@ -156,6 +141,26 @@ group: root notify: restart nomad +- name: Create basic auth credentials file for Nomad WebUI + tags: nomad + community.general.htpasswd: + path: "{{ nomad_htpasswd_path }}" + name: "{{ nomad_webui_username }}" + password: "{{ nomad_webui_password }}" + owner: root + group: www-data + mode: '0640' + +- name: Configure Nginx reverse proxy for Nomad + tags: nomad + ansible.builtin.template: + src: nginx-nomad.j2 + dest: /etc/nginx/sites-enabled/nomad.conf + owner: root + group: root + mode: '0644' + notify: reload nginx + - name: Ensure Nomad service is enabled and started tags: nomad ansible.builtin.service: diff --git a/ansible/roles/nomad/templates/02-spool-dir.conf b/ansible/roles/nomad/templates/02-spool-dir.conf deleted file mode 100644 index a333a70e..00000000 --- a/ansible/roles/nomad/templates/02-spool-dir.conf +++ /dev/null @@ -1,19 +0,0 @@ -# This nginx config will serve spool dir files. -# This is intended for the oonimeasurements component, -# which will try to find jsonl measurements in this host -# if it can't find them in S3 - -server { - listen 8475; - server_name fastpath.{{env}}.ooni.io fastpath2.{{env}}.ooni.io; - - location /measurement_spool/ { - alias /var/lib/ooniapi/measurements/incoming/; - autoindex off; - sendfile on; - tcp_nopush on; - if_modified_since off; - expires off; - etag off; - } -} \ No newline at end of file diff --git a/ansible/roles/nomad/templates/api-uploader.conf b/ansible/roles/nomad/templates/api-uploader.conf deleted file mode 100644 index 82b8c26e..00000000 --- a/ansible/roles/nomad/templates/api-uploader.conf +++ /dev/null @@ -1,10 +0,0 @@ -# OONI API measurement uploader - Python ini format -# Deployed by ansible, see roles/ooni-backend/templates/api-uploader.conf -[DEFAULT] -# arn:aws:iam::676739448697:user/ooni-pipeline, AWS: OONI Open Data -aws_access_key_id = AKIAJURD7T4DTN5JMJ5Q -aws_secret_access_key = {{ s3_ooni_open_data_access_key }} -bucket_name = {{ bucket_name }} -msmt_spool_dir = /var/lib/ooniapi/measurements -collector_id = {{ collector_id }} -db_uri = {{ clickhouse_url }} \ No newline at end of file diff --git a/ansible/roles/nomad/templates/fastpath.conf b/ansible/roles/nomad/templates/fastpath.conf deleted file mode 100644 index ac997db3..00000000 --- a/ansible/roles/nomad/templates/fastpath.conf +++ /dev/null @@ -1,21 +0,0 @@ -[DEFAULT] -# Collector hostnames, comma separated -collectors = localhost - - -{% if psql_uri is defined %} -# The password is already made public -db_uri = {{ psql_uri }} -{% else %} -db_uri = -{% endif %} - -# S3 access credentials -# Currently unused -s3_access_key = -s3_secret_key = - - -clickhouse_url = {{clickhouse_url}} - -msmt_spool_dir = /var/lib/ooniapi/measurements \ No newline at end of file diff --git a/ansible/roles/nomad/templates/ooni-api-uploader.service b/ansible/roles/nomad/templates/ooni-api-uploader.service deleted file mode 100644 index 3aff1e53..00000000 --- a/ansible/roles/nomad/templates/ooni-api-uploader.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Upload raw OONI measurements to S3 -Wants=ooni-api-uploader.timer - -[Service] -Type=oneshot -ExecStart=/usr/bin/ooni_api_uploader.py - -[Install] -WantedBy=multi-user.target - diff --git a/ansible/roles/nomad/templates/ooni-api-uploader.timer b/ansible/roles/nomad/templates/ooni-api-uploader.timer deleted file mode 100644 index b0946c58..00000000 --- a/ansible/roles/nomad/templates/ooni-api-uploader.timer +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Upload raw OONI measurements to S3 -Requires=ooni-api-uploader.service - -[Timer] -Unit=ooni-api-uploader.service -# run every hour at 10 minutes past the hour -OnCalendar=*-*-* *:10 -#OnCalendar=*-*-* *:00/5 - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/nomad/templates/ooni_api_uploader.py b/ansible/roles/nomad/templates/ooni_api_uploader.py deleted file mode 100644 index 8c9a5255..00000000 --- a/ansible/roles/nomad/templates/ooni_api_uploader.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python3 -""" -Uploads OONI API measurements to S3 -Reads /etc/ooni/api.conf -""" - -from configparser import ConfigParser -from pathlib import Path -from pathlib import PosixPath as PP -from datetime import datetime, timedelta -from typing import List, Dict -import gzip -import logging -import tarfile -import yaml - -import ujson -import boto3 -import statsd # debdeps: python3-statsd - -# debdeps: python3-clickhouse-driver -from clickhouse_driver import Client as Clickhouse - - -metrics = statsd.StatsClient("127.0.0.1", 8125, prefix="ooni_api_uploader") -log = logging.getLogger("ooni_api_uploader") - -try: - from systemd.journal import JournalHandler # debdeps: python3-systemd - - log.addHandler(JournalHandler(SYSLOG_IDENTIFIER="ooni_api_uploader")) -except ImportError: - pass - -log.setLevel(logging.DEBUG) - - -def create_s3_client(conf): - session = boto3.Session( - aws_access_key_id=conf.get("aws_access_key_id"), - aws_secret_access_key=conf.get("aws_secret_access_key"), - ) - return session.resource("s3") - - -def read_conf(): - cf = "/etc/ooni/api-uploader.conf" - log.info(f"Starting. Reading {cf}") - conf = ConfigParser() - conf.read(cf) - return conf["DEFAULT"] - - -def connect_to_db(conf): - default = "clickhouse://api:api@localhost/default" - uri = conf.get("db_uri", default) - log.debug(f"Connecting to database on {uri}") - conn = Clickhouse.from_url(uri) - return conn - - -@metrics.timer("update_db_table") -def update_db_table(conn, lookup_list, jsonl_s3path): - for d in lookup_list: - d["s3path"] = jsonl_s3path - - q = "INSERT INTO jsonl (report_id, input, s3path, linenum, measurement_uid) VALUES" - log.info(f"Writing {len(lookup_list)} rows to DB") - conn.execute(q, lookup_list) - - -@metrics.timer("upload_measurement") -def upload_to_s3(s3, bucket_name: str, tarf: PP, s3path: str) -> None: - obj = s3.Object(bucket_name, s3path) - log.info(f"Uploading {s3path}") - obj.put(Body=tarf.read_bytes()) - - -@metrics.timer("fill_postcan") -def fill_postcan(hourdir: PP, postcanf: PP) -> List[PP]: - msmt_files = sorted(f for f in hourdir.iterdir() if f.suffix == ".post") - if not msmt_files: - log.info(f"Nothing to fill {postcanf.name}") - return [] - log.info(f"Filling {postcanf.name}") - measurements = [] - postcan_byte_thresh = 20 * 1000 * 1000 - # Open postcan - with tarfile.open(str(postcanf), "w:gz") as tar: - for msmt_f in msmt_files: - # Add a msmt and delete the msmt file - metrics.incr("msmt_count") - tar.add(str(msmt_f)) - measurements.append(msmt_f) - tarsize = postcanf.stat().st_size - if tarsize > postcan_byte_thresh: - log.info(f"Reached {tarsize} bytes") - return measurements - - return measurements - - -@metrics.timer("fill_jsonl") -def fill_jsonl(measurements: List[PP], jsonlf: PP) -> List[Dict]: - log.info(f"Filling {jsonlf.name}") - # report_id, input, 2020092119_IT_tor.n0.0.jsonl.gz - lookup_list = [] - with gzip.open(jsonlf, "w") as jf: - for linenum, msmt_f in enumerate(measurements): - try: - post = ujson.load(msmt_f.open()) - except Exception: - log.error("Unable to parse measurement") - jf.write(b"{}\n") - continue - - fmt = post.get("format", "").lower() - msm = None - if fmt == "json": - msm = post.get("content", {}) - elif fmt == "yaml": - try: - msm = yaml.load(msm, Loader=yaml.CLoader) - except Exception: - pass - - if msm is None: - log.error("Unable to parse measurement") - jf.write(b"{}\n") - continue - - jf.write(ujson.dumps(msm).encode()) - jf.write(b"\n") - - rid = msm.get("report_id") or "" - input = msm.get("input") or "" - msmt_uid = msmt_f.name[:-5] - d = dict( - report_id=rid, input=input, measurement_uid=msmt_uid, linenum=linenum - ) - lookup_list.append(d) - - return lookup_list - - -def delete_msmt_posts(measurements: List[PP]) -> None: - log.info(f"Deleting {len(measurements)} measurements") - for msmt_f in measurements: - msmt_f.unlink() - - -@metrics.timer("total_run_time") -def main(): - conf = read_conf() - bucket_name = conf.get("bucket_name") - spooldir = Path(conf.get("msmt_spool_dir")) - format_char = "n" - collector_id = conf.get("collector_id") - identity = f"{format_char}{collector_id}" - log.info(f"Uploader {collector_id} starting") - assert collector_id, "collector_id is not set" - log.info(f"Using bucket {bucket_name} and spool {spooldir}") - - s3 = create_s3_client(conf) - - db_conn = connect_to_db(conf) - - # Scan spool directories, by age - idir = spooldir / "incoming" - threshold = datetime.utcnow() - timedelta(hours=1) - for hourdir in sorted(idir.iterdir()): - if not hourdir.is_dir() or hourdir.suffix == ".tmp": - continue - try: - tstamp, cc, testname = hourdir.name.split("_") - except Exception: - continue - if len(tstamp) != 10: - continue - hourdir_time = datetime.strptime(tstamp, "%Y%m%d%H") - if hourdir_time > threshold: - log.info(f"Stopping before {hourdir_time}") - break - - log.info(f"Processing {hourdir}") - # Split msmts across multiple postcans and jsonl files - can_cnt = 0 - while True: - # Compress raw POSTs into a tar.gz postcan - postcanf = hourdir.with_suffix(f".{identity}.{can_cnt}.tar.gz") - jsonlf = hourdir.with_suffix(f".{identity}.{can_cnt}.jsonl.gz") - msmfiles = fill_postcan(hourdir, postcanf) - if len(msmfiles) == 0: - break - # Also create jsonl file and delete msmt POSTs - lookup_list = fill_jsonl(msmfiles, jsonlf) - delete_msmt_posts(msmfiles) - - # Upload current postcan to S3 - postcan_s3path = ( - f"raw/{tstamp[:8]}/{tstamp[8:10]}/{cc}/{testname}/{postcanf.name}" - ) - jsonl_s3path = ( - f"raw/{tstamp[:8]}/{tstamp[8:10]}/{cc}/{testname}/{jsonlf.name}" - ) - - upload_to_s3(s3, bucket_name, postcanf, postcan_s3path) - upload_to_s3(s3, bucket_name, jsonlf, jsonl_s3path) - update_db_table(db_conn, lookup_list, jsonl_s3path) - - postcanf.unlink() - jsonlf.unlink() - - can_cnt += 1 - metrics.incr("postcan_count") - - # Delete whole hourly directory - for f in sorted(hourdir.iterdir()): - f.unlink() - hourdir.rmdir() - - log.info("Exiting") - - -if __name__ == "__main__": - main() diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 1d9951c1..fe82dc68 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1422,8 +1422,8 @@ module "ooni_anonc" { protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { - from_port = 4646, # for the nomad webui - to_port = 4646, + from_port = 4645, # for the nomad webui + to_port = 4645, protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { From 5b6093deab8f9d8c987269c8917ade68dfb529ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 16 Jul 2026 18:40:27 +0200 Subject: [PATCH 10/41] Use password in dev env --- ansible/roles/nomad/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/nomad/defaults/main.yml b/ansible/roles/nomad/defaults/main.yml index 2c5f42c2..cbe1da49 100644 --- a/ansible/roles/nomad/defaults/main.yml +++ b/ansible/roles/nomad/defaults/main.yml @@ -1,4 +1,4 @@ --- nomad_webui_username: "admin" -nomad_webui_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/nomad_webui_password', profile='oonidevops_user_prod') }}" +nomad_webui_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/nomad_webui_password', profile='oonidevops_user') }}" nomad_htpasswd_path: "/etc/nginx/nomad.htpasswd" From 212bb7ff14de17e415d66d0a1691d93656d9d3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 10:44:18 +0200 Subject: [PATCH 11/41] deactivate internal tls (nginx handles tls) --- ansible/roles/nomad/defaults/main.yml | 2 +- ansible/roles/nomad/tasks/main.yml | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ansible/roles/nomad/defaults/main.yml b/ansible/roles/nomad/defaults/main.yml index cbe1da49..14b4a756 100644 --- a/ansible/roles/nomad/defaults/main.yml +++ b/ansible/roles/nomad/defaults/main.yml @@ -1,4 +1,4 @@ --- nomad_webui_username: "admin" -nomad_webui_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/nomad_webui_password', profile='oonidevops_user') }}" +nomad_webui_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/nomad_webui_password', profile='oonidevops_user_dev') }}" nomad_htpasswd_path: "/etc/nginx/nomad.htpasswd" diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index b0afff82..bd846a42 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -114,6 +114,12 @@ bind_addr = "127.0.0.1" data_dir = "/opt/nomad/data" + advertise { + http = "127.0.0.1" + rpc = "127.0.0.1" + serf = "127.0.0.1" + } + server { enabled = true bootstrap_expect = 1 @@ -130,11 +136,7 @@ tls { http = false - rpc = true - - ca_file = "/var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem" - cert_file = "/var/lib/dehydrated/certs/{{ inventory_hostname }}/cert.pem" - key_file = "/var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem" + rpc = false } mode: '0644' owner: root @@ -148,7 +150,7 @@ name: "{{ nomad_webui_username }}" password: "{{ nomad_webui_password }}" owner: root - group: www-data + group: nginx mode: '0640' - name: Configure Nginx reverse proxy for Nomad From 8c5939b93bc0ee4b04a9bffc70130e6d6ed4cff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 16:36:19 +0200 Subject: [PATCH 12/41] Add nomad provider to terraform --- tf/environments/dev/main.tf | 16 ++++++++++++++++ tf/environments/dev/versions.tf | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index fe82dc68..7055365c 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -21,6 +21,22 @@ locals { ] } +# nomad setup +provider "nomad" { + address = data.aws_ssm_parameter.nomad_address.value + secret_id = data.aws_ssm_parameter.nomad_token.value +} + +data "aws_ssm_parameter" "nomad_address" { + name = "/oonidevops/secrets/nomad_address" + with_decryption = true +} + +data "aws_ssm_parameter" "nomad_token" { + name = "/oonidevops/secrets/nomad_token" + with_decryption = true +} + ## AWS Setup provider "aws" { diff --git a/tf/environments/dev/versions.tf b/tf/environments/dev/versions.tf index a712029e..02fed7dd 100644 --- a/tf/environments/dev/versions.tf +++ b/tf/environments/dev/versions.tf @@ -9,5 +9,9 @@ terraform { source = "hashicorp/aws" version = ">= 4.66.1" } + nomad = { + source = "hashicorp/nomad" + version = "~> 2.0" + } } } From 5eaacd8ce3a225648f534e3f0503501af6acbc22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 17:39:09 +0200 Subject: [PATCH 13/41] Add nomad to terraform --- ansible/roles/nomad/tasks/main.yml | 14 ++++++++++ tf/environments/dev/.terraform.lock.hcl | 20 +++++++++++++++ tf/environments/dev/main.tf | 34 +++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index bd846a42..55f65c27 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -97,6 +97,20 @@ state: present update_cache: yes +- name: Install podman + tags: nomad + ansible.builtin.apt: + name: podman + state: present + update_cache: yes + +- name: Install podman-driver + tags: nomad + ansible.builtin.apt: + name: nomad-driver-podman + state: present + update_cache: yes + - name: Create Nomad configuration directory tags: nomad ansible.builtin.file: diff --git a/tf/environments/dev/.terraform.lock.hcl b/tf/environments/dev/.terraform.lock.hcl index db7b353e..2d7dfbb2 100644 --- a/tf/environments/dev/.terraform.lock.hcl +++ b/tf/environments/dev/.terraform.lock.hcl @@ -111,6 +111,26 @@ provider "registry.terraform.io/hashicorp/local" { ] } +provider "registry.terraform.io/hashicorp/nomad" { + version = "2.6.1" + constraints = "~> 2.0" + hashes = [ + "h1:WeUVRTFX4hjbndw+Iq/hx68Ka8YSvkqwQ5xeRbmZqwc=", + "zh:07444f16538eee2ae210020411f6e95f778b352800cf0398705659c0aa0a733f", + "zh:11112cbfd762cb728d4d7d2e91c7abdaff0b5f0471dde923a3a742f28bb4ff9c", + "zh:38eeb6d81fbd47a71ff3bf3c4db16dd86d4b20f5a1b80d0184b79e7ba07b4183", + "zh:6fac4ff3cc522fd369e03fa4dd8ca03da4c85a25f53110fc60d1b1001946ec8b", + "zh:8bc8f61e496e96c81c46e1aa59bf2155b6acc80db1ea462f2ddd665748fcda7f", + "zh:b749d77be673ecfee60554bbb47c701ba5c90d3e83708605e4954e70edba5a36", + "zh:e32f19f7e278d8cf48e6ae3e60ee197b42df6a134f9467803ba80c6ef21536ef", + "zh:e72d01215582209c789239ddbf7a66ff6d70934f23c22e1286c6dce4c49312bc", + "zh:eb93166cf7eb56a7b3eda13699fa02a7a9cd3acbb10614890a3fd8283ff2be38", + "zh:eba43934462f24d2a027e97447304d0605802291fa6e5a5fa5cea1dcd40377b5", + "zh:f2bfdbd907ae67b22202889fc4cb7656f4a1d01db809cd52d130ba4d5d4940fa", + "zh:fab6b5bc3d3b71cdb2a8946b81b2c92d9dc89bbe0e510863112be841e4cb228c", + ] +} + provider "registry.terraform.io/hashicorp/null" { version = "3.2.3" hashes = [ diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7055365c..7e14f898 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1553,3 +1553,37 @@ resource "aws_route53_record" "jumphost_alias" { module.ooni_jumphost.aws_instance_public_dns ] } + +## Nomad services + +# These are the secrets for oonimeasurements +resource "nomad_variable" "oonimeasurements_secrets" { + path = "nomad/jobs/oonimeasurements" + + items = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + } +} + +resource "nomad_job" "oonimeasurements" { + jobspec = templatefile("${path.module}/jobs/oonimeasurements.tpl", { + docker_image = "ooni/api-oonimeasurements" + task_memory = 256 # in MB + desired_count = 1 + base_url = "https://api.${local.environment}.ooni.io" + s3_bucket_name = "ooni-data-eu-fra-test" + valkey_url = local.ooniapi_valkey_url + rate_limits = "10/minute;400000/day;200000/7day" + # replace required due to template interpolation getting bugged + other_collectors = replace(jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]), "\"", "\\\"") + rate_limits_whitelisted = replace(jsonencode(["5.9.112.244"]), "\"", "\\\"") + rate_limits_unmetered = replace(jsonencode(["/metrics", "/health"]), "\"", "\\\"") + port = 8000 + }) + + depends_on = [nomad_variable.oonimeasurements_secrets] +} From dee04ac6d420c0b60e6b0dd01d7523c5fecbec18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 17:43:47 +0200 Subject: [PATCH 14/41] remove podman --- ansible/roles/nomad/tasks/main.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 55f65c27..bd846a42 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -97,20 +97,6 @@ state: present update_cache: yes -- name: Install podman - tags: nomad - ansible.builtin.apt: - name: podman - state: present - update_cache: yes - -- name: Install podman-driver - tags: nomad - ansible.builtin.apt: - name: nomad-driver-podman - state: present - update_cache: yes - - name: Create Nomad configuration directory tags: nomad ansible.builtin.file: From ba74b1b81ac94f8d0b8f7464fcb0f170c4df5dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 17:56:01 +0200 Subject: [PATCH 15/41] Add job directory with oonimeasurements job --- tf/environments/dev/jobs/oonimeasurements.tpl | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tf/environments/dev/jobs/oonimeasurements.tpl diff --git a/tf/environments/dev/jobs/oonimeasurements.tpl b/tf/environments/dev/jobs/oonimeasurements.tpl new file mode 100644 index 00000000..f881baaf --- /dev/null +++ b/tf/environments/dev/jobs/oonimeasurements.tpl @@ -0,0 +1,53 @@ +job "oonimeasurements" { + type = "service" + group "oonimeasurements" { + count = ${desired_count} + network { + port "oonimeasurements" { + static = ${port} + } + } + service { + name = "oonimeasurements" + port = "oonimeasurements" + provider = "nomad" + } + task "oonimeasurements-task" { + driver = "docker" + config { + image = "${docker_image}" + ports = ["oonimeasurements"] + port_map = { + oonimeasurements = 80 + } + } + // Secrets go here + template { + data = <<-EOH + {{ with nomadVar "nomad/jobs/oonimeasurements" }} + POSTGRESQL_URL={{ .POSTGRESQL_URL }} + JWT_ENCRYPTION_KEY={{ .JWT_ENCRYPTION_KEY }} + PROMETHEUS_METRICS_PASSWORD={{ .PROMETHEUS_METRICS_PASSWORD }} + CLICKHOUSE_URL={{ .CLICKHOUSE_URL }} + ACCOUNT_ID_HASHING_KEY={{ .ACCOUNT_ID_HASHING_KEY }} + {{ end }} + EOH + destination = "secrets/env.txt" + env = true + } + // Normal envs go here + env { + OTHER_COLLECTORS = "${other_collectors}" + BASE_URL = "${base_url}" + S3_BUCKET_NAME = "${s3_bucket_name}" + VALKEY_URL = "${valkey_url}" + RATE_LIMITS = "${rate_limits}" + RATE_LIMITS_WHITELISTED_IPADDRS = "${rate_limits_whitelisted}" + RATE_LIMITS_UNMETERED_PAGES = "${rate_limits_unmetered}" + } + resources { + memory = ${task_memory} + } + } + } +} From 61feb22087378f03ce73427ffd0de11b47369fc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 21:38:02 +0200 Subject: [PATCH 16/41] restore podman --- ansible/roles/nomad/tasks/main.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index bd846a42..55f65c27 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -97,6 +97,20 @@ state: present update_cache: yes +- name: Install podman + tags: nomad + ansible.builtin.apt: + name: podman + state: present + update_cache: yes + +- name: Install podman-driver + tags: nomad + ansible.builtin.apt: + name: nomad-driver-podman + state: present + update_cache: yes + - name: Create Nomad configuration directory tags: nomad ansible.builtin.file: From 284fe431a40e988388bffee1f6a1e3e995a42ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 21:45:50 +0200 Subject: [PATCH 17/41] Add nomad nginx configs --- ansible/roles/nomad/templates/nginx-nomad.j2 | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 ansible/roles/nomad/templates/nginx-nomad.j2 diff --git a/ansible/roles/nomad/templates/nginx-nomad.j2 b/ansible/roles/nomad/templates/nginx-nomad.j2 new file mode 100644 index 00000000..fccc3400 --- /dev/null +++ b/ansible/roles/nomad/templates/nginx-nomad.j2 @@ -0,0 +1,51 @@ +# ansible-managed in ooni/devops.git +# Nomad-specific settings found here: +# https://developer.hashicorp.com/nomad/docs/deploy/clusters/reverse-proxy-ui#extend-connection-timeout + +server { + listen 4645 ssl http2; + + server_name {{ inventory_hostname }}; + include /etc/nginx/ssl_intermediate.conf; + + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; + + location / { + auth_basic "Nomad Admin Area"; + auth_basic_user_file {{ nomad_htpasswd_path }}; + + + # Nomad blocking queries will remain open for a default of 5 minutes. + # Increase the proxy timeout to accommodate this timeout with an + # additional grace period. + proxy_read_timeout 319s; + + # Nomad log streaming uses streaming HTTP requests. In order to + # synchronously stream logs from Nomad to NGINX to the browser + # proxy buffering needs to be turned off. + proxy_buffering off; + + # The Upgrade and Connection headers are used to establish + # a WebSockets connection. + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # The default Origin header will be the proxy address, which + # will be rejected by Nomad. It must be rewritten to be the + # host address instead. + proxy_set_header Origin "${scheme}://${proxy_host}"; + + proxy_pass http://127.0.0.1:4646; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # WebSocket support (required for Nomad's logs/shell/events) + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} From 125406e29dc003825402e7d84d0eff0fc08a9d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 22:06:25 +0200 Subject: [PATCH 18/41] Fix nginx config --- ansible/roles/nomad/templates/nginx-nomad.j2 | 22 +++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/ansible/roles/nomad/templates/nginx-nomad.j2 b/ansible/roles/nomad/templates/nginx-nomad.j2 index fccc3400..786b9c27 100644 --- a/ansible/roles/nomad/templates/nginx-nomad.j2 +++ b/ansible/roles/nomad/templates/nginx-nomad.j2 @@ -1,13 +1,10 @@ # ansible-managed in ooni/devops.git # Nomad-specific settings found here: # https://developer.hashicorp.com/nomad/docs/deploy/clusters/reverse-proxy-ui#extend-connection-timeout - server { listen 4645 ssl http2; - server_name {{ inventory_hostname }}; include /etc/nginx/ssl_intermediate.conf; - ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; @@ -16,6 +13,12 @@ server { auth_basic "Nomad Admin Area"; auth_basic_user_file {{ nomad_htpasswd_path }}; + proxy_pass http://127.0.0.1:4646; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # Nomad blocking queries will remain open for a default of 5 minutes. # Increase the proxy timeout to accommodate this timeout with an @@ -35,17 +38,6 @@ server { # The default Origin header will be the proxy address, which # will be rejected by Nomad. It must be rewritten to be the # host address instead. - proxy_set_header Origin "${scheme}://${proxy_host}"; - - proxy_pass http://127.0.0.1:4646; - - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # WebSocket support (required for Nomad's logs/shell/events) - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; + proxy_set_header Origin "http://127.0.0.1:4646"; } } From d6b054e86b2f79056865cd8640ebef579c075042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 22:13:05 +0200 Subject: [PATCH 19/41] Final settings to support exec on browser --- ansible/roles/nomad/templates/nginx-nomad.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/nomad/templates/nginx-nomad.j2 b/ansible/roles/nomad/templates/nginx-nomad.j2 index 786b9c27..5a19385c 100644 --- a/ansible/roles/nomad/templates/nginx-nomad.j2 +++ b/ansible/roles/nomad/templates/nginx-nomad.j2 @@ -38,6 +38,6 @@ server { # The default Origin header will be the proxy address, which # will be rejected by Nomad. It must be rewritten to be the # host address instead. - proxy_set_header Origin "http://127.0.0.1:4646"; + proxy_set_header Origin "${scheme}://$host"; } } From dfd1e392c1ac5c876ff492379fcc67437c6832e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 17 Jul 2026 23:49:12 +0200 Subject: [PATCH 20/41] Add nginx for api --- ansible/roles/nomad/tasks/main.yml | 12 +++++- ansible/roles/nomad/templates/nginx-api.j2 | 41 +++++++++++++++++++ tf/environments/dev/jobs/oonimeasurements.tpl | 4 +- 3 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 ansible/roles/nomad/templates/nginx-api.j2 diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 55f65c27..4d78329e 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -171,7 +171,17 @@ tags: nomad ansible.builtin.template: src: nginx-nomad.j2 - dest: /etc/nginx/sites-enabled/nomad.conf + dest: /etc/nginx/sites-enabled/02-nomad.conf + owner: root + group: root + mode: '0644' + notify: reload nginx + +- name: Configure Nginx reverse proxy for the API + tags: nomad + ansible.builtin.template: + src: nginx-api.j2 + dest: /etc/nginx/sites-enabled/03-api.conf owner: root group: root mode: '0644' diff --git a/ansible/roles/nomad/templates/nginx-api.j2 b/ansible/roles/nomad/templates/nginx-api.j2 new file mode 100644 index 00000000..a4eac551 --- /dev/null +++ b/ansible/roles/nomad/templates/nginx-api.j2 @@ -0,0 +1,41 @@ +# ansible-managed in ooni/devops.git +upstream oonimeasurements { + server 10.0.0.123:8000; +} + +server { + listen 80; + listen 443 ssl http2; + # this is the correct name, change it in the actual deployment + # server_name api.{{ environment }}.ooni.io; + server_name anonc.dev.ooni.io; + + include /etc/nginx/ssl_intermediate.conf; + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; + + # Redirect plain HTTP to HTTPS + if ($scheme = http) { + return 301 https://$host$request_uri; + } + + # All paths currently route to oonimeasurements. + # Add more location blocks below as you bring up additional services. + location / { + proxy_pass http://oonimeasurements; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Example of how you'll extend this: + # location /something { + # proxy_pass http://otherservice; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # } +} diff --git a/tf/environments/dev/jobs/oonimeasurements.tpl b/tf/environments/dev/jobs/oonimeasurements.tpl index f881baaf..1bb7cfaf 100644 --- a/tf/environments/dev/jobs/oonimeasurements.tpl +++ b/tf/environments/dev/jobs/oonimeasurements.tpl @@ -5,6 +5,7 @@ job "oonimeasurements" { network { port "oonimeasurements" { static = ${port} + to = 80 } } service { @@ -17,9 +18,6 @@ job "oonimeasurements" { config { image = "${docker_image}" ports = ["oonimeasurements"] - port_map = { - oonimeasurements = 80 - } } // Secrets go here template { From fe4a270528f1019034b91a1d6577eae52de9cf25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Sat, 18 Jul 2026 00:04:10 +0200 Subject: [PATCH 21/41] Remove redundant prometheus file --- ansible/roles/nomad/tasks/main.yml | 6 ++++++ ansible/roles/nomad/templates/nginx-api.j2 | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 4d78329e..3ed534e2 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -187,6 +187,12 @@ mode: '0644' notify: reload nginx +- name: Remove a file + tags: nomad + ansible.builtin.file: + path: /etc/nginx/sites-enabled/01-prometheus + state: absent + - name: Ensure Nomad service is enabled and started tags: nomad ansible.builtin.service: diff --git a/ansible/roles/nomad/templates/nginx-api.j2 b/ansible/roles/nomad/templates/nginx-api.j2 index a4eac551..5e046c01 100644 --- a/ansible/roles/nomad/templates/nginx-api.j2 +++ b/ansible/roles/nomad/templates/nginx-api.j2 @@ -20,6 +20,17 @@ server { return 301 https://$host$request_uri; } + location /metrics/node_exporter { + auth_basic "Administrator’s Area"; + auth_basic_user_file /etc/ooni/prometheus_passwd; + + proxy_pass http://127.0.0.1:9100/metrics; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + # All paths currently route to oonimeasurements. # Add more location blocks below as you bring up additional services. location / { From 9e3f3d7d8816192ebf4ca68e33236f34e4b04a87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Sat, 18 Jul 2026 00:15:41 +0200 Subject: [PATCH 22/41] Add anonc to clickhouse proxy rules --- tf/environments/dev/main.tf | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7e14f898..06f7d9f1 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -735,8 +735,21 @@ module "ooni_clickhouse_proxy" { from_port = 9000, to_port = 9002, // for several clickhouse instances protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, ["${module.ooni_fastpath.aws_instance_private_ip}/32", "${module.ooni_fastpath.aws_instance_public_ip}/32"], - ["${module.ooniapi_testlists.aws_instance_private_ip}/32", "${module.ooniapi_testlists.aws_instance_public_ip}/32"]), + cidr_blocks = concat( + module.network.vpc_subnet_private[*].cidr_block, + [ + "${module.ooni_fastpath.aws_instance_private_ip}/32", + "${module.ooni_fastpath.aws_instance_public_ip}/32" + ], + [ + "${module.ooniapi_testlists.aws_instance_private_ip}/32", + "${module.ooniapi_testlists.aws_instance_public_ip}/32" + ], + [ + "${module.ooni_anonc.aws_instance_private_ip}/32", + "${module.ooni_anonc.aws_instance_public_ip}/32", + ] + ), }, { // For the prometheus proxy: from_port = 9200, From 7a1857b57f1ba440518130f588c26578ac1b57fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Sat, 18 Jul 2026 00:29:00 +0200 Subject: [PATCH 23/41] Change clickhouse proxy url --- tf/environments/dev/main.tf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 06f7d9f1..7da631e8 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -37,6 +37,11 @@ data "aws_ssm_parameter" "nomad_token" { with_decryption = true } +data "aws_ssm_parameter" "clickhouse_readonly_url_test" { + name = "/oonidevops/secrets/clickhouse_readonly_test_url" + with_decryption = true +} + ## AWS Setup provider "aws" { @@ -1577,7 +1582,7 @@ resource "nomad_variable" "oonimeasurements_secrets" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url_test.value ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value } } From e72f4f63099b02b64935d4b6b707be51356674ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Sat, 18 Jul 2026 00:37:44 +0200 Subject: [PATCH 24/41] Add health checking --- tf/environments/dev/jobs/oonimeasurements.tpl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tf/environments/dev/jobs/oonimeasurements.tpl b/tf/environments/dev/jobs/oonimeasurements.tpl index 1bb7cfaf..138de613 100644 --- a/tf/environments/dev/jobs/oonimeasurements.tpl +++ b/tf/environments/dev/jobs/oonimeasurements.tpl @@ -12,6 +12,14 @@ job "oonimeasurements" { name = "oonimeasurements" port = "oonimeasurements" provider = "nomad" + + // Health checks + check { + type = "http" + path = "/health" + interval = "15s" + timeout = "5s" + } } task "oonimeasurements-task" { driver = "docker" From 38292a13ee2a70f0e0e9a2f07860ba16fc280c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 12:58:37 +0200 Subject: [PATCH 25/41] Add plugin dir configuration to nomad --- ansible/roles/nomad/tasks/main.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 3ed534e2..8b0114be 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -97,6 +97,14 @@ state: present update_cache: yes +- name: Create Nomad plugins directory + ansible.builtin.file: + path: /opt/nomad/plugins + state: directory + owner: nomad + group: nomad + mode: "0755" + - name: Install podman tags: nomad ansible.builtin.apt: @@ -104,13 +112,26 @@ state: present update_cache: yes -- name: Install podman-driver +- name: Install podman-driver for Nomad tags: nomad ansible.builtin.apt: name: nomad-driver-podman state: present update_cache: yes +- name: Find nomad-driver-podman binary path + ansible.builtin.command: which nomad-driver-podman + register: podman_driver_path + changed_when: false + +- name: Symlink podman driver into Nomad plugins directory + ansible.builtin.file: + src: "{{ podman_driver_path.stdout }}" + dest: /opt/nomad/plugins/nomad-driver-podman + state: link + owner: nomad + group: nomad + - name: Create Nomad configuration directory tags: nomad ansible.builtin.file: @@ -127,6 +148,7 @@ content: | bind_addr = "127.0.0.1" data_dir = "/opt/nomad/data" + plugin_dir = "/opt/nomad/plugins" advertise { http = "127.0.0.1" From 8bb024b85dcb6fd8ec1ad21e19cdd17da85af679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 13:04:54 +0200 Subject: [PATCH 26/41] Add nomad tags --- ansible/roles/nomad/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 8b0114be..99d57077 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -98,6 +98,7 @@ update_cache: yes - name: Create Nomad plugins directory + tags: nomad ansible.builtin.file: path: /opt/nomad/plugins state: directory @@ -120,11 +121,13 @@ update_cache: yes - name: Find nomad-driver-podman binary path + tags: nomad ansible.builtin.command: which nomad-driver-podman register: podman_driver_path changed_when: false - name: Symlink podman driver into Nomad plugins directory + tags: nomad ansible.builtin.file: src: "{{ podman_driver_path.stdout }}" dest: /opt/nomad/plugins/nomad-driver-podman From 3c0aad0433c93a42de25c134b9fe841693cd76fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 13:17:09 +0200 Subject: [PATCH 27/41] Start podman service; setup podman driver --- ansible/roles/nomad/tasks/main.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 99d57077..08b99ae2 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -113,6 +113,15 @@ state: present update_cache: yes +# Required for podman support in nomad +# https://developer.hashicorp.com/nomad/plugins/drivers/podman +- name: Enable and start Podman socket + tags: nomad + ansible.builtin.systemd: + name: podman.socket + enabled: true + state: started + - name: Install podman-driver for Nomad tags: nomad ansible.builtin.apt: @@ -152,7 +161,6 @@ bind_addr = "127.0.0.1" data_dir = "/opt/nomad/data" plugin_dir = "/opt/nomad/plugins" - advertise { http = "127.0.0.1" rpc = "127.0.0.1" @@ -177,6 +185,13 @@ http = false rpc = false } + + # TODO change this to use rootless containers instead + plugin "nomad-driver-podman" { + config { + socket_path = "unix:///run/podman/podman.sock" + } + } mode: '0644' owner: root group: root From 4e392096dfc6a8ccc706ff9ec8ab8980eb04d2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 13:20:32 +0200 Subject: [PATCH 28/41] Change driver from docker to podman --- tf/environments/dev/jobs/oonimeasurements.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/jobs/oonimeasurements.tpl b/tf/environments/dev/jobs/oonimeasurements.tpl index 138de613..33632012 100644 --- a/tf/environments/dev/jobs/oonimeasurements.tpl +++ b/tf/environments/dev/jobs/oonimeasurements.tpl @@ -22,7 +22,7 @@ job "oonimeasurements" { } } task "oonimeasurements-task" { - driver = "docker" + driver = "podman" config { image = "${docker_image}" ports = ["oonimeasurements"] From 28c3ff40599111ba1dc24fec69f41f85bbe2abb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 13:24:00 +0200 Subject: [PATCH 29/41] Add docker prefix to image name --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7da631e8..7e74477d 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1589,7 +1589,7 @@ resource "nomad_variable" "oonimeasurements_secrets" { resource "nomad_job" "oonimeasurements" { jobspec = templatefile("${path.module}/jobs/oonimeasurements.tpl", { - docker_image = "ooni/api-oonimeasurements" + docker_image = "docker.io/ooni/api-oonimeasurements" task_memory = 256 # in MB desired_count = 1 base_url = "https://api.${local.environment}.ooni.io" From 622cd5dfc147bde63a03d5e1abf086aa22f5bed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 13:59:29 +0200 Subject: [PATCH 30/41] Create podman user for rootless containers --- ansible/roles/nomad/defaults/main.yml | 4 ++++ ansible/roles/nomad/tasks/main.yml | 32 +++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/ansible/roles/nomad/defaults/main.yml b/ansible/roles/nomad/defaults/main.yml index 14b4a756..12987f22 100644 --- a/ansible/roles/nomad/defaults/main.yml +++ b/ansible/roles/nomad/defaults/main.yml @@ -2,3 +2,7 @@ nomad_webui_username: "admin" nomad_webui_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/nomad_webui_password', profile='oonidevops_user_dev') }}" nomad_htpasswd_path: "/etc/nginx/nomad.htpasswd" + +# Choose one that it's outside of system users ranges (0-100) +# and human user ranges > 1000 +podman_uid: 64501 diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 08b99ae2..4a711073 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -113,14 +113,42 @@ state: present update_cache: yes +- name: Create podman-svc group with fixed GID + tags: nomad + ansible.builtin.group: + name: podman-svc + gid: "{{ podman_uid }}" + system: yes + +- name: Create podman-svc user with fixed UID + tags: nomad + ansible.builtin.user: + name: podman-svc + uid: "{{ podman_uid }}" + group: podman-svc + system: yes + shell: /usr/sbin/nologin + create_home: yes + +- name: Enable linger for podman-svc + tags: nomad + ansible.builtin.command: loginctl enable-linger podman-svc + changed_when: false + + # Required for podman support in nomad # https://developer.hashicorp.com/nomad/plugins/drivers/podman -- name: Enable and start Podman socket +- name: Enable rootless podman socket tags: nomad ansible.builtin.systemd: name: podman.socket + scope: user enabled: true state: started + become: true + become_user: podman-svc + environment: + XDG_RUNTIME_DIR: "/run/user/{{ podman_uid }}" - name: Install podman-driver for Nomad tags: nomad @@ -189,7 +217,7 @@ # TODO change this to use rootless containers instead plugin "nomad-driver-podman" { config { - socket_path = "unix:///run/podman/podman.sock" + socket_path = "unix:///run/user/{{ podman_uid }}/podman/podman.sock" } } mode: '0644' From 9a12d22ef00259421e9c04576523e35621747fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 14:07:21 +0200 Subject: [PATCH 31/41] Setup service to run as rootless --- ansible/roles/nomad/tasks/main.yml | 7 +++++++ tf/environments/dev/jobs/oonimeasurements.tpl | 1 + 2 files changed, 8 insertions(+) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 4a711073..4b04acad 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -3,6 +3,13 @@ - name: Flush all handlers meta: flush_handlers +- name: Install acl package (required for Ansible become to unprivileged users) + tags: nomad + ansible.builtin.apt: + name: acl + state: present + update_cache: true + - name: Allow Nginx HTTPS traffic for Nomad (Port 4645) become: true tags: nomad diff --git a/tf/environments/dev/jobs/oonimeasurements.tpl b/tf/environments/dev/jobs/oonimeasurements.tpl index 33632012..6a724e4c 100644 --- a/tf/environments/dev/jobs/oonimeasurements.tpl +++ b/tf/environments/dev/jobs/oonimeasurements.tpl @@ -23,6 +23,7 @@ job "oonimeasurements" { } task "oonimeasurements-task" { driver = "podman" + user = "podman-svc" config { image = "${docker_image}" ports = ["oonimeasurements"] From 49e91e5b6ce9696239f2b22586fad4731a42acf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 14:12:48 +0200 Subject: [PATCH 32/41] Fix bug with uid space --- ansible/roles/nomad/tasks/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 4b04acad..725541b4 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -137,6 +137,22 @@ shell: /usr/sbin/nologin create_home: yes +# The following two actions fix a uid remapping bug +# Error processing tar file(exit status 1): potentially insufficient UIDs or GIDs available in user namespace (requested 0:42 for /etc/gshadow): Check /etc/subuid and /etc/subgid: lchown /etc/gshadow: invalid argument +- name: Set subuid range for podman-svc + ansible.builtin.lineinfile: + path: /etc/subuid + line: "podman-svc:100000:65536" + create: yes + regexp: "^podman-svc:" + +- name: Set subgid range for podman-svc + ansible.builtin.lineinfile: + path: /etc/subgid + line: "podman-svc:100000:65536" + create: yes + regexp: "^podman-svc:" + - name: Enable linger for podman-svc tags: nomad ansible.builtin.command: loginctl enable-linger podman-svc From be19062bdb8251ccb934a83ea156d8343db1f408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 14:21:51 +0200 Subject: [PATCH 33/41] Add nomad tags --- ansible/roles/nomad/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml index 725541b4..c34738ff 100644 --- a/ansible/roles/nomad/tasks/main.yml +++ b/ansible/roles/nomad/tasks/main.yml @@ -140,6 +140,7 @@ # The following two actions fix a uid remapping bug # Error processing tar file(exit status 1): potentially insufficient UIDs or GIDs available in user namespace (requested 0:42 for /etc/gshadow): Check /etc/subuid and /etc/subgid: lchown /etc/gshadow: invalid argument - name: Set subuid range for podman-svc + tags: nomad ansible.builtin.lineinfile: path: /etc/subuid line: "podman-svc:100000:65536" @@ -147,6 +148,7 @@ regexp: "^podman-svc:" - name: Set subgid range for podman-svc + tags: nomad ansible.builtin.lineinfile: path: /etc/subgid line: "podman-svc:100000:65536" From 452df97564caa7326567b244f70fa5a006c7962e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 15:08:10 +0200 Subject: [PATCH 34/41] Remove user config in job --- tf/environments/dev/jobs/oonimeasurements.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/tf/environments/dev/jobs/oonimeasurements.tpl b/tf/environments/dev/jobs/oonimeasurements.tpl index 6a724e4c..33632012 100644 --- a/tf/environments/dev/jobs/oonimeasurements.tpl +++ b/tf/environments/dev/jobs/oonimeasurements.tpl @@ -23,7 +23,6 @@ job "oonimeasurements" { } task "oonimeasurements-task" { driver = "podman" - user = "podman-svc" config { image = "${docker_image}" ports = ["oonimeasurements"] From 41890920527d35e18b423680f258f69e641fe7b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Jul 2026 15:48:51 +0200 Subject: [PATCH 35/41] Add template for generic jobs --- tf/environments/dev/jobs/base_job.tpl | 56 ++++++++++++++++++ tf/environments/dev/jobs/oonimeasurements.tpl | 59 ------------------- tf/environments/dev/main.tf | 24 ++++---- 3 files changed, 70 insertions(+), 69 deletions(-) create mode 100644 tf/environments/dev/jobs/base_job.tpl delete mode 100644 tf/environments/dev/jobs/oonimeasurements.tpl diff --git a/tf/environments/dev/jobs/base_job.tpl b/tf/environments/dev/jobs/base_job.tpl new file mode 100644 index 00000000..cbf00d36 --- /dev/null +++ b/tf/environments/dev/jobs/base_job.tpl @@ -0,0 +1,56 @@ +job "${service_name}" { + type = "service" + group "${service_name}" { + count = ${desired_count} + network { + port "${service_name}" { + static = ${port} + to = 80 + } + } + service { + name = "${service_name}" + port = "${service_name}" + provider = "nomad" + + check { + type = "http" + path = "/health" + interval = "15s" + timeout = "5s" + } + } + + task "${service_name}-task" { + driver = "podman" + config { + image = "${docker_image}" + ports = ["${service_name}"] + } + +%{ if length(secret_keys) > 0 } + template { + data = <<-EOH + {{ with nomadVar "nomad/jobs/${service_name}" }} +%{ for key in secret_keys ~} + ${key}={{ .${key} }} +%{ endfor ~} + {{ end }} + EOH + destination = "secrets/env.txt" + env = true + } +%{ endif } + + env { +%{ for key, value in env_vars ~} + ${key} = "${value}" +%{ endfor ~} + } + + resources { + memory = ${task_memory} + } + } + } +} diff --git a/tf/environments/dev/jobs/oonimeasurements.tpl b/tf/environments/dev/jobs/oonimeasurements.tpl deleted file mode 100644 index 33632012..00000000 --- a/tf/environments/dev/jobs/oonimeasurements.tpl +++ /dev/null @@ -1,59 +0,0 @@ -job "oonimeasurements" { - type = "service" - group "oonimeasurements" { - count = ${desired_count} - network { - port "oonimeasurements" { - static = ${port} - to = 80 - } - } - service { - name = "oonimeasurements" - port = "oonimeasurements" - provider = "nomad" - - // Health checks - check { - type = "http" - path = "/health" - interval = "15s" - timeout = "5s" - } - } - task "oonimeasurements-task" { - driver = "podman" - config { - image = "${docker_image}" - ports = ["oonimeasurements"] - } - // Secrets go here - template { - data = <<-EOH - {{ with nomadVar "nomad/jobs/oonimeasurements" }} - POSTGRESQL_URL={{ .POSTGRESQL_URL }} - JWT_ENCRYPTION_KEY={{ .JWT_ENCRYPTION_KEY }} - PROMETHEUS_METRICS_PASSWORD={{ .PROMETHEUS_METRICS_PASSWORD }} - CLICKHOUSE_URL={{ .CLICKHOUSE_URL }} - ACCOUNT_ID_HASHING_KEY={{ .ACCOUNT_ID_HASHING_KEY }} - {{ end }} - EOH - destination = "secrets/env.txt" - env = true - } - // Normal envs go here - env { - OTHER_COLLECTORS = "${other_collectors}" - BASE_URL = "${base_url}" - S3_BUCKET_NAME = "${s3_bucket_name}" - VALKEY_URL = "${valkey_url}" - RATE_LIMITS = "${rate_limits}" - RATE_LIMITS_WHITELISTED_IPADDRS = "${rate_limits_whitelisted}" - RATE_LIMITS_UNMETERED_PAGES = "${rate_limits_unmetered}" - } - resources { - memory = ${task_memory} - } - } - } -} diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7e74477d..2abdf25c 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1588,19 +1588,23 @@ resource "nomad_variable" "oonimeasurements_secrets" { } resource "nomad_job" "oonimeasurements" { - jobspec = templatefile("${path.module}/jobs/oonimeasurements.tpl", { + jobspec = templatefile("${path.module}/jobs/base_job.tpl", { + service_name = "oonimeasurements" docker_image = "docker.io/ooni/api-oonimeasurements" task_memory = 256 # in MB desired_count = 1 - base_url = "https://api.${local.environment}.ooni.io" - s3_bucket_name = "ooni-data-eu-fra-test" - valkey_url = local.ooniapi_valkey_url - rate_limits = "10/minute;400000/day;200000/7day" - # replace required due to template interpolation getting bugged - other_collectors = replace(jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]), "\"", "\\\"") - rate_limits_whitelisted = replace(jsonencode(["5.9.112.244"]), "\"", "\\\"") - rate_limits_unmetered = replace(jsonencode(["/metrics", "/health"]), "\"", "\\\"") - port = 8000 + port = 8000 + secret_keys = keys(nomad_variable.oonimeasurements_secrets.items) + env_vars = { + BASE_URL = "https://api.${local.environment}.ooni.io" + S3_BUCKET_NAME = "ooni-data-eu-fra-test" + VALKEY_URL = local.ooniapi_valkey_url + RATE_LIMITS = "10/minute;400000/day;200000/7day" + # replace required due to template interpolation getting bugged + RATE_LIMITS_WHITELISTED_IPADDRS = replace(jsonencode(["5.9.112.244"]), "\"", "\\\"") + RATE_LIMITS_UNMETERED_PAGES = replace(jsonencode(["/metrics", "/health"]), "\"", "\\\"") + OTHER_COLLECTORS = replace(jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]), "\"", "\\\"") + } }) depends_on = [nomad_variable.oonimeasurements_secrets] From 518dc3a831a94e495853075120df39eb3c612093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 27 Jul 2026 13:00:07 +0200 Subject: [PATCH 36/41] Add valkey job --- tf/environments/dev/jobs/base_job.tpl | 12 ++++++++- tf/environments/dev/jobs/valkey.hcl | 37 +++++++++++++++++++++++++++ tf/environments/dev/main.tf | 5 +++- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 tf/environments/dev/jobs/valkey.hcl diff --git a/tf/environments/dev/jobs/base_job.tpl b/tf/environments/dev/jobs/base_job.tpl index cbf00d36..d9a85a5f 100644 --- a/tf/environments/dev/jobs/base_job.tpl +++ b/tf/environments/dev/jobs/base_job.tpl @@ -22,7 +22,7 @@ job "${service_name}" { } task "${service_name}-task" { - driver = "podman" + driver = "docker" config { image = "${docker_image}" ports = ["${service_name}"] @@ -42,6 +42,16 @@ job "${service_name}" { } %{ endif } + template { + data = <<-EOH + {{ range (nomadService "valkey-svc") }} + VALKEY_URL=valkey://{{ .Address }}:{{ .Port }} + {{ end }} + EOH + destination = "local/valkey.env" + env = true + } + env { %{ for key, value in env_vars ~} ${key} = "${value}" diff --git a/tf/environments/dev/jobs/valkey.hcl b/tf/environments/dev/jobs/valkey.hcl new file mode 100644 index 00000000..0e3dfafe --- /dev/null +++ b/tf/environments/dev/jobs/valkey.hcl @@ -0,0 +1,37 @@ +job "valkey" { + type = "service" + group "valkey" { + count = 1 + + network { + port "valkey" { + to = 6379 + } + } + + task "valkey-task" { + driver = "docker" + config { + image = "docker.io/valkey/valkey:8" + ports = ["valkey"] + } + + service { + name = "valkey-svc" + port = "valkey" + provider = "nomad" + address_mode = "driver" + + check { + type = "tcp" + interval = "10s" + timeout = "2s" + } + } + + resources { + memory = 256 + } + } + } +} diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 2abdf25c..7d62d112 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1598,7 +1598,6 @@ resource "nomad_job" "oonimeasurements" { env_vars = { BASE_URL = "https://api.${local.environment}.ooni.io" S3_BUCKET_NAME = "ooni-data-eu-fra-test" - VALKEY_URL = local.ooniapi_valkey_url RATE_LIMITS = "10/minute;400000/day;200000/7day" # replace required due to template interpolation getting bugged RATE_LIMITS_WHITELISTED_IPADDRS = replace(jsonencode(["5.9.112.244"]), "\"", "\\\"") @@ -1609,3 +1608,7 @@ resource "nomad_job" "oonimeasurements" { depends_on = [nomad_variable.oonimeasurements_secrets] } + +resource "nomad_job" "valkey" { + jobspec = file("${path.module}/jobs/valkey.hcl") +} From 6301f73e9e93ec9cbe82322b89c7e9da5b2fabe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 27 Jul 2026 13:14:58 +0200 Subject: [PATCH 37/41] Add canary deployment to oonimeasurements --- tf/environments/dev/jobs/valkey.hcl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tf/environments/dev/jobs/valkey.hcl b/tf/environments/dev/jobs/valkey.hcl index 0e3dfafe..e7a4a82e 100644 --- a/tf/environments/dev/jobs/valkey.hcl +++ b/tf/environments/dev/jobs/valkey.hcl @@ -9,6 +9,16 @@ job "valkey" { } } + update { + max_parallel = 1 + canary = 1 + min_healthy_time = "10s" + healthy_deadline = "3m" + progress_deadline = "5m" + auto_revert = true + auto_promote = true + } + task "valkey-task" { driver = "docker" config { From 27c1501416d9ebd42e69d1ecc6390da0e9bc2cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 28 Jul 2026 13:19:20 +0200 Subject: [PATCH 38/41] Add var for oonimeasurements tag --- tf/environments/dev/jobs/valkey.hcl | 2 +- tf/environments/dev/main.tf | 6 +++++- tf/environments/dev/service_tags.auto.tfvars.json | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 tf/environments/dev/service_tags.auto.tfvars.json diff --git a/tf/environments/dev/jobs/valkey.hcl b/tf/environments/dev/jobs/valkey.hcl index e7a4a82e..3544bddb 100644 --- a/tf/environments/dev/jobs/valkey.hcl +++ b/tf/environments/dev/jobs/valkey.hcl @@ -17,7 +17,7 @@ job "valkey" { progress_deadline = "5m" auto_revert = true auto_promote = true - } + } task "valkey-task" { driver = "docker" diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7d62d112..e4aa595f 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1587,10 +1587,14 @@ resource "nomad_variable" "oonimeasurements_secrets" { } } +variable "oonimeasurements_tag" { + type = string +} + resource "nomad_job" "oonimeasurements" { jobspec = templatefile("${path.module}/jobs/base_job.tpl", { service_name = "oonimeasurements" - docker_image = "docker.io/ooni/api-oonimeasurements" + docker_image = "docker.io/ooni/api-oonimeasurements:${var.oonimeasurements_tag}" task_memory = 256 # in MB desired_count = 1 port = 8000 diff --git a/tf/environments/dev/service_tags.auto.tfvars.json b/tf/environments/dev/service_tags.auto.tfvars.json new file mode 100644 index 00000000..213ead24 --- /dev/null +++ b/tf/environments/dev/service_tags.auto.tfvars.json @@ -0,0 +1,3 @@ +{ + "oonimeasurements_tag": "20260520-fe913ea8" +} From 598aa4762eb9891c0b7679aac46d34d8e3c88e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 28 Jul 2026 14:09:06 +0200 Subject: [PATCH 39/41] Add dispatch to trigger a deployment of updated services --- .github/workflows/update_service.yml | 72 ++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/update_service.yml diff --git a/.github/workflows/update_service.yml b/.github/workflows/update_service.yml new file mode 100644 index 00000000..7801643e --- /dev/null +++ b/.github/workflows/update_service.yml @@ -0,0 +1,72 @@ +name: Update Service + +on: + repository_dispatch: + types: [deploy] + +concurrency: + # with this group each combination - can run in parallel. + # Requests to the same - get queued + group: deploy-${{ github.event.client_payload.env }}-${{ github.event.client_payload.service }} + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Validate inputs + env: + ENV: ${{ github.event.client_payload.env }} + SERVICE: ${{ github.event.client_payload.service }} + TAG: ${{ github.event.client_payload.tag }} + run: | + case "$ENV" in dev|prod) ;; *) echo "Invalid env: $ENV"; exit 1 ;; esac + case "$SERVICE" in + oonimeasurements|ooniprobe|oonifindings|ooniauth|oonirun) ;; + *) echo "Invalid service: $SERVICE"; exit 1 ;; + esac + if [ -z "$TAG" ]; then echo "Missing tag"; exit 1; fi + + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_PAT }} # needs repo write to commit back + + - name: Update service_tags.auto.tfvars.json + env: + ENV: ${{ github.event.client_payload.env }} + SERVICE: ${{ github.event.client_payload.service }} + TAG: ${{ github.event.client_payload.tag }} + run: | + FILE="tf/environments/${ENV}/service_tags.auto.tfvars.json" + [ -f "$FILE" ] || echo '{}' > "$FILE" + jq --arg key "${SERVICE}_tag" --arg tag "$TAG" \ + '.[$key] = $tag' "$FILE" > tmp.json && mv tmp.json "$FILE" + + - name: Commit and push + env: + ENV: ${{ github.event.client_payload.env }} + SERVICE: ${{ github.event.client_payload.service }} + TAG: ${{ github.event.client_payload.tag }} + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git add "tf/environments/${ENV}/service_tags.auto.tfvars.json" + git commit -m "deploy: ${SERVICE} -> ${TAG} (${ENV})" + git push + + - uses: hashicorp/setup-terraform@v3 + + - name: Terraform init + working-directory: tf/environments/${{ github.event.client_payload.env }} + run: terraform init + + - name: Terraform apply + working-directory: tf/environments/${{ github.event.client_payload.env }} + env: + NOMAD_ADDR: ${{ secrets.NOMAD_ADDR }} + NOMAD_TOKEN: ${{ secrets.NOMAD_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.OONIDEVOPS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.OONIDEVOPS_AWS_SECRET_ACCESS_KEY }} + SERVICE: ${{ github.event.client_payload.service }} + run: | + terraform apply -auto-approve -target="nomad_job.${SERVICE}" From 43a843acce1ac9da417a419b6ccec123663ddd6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 30 Jul 2026 11:06:43 +0200 Subject: [PATCH 40/41] Checkout to the current branch for experiments --- .github/workflows/update_service.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update_service.yml b/.github/workflows/update_service.yml index 7801643e..debdd8d6 100644 --- a/.github/workflows/update_service.yml +++ b/.github/workflows/update_service.yml @@ -30,6 +30,7 @@ jobs: - uses: actions/checkout@v4 with: token: ${{ secrets.GH_PAT }} # needs repo write to commit back + ref: nomad-experiment # TODO remove on actual deploy - name: Update service_tags.auto.tfvars.json env: From 66537e098bd7f0b5874329b4f5f0f83863ee25d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 31 Jul 2026 16:21:11 +0200 Subject: [PATCH 41/41] Cleanup nomad installation --- tf/environments/dev/.terraform.lock.hcl | 20 ---------- tf/environments/dev/main.tf | 50 ------------------------- tf/environments/dev/versions.tf | 4 -- 3 files changed, 74 deletions(-) diff --git a/tf/environments/dev/.terraform.lock.hcl b/tf/environments/dev/.terraform.lock.hcl index 2d7dfbb2..db7b353e 100644 --- a/tf/environments/dev/.terraform.lock.hcl +++ b/tf/environments/dev/.terraform.lock.hcl @@ -111,26 +111,6 @@ provider "registry.terraform.io/hashicorp/local" { ] } -provider "registry.terraform.io/hashicorp/nomad" { - version = "2.6.1" - constraints = "~> 2.0" - hashes = [ - "h1:WeUVRTFX4hjbndw+Iq/hx68Ka8YSvkqwQ5xeRbmZqwc=", - "zh:07444f16538eee2ae210020411f6e95f778b352800cf0398705659c0aa0a733f", - "zh:11112cbfd762cb728d4d7d2e91c7abdaff0b5f0471dde923a3a742f28bb4ff9c", - "zh:38eeb6d81fbd47a71ff3bf3c4db16dd86d4b20f5a1b80d0184b79e7ba07b4183", - "zh:6fac4ff3cc522fd369e03fa4dd8ca03da4c85a25f53110fc60d1b1001946ec8b", - "zh:8bc8f61e496e96c81c46e1aa59bf2155b6acc80db1ea462f2ddd665748fcda7f", - "zh:b749d77be673ecfee60554bbb47c701ba5c90d3e83708605e4954e70edba5a36", - "zh:e32f19f7e278d8cf48e6ae3e60ee197b42df6a134f9467803ba80c6ef21536ef", - "zh:e72d01215582209c789239ddbf7a66ff6d70934f23c22e1286c6dce4c49312bc", - "zh:eb93166cf7eb56a7b3eda13699fa02a7a9cd3acbb10614890a3fd8283ff2be38", - "zh:eba43934462f24d2a027e97447304d0605802291fa6e5a5fa5cea1dcd40377b5", - "zh:f2bfdbd907ae67b22202889fc4cb7656f4a1d01db809cd52d130ba4d5d4940fa", - "zh:fab6b5bc3d3b71cdb2a8946b81b2c92d9dc89bbe0e510863112be841e4cb228c", - ] -} - provider "registry.terraform.io/hashicorp/null" { version = "3.2.3" hashes = [ diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index a9a2d445..f74317d9 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -21,11 +21,6 @@ locals { ] } -# nomad setup -provider "nomad" { - address = data.aws_ssm_parameter.nomad_address.value - secret_id = data.aws_ssm_parameter.nomad_token.value -} data "aws_ssm_parameter" "nomad_address" { name = "/oonidevops/secrets/nomad_address" @@ -1571,48 +1566,3 @@ resource "aws_route53_record" "jumphost_alias" { module.ooni_jumphost.aws_instance_public_dns ] } - -## Nomad services - -# These are the secrets for oonimeasurements -resource "nomad_variable" "oonimeasurements_secrets" { - path = "nomad/jobs/oonimeasurements" - - items = { - POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value - JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value - PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url_test.value - ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value - } -} - -variable "oonimeasurements_tag" { - type = string -} - -resource "nomad_job" "oonimeasurements" { - jobspec = templatefile("${path.module}/jobs/base_job.tpl", { - service_name = "oonimeasurements" - docker_image = "docker.io/ooni/api-oonimeasurements:${var.oonimeasurements_tag}" - task_memory = 256 # in MB - desired_count = 1 - port = 8000 - secret_keys = keys(nomad_variable.oonimeasurements_secrets.items) - env_vars = { - BASE_URL = "https://api.${local.environment}.ooni.io" - S3_BUCKET_NAME = "ooni-data-eu-fra-test" - RATE_LIMITS = "10/minute;400000/day;200000/7day" - # replace required due to template interpolation getting bugged - RATE_LIMITS_WHITELISTED_IPADDRS = replace(jsonencode(["5.9.112.244"]), "\"", "\\\"") - RATE_LIMITS_UNMETERED_PAGES = replace(jsonencode(["/metrics", "/health"]), "\"", "\\\"") - OTHER_COLLECTORS = replace(jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]), "\"", "\\\"") - } - }) - - depends_on = [nomad_variable.oonimeasurements_secrets] -} - -resource "nomad_job" "valkey" { - jobspec = file("${path.module}/jobs/valkey.hcl") -} diff --git a/tf/environments/dev/versions.tf b/tf/environments/dev/versions.tf index 02fed7dd..a712029e 100644 --- a/tf/environments/dev/versions.tf +++ b/tf/environments/dev/versions.tf @@ -9,9 +9,5 @@ terraform { source = "hashicorp/aws" version = ">= 4.66.1" } - nomad = { - source = "hashicorp/nomad" - version = "~> 2.0" - } } }