From 5c241fdd0e9fd8531af39b0542b896ec19571306 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 14 May 2026 14:24:46 +0200 Subject: [PATCH 1/7] update nftables rule for data2 (clickhouse2) add data2 to inventory add data2 to deploy-clickhouse.yml hosts make data2 a replica --- ansible/deploy-clickhouse.yml | 2 +- ansible/group_vars/clickhouse/vars.yml | 13 ++++++++----- ansible/inventory | 2 ++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index 5ebc7634..50261e34 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -3,7 +3,7 @@ hosts: - notebook1.htz-fsn.prod.ooni.nu - data1.htz-fsn.prod.ooni.nu - # - data2.htz-fsn.prod.ooni.nu + - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu become: true tags: diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 7b48ebe1..2c01d79c 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -2,7 +2,7 @@ nftables_clickhouse_allow: - fqdn: data1.htz-fsn.prod.ooni.nu ip: 142.132.254.225 - fqdn: data2.htz-fsn.prod.ooni.nu - ip: 88.198.54.12 + ip: 23.88.74.249 - fqdn: data3.htz-fsn.prod.ooni.nu ip: 168.119.7.188 - fqdn: notebook1.htz-fsn.prod.ooni.nu @@ -26,7 +26,7 @@ nftables_zookeeper_allow: - fqdn: data1.htz-fsn.prod.ooni.nu ip: 142.132.254.225 - fqdn: data2.htz-fsn.prod.ooni.nu - ip: 88.198.54.12 + ip: 23.88.74.249 - fqdn: data3.htz-fsn.prod.ooni.nu ip: 168.119.7.188 - fqdn: notebook1.htz-fsn.prod.ooni.nu @@ -113,6 +113,9 @@ clickhouse_zookeeper: - node: host: clickhouse1.prod.ooni.io port: 9181 +# - node: +# host: clickhouse2.prod.ooni.io +# port: 9181 - node: host: clickhouse3.prod.ooni.io port: 9181 @@ -131,9 +134,9 @@ clickhouse_remote_servers: - replica: host: clickhouse1.prod.ooni.io port: 9000 - #- replica: - # host: clickhouse2.prod.ooni.io - # port: 9000 + - replica: + host: clickhouse2.prod.ooni.io + port: 9000 - replica: host: clickhouse3.prod.ooni.io port: 9000 diff --git a/ansible/inventory b/ansible/inventory index e48af402..56fe49e2 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -7,6 +7,7 @@ ghs_ams [clickhouse] notebook1.htz-fsn.prod.ooni.nu data1.htz-fsn.prod.ooni.nu +data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu [airflow] @@ -18,6 +19,7 @@ data1.htz-fsn.prod.ooni.nu monitoring.ooni.org notebook1.htz-fsn.prod.ooni.nu data1.htz-fsn.prod.ooni.nu +data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu openvpn1.htz-fsn.prod.ooni.nu openvpn2.htz-fsn.prod.ooni.nu From 2f8137c2d43327958dffaf1458fe07b0b7254d08 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 15 May 2026 11:37:35 +0200 Subject: [PATCH 2/7] node_exporter: use dehydrated when use_https is enabled --- ansible/roles/prometheus_node_exporter/tasks/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index 698b34ac..21726b00 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -5,6 +5,16 @@ - node_exporter when: use_nginx +- ansible.builtin.include_role: + name: dehydrated + tags: + - oonidata + - dehydrated + vars: + ssl_domains: + - "{{ inventory_hostname }}" + when: use_https + - name: create ooni configuration directory ansible.builtin.file: path: "/etc/ooni/" From dffc917f8b13dc18a30289bae512d15428f7df54 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 15 May 2026 12:01:16 +0200 Subject: [PATCH 3/7] add data2 to prometheus.yml --- ansible/roles/prometheus/templates/prometheus.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 41e80eae..22f3bb3f 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -81,6 +81,7 @@ scrape_configs: static_configs: - targets: - https://data1.htz-fsn.prod.ooni.nu/metrics/node_exporter + - https://data2.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://data3.htz-fsn.prod.ooni.nu/metrics/node_exporter - https://notebook1.htz-fsn.prod.ooni.nu/metrics/node_exporter - http://0.do.th.prod.ooni.io:9001/metrics @@ -151,6 +152,7 @@ scrape_configs: static_configs: - targets: - data1.htz-fsn.prod.ooni.nu + - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu - notebook1.htz-fsn.prod.ooni.nu From ac1cdc4e0668b7d5dbf844500453e76938195e6a Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 18 May 2026 16:19:02 +0200 Subject: [PATCH 4/7] throttle replication to 50% of capacity --- ansible/group_vars/clickhouse/vars.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 2c01d79c..7de27f93 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -41,6 +41,9 @@ clickhouse_config: max_server_memory_usage: 0 max_thread_pool_size: 10000 max_server_memory_usage_to_ram_ratio: 0.9 + # for 1GB/s 50% utilization cap + max_replicated_sends_network_bandwidth_for_server: 62500000 + max_replicated_fetches_network_bandwidth_for_server: 62500000 total_memory_profiler_step: 4194304 total_memory_tracker_sample_probability: 0 uncompressed_cache_size: 8589934592 From e46207436e76254c13e1a56b29ce569855ebcb0f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 18 May 2026 17:52:53 +0200 Subject: [PATCH 5/7] re-add default admin profile, missing on notebook1 --- ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu b/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu index 3732f87a..0a2509eb 100644 --- a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu @@ -136,6 +136,8 @@ clickhouse_default_profiles: readonly: 2 write: readonly: 0 + admin: + readonly: 0 clickhouse_version: "24.10.2.80" clickhouse_release_type: stable From a09d8eb18698338e8be202f0578b700b2ece9202 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 18 May 2026 17:53:12 +0200 Subject: [PATCH 6/7] remove clickhouse_version override from notebook1 --- ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu b/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu index 0a2509eb..0f660817 100644 --- a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu @@ -139,7 +139,6 @@ clickhouse_default_profiles: admin: readonly: 0 -clickhouse_version: "24.10.2.80" clickhouse_release_type: stable clickhouse_listen_hosts: - "127.0.0.1" From 6221f8fc4c8662a2b327195cbb13eb7c5f887046 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 18 May 2026 17:56:07 +0200 Subject: [PATCH 7/7] add extra domains to dehydrated --- ansible/host_vars/data1.htz-fsn.prod.ooni.nu | 4 ++++ ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 ansible/host_vars/data1.htz-fsn.prod.ooni.nu diff --git a/ansible/host_vars/data1.htz-fsn.prod.ooni.nu b/ansible/host_vars/data1.htz-fsn.prod.ooni.nu new file mode 100644 index 00000000..ba22bcb0 --- /dev/null +++ b/ansible/host_vars/data1.htz-fsn.prod.ooni.nu @@ -0,0 +1,4 @@ +# configure extra domains for dehydrated +ssl_domains: + - data1.htz-fns.pr:qod.ooni.nu + - airflow.prod.ooni.io diff --git a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu b/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu index 0f660817..d244d2c3 100644 --- a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu @@ -161,3 +161,8 @@ clickhouse_default_users: - "127.0.0.1" profile: write quota: default + +# configure extra domains for dehydrated +ssl_domains: + - notebook1.htz-fns.prod.ooni.nu + - notebook.ooni.org