From 1710c2c2d14e069dab0ed94d70d7ee36a118d33e Mon Sep 17 00:00:00 2001 From: Ramakrishna Sakhamuru Date: Mon, 27 Jul 2026 11:38:13 +0530 Subject: [PATCH 1/6] initial updates for a single system recovery playbook --- ansible/provision/create_app_server.yml | 6 +- ansible/provision/create_servers.yml | 130 ++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 ansible/provision/create_servers.yml diff --git a/ansible/provision/create_app_server.yml b/ansible/provision/create_app_server.yml index f5d0a95..c780364 100644 --- a/ansible/provision/create_app_server.yml +++ b/ansible/provision/create_app_server.yml @@ -7,10 +7,12 @@ vars: digitalocean_token: "{{ lookup('ansible.builtin.env', 'DIGITALOCEAN_TOKEN') }}" server_config: "{{ lookup('ansible.builtin.file', '../../cloud-init_userdata/doaj-app_cloud-config.txt') }}" + droplet_size: s-4vcpu-8gb droplet_name: doaj-public-3 # We initially want this server accessible via public2.doaj.cottagelabs.com dns_prefix: public-3 dns_domain: doaj.cottagelabs.com + server_group: app tasks: @@ -21,7 +23,7 @@ name: '{{droplet_name}}' region: lon1 #size: c-4 # 4 CPUs optimised - size: s-4vcpu-8gb # 4 CPUs general purpose + size: "{{ droplet_size }}" # 4 CPUs general purpose #size: s-2vcpu-4gb tags: - doaj @@ -62,7 +64,7 @@ - add_host: name: "{{ public_ip }}" - groups: app + groups: "{{ server_group }}" - name: Output a summary including login command ansible.builtin.debug: diff --git a/ansible/provision/create_servers.yml b/ansible/provision/create_servers.yml new file mode 100644 index 0000000..aeedcc1 --- /dev/null +++ b/ansible/provision/create_servers.yml @@ -0,0 +1,130 @@ +# +# Unified playbook to create one or more DOAJ servers. +# +# This playbook calls the existing individual server creation playbooks, +# keeping things simple and avoiding duplication. +# +# Usage: +# # Create all servers: +# ansible-playbook create_servers.yml --private-key=~/.ssh/id_rsa -vv +# +# # Create only specific server types (comma-separated): +# ansible-playbook create_servers.yml -e "server_types=app,lb" --private-key=~/.ssh/id_rsa -vv +# +# # Create an app server with custom name and DNS: +# ansible-playbook create_servers.yml -e "server_types=app app_droplet_name=doaj-public-1 app_dns_prefix=public-1" --private-key=~/.ssh/id_rsa -vv +# +# Supported server_types: app, lb, test, kibana, static, maintenance +# +# App server variables (override with -e): +# app_droplet_name - Droplet name (default: doaj-public-3) +# app_dns_prefix - DNS prefix for .doaj.cottagelabs.com (default: public-3) +# app_dns_domain - DNS domain (default: doaj.cottagelabs.com) +# app_size - Droplet size slug (default: s-4vcpu-8gb) +# app_git_branch - Git branch to deploy (default: master) +# app_deploy_env - Deployment environment (default: production) +# +# Configuration: +# To add or remove servers, simply add or remove import_playbook entries below. +# Each entry calls the existing standalone playbook for that server type. +# +--- + +############################################################################### +# Play 0: Determine which servers to create +############################################################################### +- name: Determine which servers to create + hosts: localhost + connection: local + gather_facts: false + + vars: + # By default create all servers; override with -e "server_types=app,lb" + server_types: "all" + all_types: + - app + - app + - lb + - test + - kibana + - static + - maintenance + + # App server variables (override with -e) + app_droplet_name: doaj-public-3 + app_dns_prefix: public-3 + app_dns_domain: doaj.cottagelabs.com + app_size: s-4vcpu-8gb + app_git_branch: master + app_deploy_env: production + + tasks: + - name: Set list of server types to create + ansible.builtin.set_fact: + create_types: "{{ (server_types == 'all') | ternary(all_types, server_types.split(',') | map('trim') | list) }}" + app_droplet_name: "{{ app_droplet_name }}" + app_dns_prefix: "{{ app_dns_prefix }}" + app_dns_domain: "{{ app_dns_domain }}" + app_size: "{{ app_size }}" + app_git_branch: "{{ app_git_branch }}" + app_deploy_env: "{{ app_deploy_env }}" + + - name: Show which servers will be created + ansible.builtin.debug: + msg: "Will create server(s): {{ create_types | join(', ') }}" + +############################################################################### +# Import existing playbooks for each server type +############################################################################### +- name: Create app server public-1 + ansible.builtin.import_playbook: create_app_server.yml + vars: + droplet_name: doaj-public-1 + dns_prefix: public-1 + certbot_domain: public-1.doaj.cottagelabs.com + server_group: app + when: "'app1' in hostvars.localhost.create_types" + +- name: Create app server public-2 + ansible.builtin.import_playbook: create_app_server.yml + vars: + droplet_name: doaj-public-2 + dns_prefix: public-2 + certbot_domain: public-2.doaj.cottagelabs.com + server_group: app + when: "'app2' in hostvars.localhost.create_types" + +- name: Create app server public-3 + ansible.builtin.import_playbook: create_app_server.yml + vars: + droplet_name: doaj-public-3 + dns_prefix: public-3 + certbot_domain: public-3.doaj.cottagelabs.com + server_group: app + when: "'app3' in hostvars.localhost.create_types" + +- name: Create editor server editor-1 + ansible.builtin.import_playbook: create_app_server.yml + vars: + droplet_name: doaj-editor-1 + dns_prefix: public-3 + certbot_domain: editor-1.doaj.cottagelabs.com + server_group: app + when: "'editor' in hostvars.localhost.create_types" + +- name: Create background server + ansible.builtin.import_playbook: create_app_server.yml + vars: + droplet_name: doaj-background-1 + dns_prefix: background-1 + certbot_domain: background-1.doaj.cottagelabs.com + server_group: background + when: "'background' in hostvars.localhost.create_types" + +- name: Create load balancer server + ansible.builtin.import_playbook: create_lb_server.yml + when: "'lb' in hostvars.localhost.create_types" + +- name: Create kibana server + ansible.builtin.import_playbook: create_kibana_system.yml + when: "'kibana' in hostvars.localhost.create_types" From 12582f20e83d076f95bd254a41a1c94e03119566 Mon Sep 17 00:00:00 2001 From: Ramakrishna Sakhamuru Date: Mon, 27 Jul 2026 15:06:10 +0530 Subject: [PATCH 2/6] Added index server playbook --- ansible/provision/create_index_server.yml | 82 +++++++++++++++++++++++ ansible/provision/create_servers.yml | 49 ++++++++++++-- 2 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 ansible/provision/create_index_server.yml diff --git a/ansible/provision/create_index_server.yml b/ansible/provision/create_index_server.yml new file mode 100644 index 0000000..bf89e1b --- /dev/null +++ b/ansible/provision/create_index_server.yml @@ -0,0 +1,82 @@ +--- +- name: Bring up a new DOAJ index (OpenSearch) server + hosts: localhost + connection: local + gather_facts: true + + vars: + digitalocean_token: "{{ lookup('ansible.builtin.env', 'DIGITALOCEAN_TOKEN') }}" + server_config: "{{ lookup('ansible.builtin.file', '../../cloud-init_userdata/doaj-opensearch_cloud-config.txt') }}" + droplet_size: m3-4vcpu-32gb + droplet_name: doaj-index-rw1 + dns_prefix: index-rw1 + dns_domain: doaj.cottagelabs.com + server_group: index + + tasks: + + - name: Create Droplet with root access Steve and Rama, using opensearch server userdata + digitalocean.cloud.droplet: + state: present + token: "{{ digitalocean_token }}" + name: '{{ droplet_name }}' + region: lon1 + size: "{{ droplet_size }}" + tags: + - doaj + - firewall-doaj-index + image: ubuntu-22-04-x64 + # Run task list_ssh_keys.yml to get the key IDs + ssh_keys: ["30242381", "40223915"] + ipv6: true + unique_name: true + monitoring: true + backups: false + with_droplet_agent: true + user_data: "{{ server_config }}" + + - name: Get info on our new droplet to get the IP + digitalocean.cloud.droplets_info: + name: '{{ droplet_name }}' + register: dropletinfo + + - name: Parse out the assigned IP + ansible.builtin.set_fact: + public_ip: "{{ dropletinfo['droplets'][0]['networks']['v4'] | selectattr('type', '==', 'public') | map(attribute='ip_address') | first }}" + droplet_id: "{{ dropletinfo['droplets'][0]['id'] }}" + + - name: Create domain A record + digitalocean.cloud.domain_record: + token: "{{ digitalocean_token }}" + domain_name: '{{ dns_domain }}' + type: A + name: '{{ dns_prefix }}' + data: "{{ public_ip }}" + + - name: Assign the new droplet to the DOAJ project using doctl + local_action: ansible.builtin.command doctl projects resources assign f7668431-327e-47d8-9d7e-73e713fe1d4d --resource=do:droplet:{{ droplet_id }} + + - set_fact: + remote_host_name: "{{ droplet_name }}" + + - add_host: + name: "{{ public_ip }}" + groups: "{{ server_group }}" + + - name: Output a summary including login command + ansible.builtin.debug: + msg: "Index server created: cloo@{{ public_ip }}" + + - name: Give 7 minutes for the server to be up + pause: + minutes: 7 + +- name: Install OpenSearch on the new index server + hosts: index + gather_facts: yes + vars: + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + ansible_user: "cloo" + tasks: + - name: Install opensearch + include_tasks: tasks/install_opensearch.yml diff --git a/ansible/provision/create_servers.yml b/ansible/provision/create_servers.yml index aeedcc1..b4c89ea 100644 --- a/ansible/provision/create_servers.yml +++ b/ansible/provision/create_servers.yml @@ -14,7 +14,7 @@ # # Create an app server with custom name and DNS: # ansible-playbook create_servers.yml -e "server_types=app app_droplet_name=doaj-public-1 app_dns_prefix=public-1" --private-key=~/.ssh/id_rsa -vv # -# Supported server_types: app, lb, test, kibana, static, maintenance +# Supported server_types: app, lb, test, kibana, static, maintenance, index # # App server variables (override with -e): # app_droplet_name - Droplet name (default: doaj-public-3) @@ -42,13 +42,18 @@ # By default create all servers; override with -e "server_types=app,lb" server_types: "all" all_types: - - app - - app + - app1 + - app2 + - app3 + - editor + - background - lb - - test - kibana - - static - maintenance + - index_rw1 + - index_rw2 + - index_r1 + - index_r2 # App server variables (override with -e) app_droplet_name: doaj-public-3 @@ -74,8 +79,40 @@ msg: "Will create server(s): {{ create_types | join(', ') }}" ############################################################################### -# Import existing playbooks for each server type +# create servers as per server types ############################################################################### +- name: Create index server readwrite-1 + ansible.builtin.import_playbook: create_index_server.yml + vars: + droplet_name: doaj-index-rw1 + dns_prefix: index-rw1 + server_group: index + when: "'index_rw1' in hostvars.localhost.create_types" + +- name: Create index server readwrite-2 + ansible.builtin.import_playbook: create_index_server.yml + vars: + droplet_name: doaj-index-rw2 + dns_prefix: index-rw2 + server_group: index + when: "'index_rw2' in hostvars.localhost.create_types" + +- name: Create index server readonly-1 + ansible.builtin.import_playbook: create_index_server.yml + vars: + droplet_name: doaj-index-r1 + dns_prefix: index-r1 + server_group: index + when: "'index_r1' in hostvars.localhost.create_types" + +- name: Create index server readonly-2 + ansible.builtin.import_playbook: create_index_server.yml + vars: + droplet_name: doaj-index-r2 + dns_prefix: index-r2 + server_group: index + when: "'index_r2' in hostvars.localhost.create_types" + - name: Create app server public-1 ansible.builtin.import_playbook: create_app_server.yml vars: From 78a8062fac170515c31b3671c5e9126d9e329054 Mon Sep 17 00:00:00 2001 From: Ramakrishna Sakhamuru Date: Mon, 27 Jul 2026 18:18:41 +0530 Subject: [PATCH 3/6] index servers read only cluster --- ansible/provision/create_index_server.yml | 6 ++ ansible/provision/create_servers.yml | 16 ++++ .../setup_index_cluster_replication.yml | 93 +++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 ansible/provision/setup_index_cluster_replication.yml diff --git a/ansible/provision/create_index_server.yml b/ansible/provision/create_index_server.yml index bf89e1b..fdd0f6f 100644 --- a/ansible/provision/create_index_server.yml +++ b/ansible/provision/create_index_server.yml @@ -12,6 +12,9 @@ dns_prefix: index-rw1 dns_domain: doaj.cottagelabs.com server_group: index + opensearch_cluster_name: doaj-index-rw + opensearch_seed_hosts: [] + opensearch_s3_backup: false tasks: @@ -77,6 +80,9 @@ vars: ansible_ssh_common_args: "-o StrictHostKeyChecking=no" ansible_user: "cloo" + opensearch_cluster_name: "{{ hostvars.localhost.opensearch_cluster_name }}" + opensearch_seed_hosts: "{{ hostvars.localhost.opensearch_seed_hosts }}" + opensearch_s3_backup: "{{ hostvars.localhost.opensearch_s3_backup }}" tasks: - name: Install opensearch include_tasks: tasks/install_opensearch.yml diff --git a/ansible/provision/create_servers.yml b/ansible/provision/create_servers.yml index b4c89ea..8c515e3 100644 --- a/ansible/provision/create_servers.yml +++ b/ansible/provision/create_servers.yml @@ -87,6 +87,10 @@ droplet_name: doaj-index-rw1 dns_prefix: index-rw1 server_group: index + opensearch_cluster_name: doaj-index-rw + opensearch_seed_hosts: + - index-rw1.doaj.cottagelabs.com + - index-rw2.doaj.cottagelabs.com when: "'index_rw1' in hostvars.localhost.create_types" - name: Create index server readwrite-2 @@ -95,6 +99,10 @@ droplet_name: doaj-index-rw2 dns_prefix: index-rw2 server_group: index + opensearch_cluster_name: doaj-index-rw + opensearch_seed_hosts: + - index-rw1.doaj.cottagelabs.com + - index-rw2.doaj.cottagelabs.com when: "'index_rw2' in hostvars.localhost.create_types" - name: Create index server readonly-1 @@ -103,6 +111,10 @@ droplet_name: doaj-index-r1 dns_prefix: index-r1 server_group: index + opensearch_cluster_name: doaj-index-r + opensearch_seed_hosts: + - index-r1.doaj.cottagelabs.com + - index-r2.doaj.cottagelabs.com when: "'index_r1' in hostvars.localhost.create_types" - name: Create index server readonly-2 @@ -111,6 +123,10 @@ droplet_name: doaj-index-r2 dns_prefix: index-r2 server_group: index + opensearch_cluster_name: doaj-index-r + opensearch_seed_hosts: + - index-r1.doaj.cottagelabs.com + - index-r2.doaj.cottagelabs.com when: "'index_r2' in hostvars.localhost.create_types" - name: Create app server public-1 diff --git a/ansible/provision/setup_index_cluster_replication.yml b/ansible/provision/setup_index_cluster_replication.yml new file mode 100644 index 0000000..4af0e57 --- /dev/null +++ b/ansible/provision/setup_index_cluster_replication.yml @@ -0,0 +1,93 @@ +# +# Set up cross-cluster replication from the read-write (leader) OpenSearch cluster +# to the read-only (follower) OpenSearch cluster. +# +# This playbook should be run after all 4 index servers are provisioned and +# OpenSearch is running on each. +# +# Usage: +# ansible-playbook -i ../doaj-hosts.ini setup_index_cluster_replication.yml --private-key=~/.ssh/cl-doaj -vv +# +# To replicate a specific index, override with: +# ansible-playbook -i ../doaj-hosts.ini setup_index_cluster_replication.yml -e "replicate_index=my-index" --private-key=~/.ssh/cl-doaj -vv +# +--- + +- name: Configure the read-only cluster to replicate from the read-write cluster + hosts: opensearch-r[0] + gather_facts: yes + vars: + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + ansible_user: "cloo" + # The read-write (leader) cluster seed host - uses the private network interface + leader_seed_host: "index-rw1.doaj.cottagelabs.com:9300" + leader_cluster_alias: "doaj-index-rw" + replicate_index: "" + + tasks: + - name: Get the private IP of this node for API calls + ansible.builtin.set_fact: + opensearch_api: "http://{{ ansible_eth1.ipv4.address }}:9200" + + - name: Register the leader (read-write) cluster on the follower (read-only) cluster + ansible.builtin.uri: + url: "{{ opensearch_api }}/_cluster/settings" + method: PUT + body_format: json + body: + persistent: + cluster: + remote: + "{{ leader_cluster_alias }}": + seeds: + - "{{ leader_seed_host }}" + status_code: 200 + register: cluster_settings_result + + - name: Show cluster settings result + ansible.builtin.debug: + var: cluster_settings_result.json + + - name: Start replication for a specific index + ansible.builtin.uri: + url: "{{ opensearch_api }}/_plugins/_replication/{{ replicate_index }}/_start" + method: PUT + body_format: json + body: + remote_cluster: "{{ leader_cluster_alias }}" + remote_index: "{{ replicate_index }}" + status_code: 200 + when: replicate_index | length > 0 + register: replication_result + + - name: Show replication result + ansible.builtin.debug: + var: replication_result.json + when: replicate_index | length > 0 + +- name: Set the read-only cluster to read-only mode + hosts: opensearch-r[0] + gather_facts: yes + vars: + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + ansible_user: "cloo" + + tasks: + - name: Get the private IP of this node for API calls + ansible.builtin.set_fact: + opensearch_api: "http://{{ ansible_eth1.ipv4.address }}:9200" + + - name: Set cluster to read-only mode + ansible.builtin.uri: + url: "{{ opensearch_api }}/_cluster/settings" + method: PUT + body_format: json + body: + persistent: + cluster.blocks.read_only: true + status_code: 200 + register: readonly_result + + - name: Show read-only setting result + ansible.builtin.debug: + var: readonly_result.json From 3248f3dca8b100d083288e1fc44b8aafeeba37e5 Mon Sep 17 00:00:00 2001 From: Ramakrishna Sakhamuru Date: Mon, 27 Jul 2026 18:21:12 +0530 Subject: [PATCH 4/6] Updated documentation --- ansible/provision/create_servers.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ansible/provision/create_servers.yml b/ansible/provision/create_servers.yml index 8c515e3..1b50730 100644 --- a/ansible/provision/create_servers.yml +++ b/ansible/provision/create_servers.yml @@ -2,7 +2,6 @@ # Unified playbook to create one or more DOAJ servers. # # This playbook calls the existing individual server creation playbooks, -# keeping things simple and avoiding duplication. # # Usage: # # Create all servers: @@ -14,7 +13,6 @@ # # Create an app server with custom name and DNS: # ansible-playbook create_servers.yml -e "server_types=app app_droplet_name=doaj-public-1 app_dns_prefix=public-1" --private-key=~/.ssh/id_rsa -vv # -# Supported server_types: app, lb, test, kibana, static, maintenance, index # # App server variables (override with -e): # app_droplet_name - Droplet name (default: doaj-public-3) @@ -24,10 +22,6 @@ # app_git_branch - Git branch to deploy (default: master) # app_deploy_env - Deployment environment (default: production) # -# Configuration: -# To add or remove servers, simply add or remove import_playbook entries below. -# Each entry calls the existing standalone playbook for that server type. -# --- ############################################################################### From 49227f007e8c4445cd66a6faa7aced548fd7e734 Mon Sep 17 00:00:00 2001 From: Ramakrishna Sakhamuru Date: Mon, 27 Jul 2026 18:21:53 +0530 Subject: [PATCH 5/6] Updated documentation --- ansible/provision/create_servers.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/provision/create_servers.yml b/ansible/provision/create_servers.yml index 1b50730..b6debd3 100644 --- a/ansible/provision/create_servers.yml +++ b/ansible/provision/create_servers.yml @@ -1,8 +1,6 @@ # # Unified playbook to create one or more DOAJ servers. # -# This playbook calls the existing individual server creation playbooks, -# # Usage: # # Create all servers: # ansible-playbook create_servers.yml --private-key=~/.ssh/id_rsa -vv From d5b5aa96db41ef25e989d0ba82288c411c8117cd Mon Sep 17 00:00:00 2001 From: Ramakrishna Sakhamuru Date: Mon, 27 Jul 2026 18:27:04 +0530 Subject: [PATCH 6/6] removed unnecessary server type --- ansible/provision/create_servers.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/provision/create_servers.yml b/ansible/provision/create_servers.yml index b6debd3..a4bdb23 100644 --- a/ansible/provision/create_servers.yml +++ b/ansible/provision/create_servers.yml @@ -41,7 +41,6 @@ - background - lb - kibana - - maintenance - index_rw1 - index_rw2 - index_r1