Feature/4162 system recovery single script

ansible/provision/create_app_server.yml

@@ -7,10 +7,12 @@
   vars:
     digitalocean_token: "{{ lookup('ansible.builtin.env', 'DIGITALOCEAN_TOKEN') }}"
     server_config: "{{ lookup('ansible.builtin.file', '../../cloud-init_userdata/doaj-app_cloud-config.txt') }}"
+    droplet_size: s-4vcpu-8gb
     droplet_name: doaj-public-3
     # We initially want this server accessible via public2.doaj.cottagelabs.com
     dns_prefix: public-3
     dns_domain: doaj.cottagelabs.com
+    server_group: app
 
   tasks:
 
@@ -21,7 +23,7 @@
         name: '{{droplet_name}}'
         region: lon1
         #size: c-4   # 4 CPUs optimised
-        size: s-4vcpu-8gb   # 4 CPUs general purpose
+        size: "{{ droplet_size }}"   # 4 CPUs general purpose
         #size: s-2vcpu-4gb
         tags:
           - doaj
@@ -62,7 +64,7 @@
 
     - add_host:
         name: "{{ public_ip }}"
-        groups: app
+        groups: "{{ server_group }}"
 
     - name: Output a summary including login command
       ansible.builtin.debug:

ansible/provision/create_index_server.yml

@@ -0,0 +1,88 @@
+---
+- name: Bring up a new DOAJ index (OpenSearch) server
+  hosts: localhost
+  connection: local
+  gather_facts: true
+
+  vars:
+    digitalocean_token: "{{ lookup('ansible.builtin.env', 'DIGITALOCEAN_TOKEN') }}"
+    server_config: "{{ lookup('ansible.builtin.file', '../../cloud-init_userdata/doaj-opensearch_cloud-config.txt') }}"
+    droplet_size: m3-4vcpu-32gb
+    droplet_name: doaj-index-rw1
+    dns_prefix: index-rw1
+    dns_domain: doaj.cottagelabs.com
+    server_group: index
+    opensearch_cluster_name: doaj-index-rw
+    opensearch_seed_hosts: []
+    opensearch_s3_backup: false
+
+  tasks:
+
+    - name: Create Droplet with root access Steve and Rama, using opensearch server userdata
+      digitalocean.cloud.droplet:
+        state: present
+        token: "{{ digitalocean_token }}"
+        name: '{{ droplet_name }}'
+        region: lon1
+        size: "{{ droplet_size }}"
+        tags:
+          - doaj
+          - firewall-doaj-index
+        image: ubuntu-22-04-x64
+        # Run task list_ssh_keys.yml to get the key IDs
+        ssh_keys: ["30242381", "40223915"]
+        ipv6: true
+        unique_name: true
+        monitoring: true
+        backups: false
+        with_droplet_agent: true
+        user_data: "{{ server_config }}"
+
+    - name: Get info on our new droplet to get the IP
+      digitalocean.cloud.droplets_info:
+        name: '{{ droplet_name }}'
+      register: dropletinfo
+
+    - name: Parse out the assigned IP
+      ansible.builtin.set_fact:
+        public_ip: "{{ dropletinfo['droplets'][0]['networks']['v4'] | selectattr('type', '==', 'public') | map(attribute='ip_address') | first }}"
+        droplet_id: "{{ dropletinfo['droplets'][0]['id'] }}"
+
+    - name: Create domain A record
+      digitalocean.cloud.domain_record:
+        token: "{{ digitalocean_token }}"
+        domain_name: '{{ dns_domain }}'
+        type: A
+        name: '{{ dns_prefix }}'
+        data: "{{ public_ip }}"
+
+    - name: Assign the new droplet to the DOAJ project using doctl
+      local_action: ansible.builtin.command doctl projects resources assign f7668431-327e-47d8-9d7e-73e713fe1d4d --resource=do:droplet:{{ droplet_id }}
+
+    - set_fact:
+        remote_host_name: "{{ droplet_name }}"
+
+    - add_host:
+        name: "{{ public_ip }}"
+        groups: "{{ server_group }}"
+
+    - name: Output a summary including login command
+      ansible.builtin.debug:
+        msg: "Index server created: cloo@{{ public_ip }}"
+
+    - name: Give 7 minutes for the server to be up
+      pause:
+        minutes: 7
+
+- name: Install OpenSearch on the new index server
+  hosts: index
+  gather_facts: yes
+  vars:
+    ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+    ansible_user: "cloo"
+    opensearch_cluster_name: "{{ hostvars.localhost.opensearch_cluster_name }}"
+    opensearch_seed_hosts: "{{ hostvars.localhost.opensearch_seed_hosts }}"
+    opensearch_s3_backup: "{{ hostvars.localhost.opensearch_s3_backup }}"
+  tasks:
+    - name: Install opensearch
+      include_tasks: tasks/install_opensearch.yml

ansible/provision/create_servers.yml

@@ -0,0 +1,174 @@
+#
+# Unified playbook to create one or more DOAJ servers.
+#
+# Usage:
+#   # Create all servers:
+#   ansible-playbook create_servers.yml --private-key=~/.ssh/id_rsa -vv
+#
+#   # Create only specific server types (comma-separated):
+#   ansible-playbook create_servers.yml -e "server_types=app,lb" --private-key=~/.ssh/id_rsa -vv
+#
+#   # Create an app server with custom name and DNS:
+#   ansible-playbook create_servers.yml -e "server_types=app app_droplet_name=doaj-public-1 app_dns_prefix=public-1" --private-key=~/.ssh/id_rsa -vv
+#
+#
+# App server variables (override with -e):
+#   app_droplet_name  - Droplet name (default: doaj-public-3)
+#   app_dns_prefix    - DNS prefix for <prefix>.doaj.cottagelabs.com (default: public-3)
+#   app_dns_domain    - DNS domain (default: doaj.cottagelabs.com)
+#   app_size          - Droplet size slug (default: s-4vcpu-8gb)
+#   app_git_branch    - Git branch to deploy (default: master)
+#   app_deploy_env    - Deployment environment (default: production)
+#
+---
+
+###############################################################################
+# Play 0: Determine which servers to create
+###############################################################################
+- name: Determine which servers to create
+  hosts: localhost
+  connection: local
+  gather_facts: false
+
+  vars:
+    # By default create all servers; override with -e "server_types=app,lb"
+    server_types: "all"
+    all_types:
+      - app1
+      - app2
+      - app3
+      - editor
+      - background
+      - lb
+      - kibana
+      - index_rw1
+      - index_rw2
+      - index_r1
+      - index_r2
+
+    # App server variables (override with -e)
+    app_droplet_name: doaj-public-3
+    app_dns_prefix: public-3
+    app_dns_domain: doaj.cottagelabs.com
+    app_size: s-4vcpu-8gb
+    app_git_branch: master
+    app_deploy_env: production
+
+  tasks:
+    - name: Set list of server types to create
+      ansible.builtin.set_fact:
+        create_types: "{{ (server_types == 'all') | ternary(all_types, server_types.split(',') | map('trim') | list) }}"
+        app_droplet_name: "{{ app_droplet_name }}"
+        app_dns_prefix: "{{ app_dns_prefix }}"
+        app_dns_domain: "{{ app_dns_domain }}"
+        app_size: "{{ app_size }}"
+        app_git_branch: "{{ app_git_branch }}"
+        app_deploy_env: "{{ app_deploy_env }}"
+
+    - name: Show which servers will be created
+      ansible.builtin.debug:
+        msg: "Will create server(s): {{ create_types | join(', ') }}"
+
+###############################################################################
+# create servers as per server types
+###############################################################################
+- name: Create index server readwrite-1
+  ansible.builtin.import_playbook: create_index_server.yml
+  vars:
+    droplet_name: doaj-index-rw1
+    dns_prefix: index-rw1
+    server_group: index
+    opensearch_cluster_name: doaj-index-rw
+    opensearch_seed_hosts:
+      - index-rw1.doaj.cottagelabs.com
+      - index-rw2.doaj.cottagelabs.com
+  when: "'index_rw1' in hostvars.localhost.create_types"
+
+- name: Create index server readwrite-2
+  ansible.builtin.import_playbook: create_index_server.yml
+  vars:
+    droplet_name: doaj-index-rw2
+    dns_prefix: index-rw2
+    server_group: index
+    opensearch_cluster_name: doaj-index-rw
+    opensearch_seed_hosts:
+      - index-rw1.doaj.cottagelabs.com
+      - index-rw2.doaj.cottagelabs.com
+  when: "'index_rw2' in hostvars.localhost.create_types"
+
+- name: Create index server readonly-1
+  ansible.builtin.import_playbook: create_index_server.yml
+  vars:
+    droplet_name: doaj-index-r1
+    dns_prefix: index-r1
+    server_group: index
+    opensearch_cluster_name: doaj-index-r
+    opensearch_seed_hosts:
+      - index-r1.doaj.cottagelabs.com
+      - index-r2.doaj.cottagelabs.com
+  when: "'index_r1' in hostvars.localhost.create_types"
+
+- name: Create index server readonly-2
+  ansible.builtin.import_playbook: create_index_server.yml
+  vars:
+    droplet_name: doaj-index-r2
+    dns_prefix: index-r2
+    server_group: index
+    opensearch_cluster_name: doaj-index-r
+    opensearch_seed_hosts:
+      - index-r1.doaj.cottagelabs.com
+      - index-r2.doaj.cottagelabs.com
+  when: "'index_r2' in hostvars.localhost.create_types"
+
+- name: Create app server public-1
+  ansible.builtin.import_playbook: create_app_server.yml
+  vars:
+    droplet_name: doaj-public-1
+    dns_prefix: public-1
+    certbot_domain: public-1.doaj.cottagelabs.com
+    server_group: app
+  when: "'app1' in hostvars.localhost.create_types"
+
+- name: Create app server public-2
+  ansible.builtin.import_playbook: create_app_server.yml
+  vars:
+    droplet_name: doaj-public-2
+    dns_prefix: public-2
+    certbot_domain: public-2.doaj.cottagelabs.com
+    server_group: app
+  when: "'app2' in hostvars.localhost.create_types"
+
+- name: Create app server public-3
+  ansible.builtin.import_playbook: create_app_server.yml
+  vars:
+    droplet_name: doaj-public-3
+    dns_prefix: public-3
+    certbot_domain: public-3.doaj.cottagelabs.com
+    server_group: app
+  when: "'app3' in hostvars.localhost.create_types"
+
+- name: Create editor server editor-1
+  ansible.builtin.import_playbook: create_app_server.yml
+  vars:
+    droplet_name: doaj-editor-1
+    dns_prefix: public-3
+    certbot_domain: editor-1.doaj.cottagelabs.com
+    server_group: app
+  when: "'editor' in hostvars.localhost.create_types"
+
+- name: Create background server
+  ansible.builtin.import_playbook: create_app_server.yml
+  vars:
+    droplet_name: doaj-background-1
+    dns_prefix: background-1
+    certbot_domain: background-1.doaj.cottagelabs.com
+    server_group: background
+  when: "'background' in hostvars.localhost.create_types"
+
+- name: Create load balancer server
+  ansible.builtin.import_playbook: create_lb_server.yml
+  when: "'lb' in hostvars.localhost.create_types"
+
+- name: Create kibana server
+  ansible.builtin.import_playbook: create_kibana_system.yml
+  when: "'kibana' in hostvars.localhost.create_types"

ansible/provision/setup_index_cluster_replication.yml

@@ -0,0 +1,93 @@
+#
+# Set up cross-cluster replication from the read-write (leader) OpenSearch cluster
+# to the read-only (follower) OpenSearch cluster.
+#
+# This playbook should be run after all 4 index servers are provisioned and
+# OpenSearch is running on each.
+#
+# Usage:
+#   ansible-playbook -i ../doaj-hosts.ini setup_index_cluster_replication.yml --private-key=~/.ssh/cl-doaj -vv
+#
+# To replicate a specific index, override with:
+#   ansible-playbook -i ../doaj-hosts.ini setup_index_cluster_replication.yml -e "replicate_index=my-index" --private-key=~/.ssh/cl-doaj -vv
+#
+---
+
+- name: Configure the read-only cluster to replicate from the read-write cluster
+  hosts: opensearch-r[0]
+  gather_facts: yes
+  vars:
+    ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+    ansible_user: "cloo"
+    # The read-write (leader) cluster seed host - uses the private network interface
+    leader_seed_host: "index-rw1.doaj.cottagelabs.com:9300"
+    leader_cluster_alias: "doaj-index-rw"
+    replicate_index: ""
+
+  tasks:
+    - name: Get the private IP of this node for API calls
+      ansible.builtin.set_fact:
+        opensearch_api: "http://{{ ansible_eth1.ipv4.address }}:9200"
+
+    - name: Register the leader (read-write) cluster on the follower (read-only) cluster
+      ansible.builtin.uri:
+        url: "{{ opensearch_api }}/_cluster/settings"
+        method: PUT
+        body_format: json
+        body:
+          persistent:
+            cluster:
+              remote:
+                "{{ leader_cluster_alias }}":
+                  seeds:
+                    - "{{ leader_seed_host }}"
+        status_code: 200
+      register: cluster_settings_result
+
+    - name: Show cluster settings result
+      ansible.builtin.debug:
+        var: cluster_settings_result.json
+
+    - name: Start replication for a specific index
+      ansible.builtin.uri:
+        url: "{{ opensearch_api }}/_plugins/_replication/{{ replicate_index }}/_start"
+        method: PUT
+        body_format: json
+        body:
+          remote_cluster: "{{ leader_cluster_alias }}"
+          remote_index: "{{ replicate_index }}"
+        status_code: 200
+      when: replicate_index | length > 0
+      register: replication_result
+
+    - name: Show replication result
+      ansible.builtin.debug:
+        var: replication_result.json
+      when: replicate_index | length > 0
+
+- name: Set the read-only cluster to read-only mode
+  hosts: opensearch-r[0]
+  gather_facts: yes
+  vars:
+    ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+    ansible_user: "cloo"
+
+  tasks:
+    - name: Get the private IP of this node for API calls
+      ansible.builtin.set_fact:
+        opensearch_api: "http://{{ ansible_eth1.ipv4.address }}:9200"
+
+    - name: Set cluster to read-only mode
+      ansible.builtin.uri:
+        url: "{{ opensearch_api }}/_cluster/settings"
+        method: PUT
+        body_format: json
+        body:
+          persistent:
+            cluster.blocks.read_only: true
+        status_code: 200
+      register: readonly_result
+
+    - name: Show read-only setting result
+      ansible.builtin.debug:
+        var: readonly_result.json