From dd9c8e7b39772efb7a784a632531a86d918b575a Mon Sep 17 00:00:00 2001 From: Laura Hinson Date: Fri, 17 Jul 2026 11:24:26 -0400 Subject: [PATCH] [OSDOCS-16926]: CQA: etcd performance (2 of 3) --- etcd/etcd-performance.adoc | 4 ++-- modules/etcd-consensus-latency.adoc | 7 ++++-- modules/etcd-defrag-about.adoc | 2 +- modules/etcd-defrag-automatic.adoc | 4 ++-- modules/etcd-defrag.adoc | 8 +++---- modules/move-etcd-different-disk.adoc | 32 ++++++++++++++------------- 6 files changed, 31 insertions(+), 26 deletions(-) diff --git a/etcd/etcd-performance.adoc b/etcd/etcd-performance.adoc index c64b76daa425..26452c6d861d 100644 --- a/etcd/etcd-performance.adoc +++ b/etcd/etcd-performance.adoc @@ -48,9 +48,9 @@ include::modules/move-etcd-different-disk.adoc[leveloffset=+1] //Defragmenting etcd data include::modules/etcd-defrag-about.adoc[leveloffset=+1] -include::modules/etcd-defrag-automatic.adoc[leveloffset=+1] +include::modules/etcd-defrag-automatic.adoc[leveloffset=+2] -include::modules/etcd-defrag.adoc[leveloffset=+1] +include::modules/etcd-defrag.adoc[leveloffset=+2] //Setting tuning parameters for etcd include::modules/etcd-tuning-parameters.adoc[leveloffset=+1] diff --git a/modules/etcd-consensus-latency.adoc b/modules/etcd-consensus-latency.adoc index 66f306c122fe..99cd988adcab 100644 --- a/modules/etcd-consensus-latency.adoc +++ b/modules/etcd-consensus-latency.adoc @@ -6,6 +6,9 @@ [id="etcd-consensus-latency_{context}"] = Monitoring consensus latency for etcd +[role="_abstract"] +Use the `etcdctl` command-line interface (CLI) to check endpoint health and consensus latency on a running cluster. Regular monitoring helps you spot delays before they cause leader elections and Kubernetes API instability. + By using the `etcdctl` CLI, you can monitor the latency for reaching consensus as experienced by etcd. You must identify one of the etcd pods and then retrieve the endpoint health. This procedure, which validates and monitors cluster health, can be run only on an active cluster. @@ -32,7 +35,7 @@ etcd-m1 4/4 Running 4 8h etcd-m2 4/4 Running 4 8h ---- -. Enter the following command. To better understand the etcd latency for consensus, you can run this command on a precise watch cycle for a few minutes to observe that the numbers remain below the ~66 ms threshold. The closer the consensus time is to 100 ms, the more likely the cluster will experience service-affecting events and instability. +. Enter the following command. To better understand the etcd latency for consensus, run this command on a precise watch cycle for a few minutes. Observe that the numbers remain below the ~66 ms threshold. The closer the consensus time is to 100 ms, the more likely the cluster experiences service-affecting events and instability. + [source,terminal] ---- @@ -68,4 +71,4 @@ etcd-m2 4/4 Running 4 8h | https://198.18.111.13:2379 | true | 4.628054ms | | | https://198.18.111.14:2379 | true | 5.803378ms | | +----------------------------+--------+-------------+-------+ ----- \ No newline at end of file +---- diff --git a/modules/etcd-defrag-about.adoc b/modules/etcd-defrag-about.adoc index d9c589638107..9741b9dff170 100644 --- a/modules/etcd-defrag-about.adoc +++ b/modules/etcd-defrag-about.adoc @@ -10,7 +10,7 @@ [role="_abstract"] To prevent etcd performance degradation and cluster-wide maintenance alarms on large clusters, monitor etcd database metrics and defragment the data store when the keyspace grows too large. -For large and dense clusters, etcd can suffer from poor performance if the keyspace grows too large and exceeds the space quota. Periodically maintain and defragment etcd to free up space in the data store. Monitor Prometheus for etcd metrics and defragment it when required; otherwise, etcd can raise a cluster-wide alarm that puts the cluster into a maintenance mode that accepts only key reads and deletes. +For large and dense clusters, etcd can suffer from poor performance if the keyspace grows too large and exceeds the space quota. Periodically maintain and defragment etcd to free up space in the data store. Monitor Prometheus for etcd metrics and defragment it when required. Otherwise, etcd can raise a cluster-wide alarm that puts the cluster into a maintenance mode, which accepts only key reads and deletes. Monitor these key metrics: diff --git a/modules/etcd-defrag-automatic.adoc b/modules/etcd-defrag-automatic.adoc index 7ab91ad084a2..8077d512791d 100644 --- a/modules/etcd-defrag-automatic.adoc +++ b/modules/etcd-defrag-automatic.adoc @@ -12,12 +12,12 @@ When etcd database growth affects performance, the etcd Operator can automatical [NOTE] ==== -Automatic defragmentation is good for most cases, because the etcd operator uses cluster information to determine the most efficient operation for the user. +Automatic defragmentation works well in most cases because the etcd Operator uses cluster metrics to choose the most efficient defragmentation approach. ==== The etcd Operator automatically defragments disks. No manual intervention is needed. -Verify that the defragmentation process is successful by viewing one of these logs: +Verify that defragmentation succeeded by checking one of these logs: * etcd logs * cluster-etcd-operator pod diff --git a/modules/etcd-defrag.adoc b/modules/etcd-defrag.adoc index 2fcb556f2fee..1a120c5ba174 100644 --- a/modules/etcd-defrag.adoc +++ b/modules/etcd-defrag.adoc @@ -5,10 +5,10 @@ :_mod-docs-content-type: PROCEDURE [id="manual-defrag-etcd-data_{context}"] -= Manual defragmentation += Manually defragmenting etcd data [role="_abstract"] -When automatic ectd defragmentation cannot reclaim enough space, manually defragment etcd on each member to restore disk availability and normal cluster operation. +When automatic etcd defragmentation cannot reclaim enough space, manually defragment etcd on each member to restore disk availability and normal cluster operation. //You can monitor the `etcd_db_total_size_in_bytes` metric to determine whether manual defragmentation is necessary. @@ -21,7 +21,7 @@ You can also determine whether defragmentation is needed by checking the etcd da [WARNING] ==== -Defragmenting etcd is a blocking action. The etcd member will not respond until defragmentation is complete. For this reason, wait at least one minute between defragmentation actions on each of the pods to allow the cluster to recover. +Defragmenting etcd is a blocking action. The etcd member does not respond until defragmentation is complete. For this reason, wait at least one minute between defragmentation actions on each of the pods to allow the cluster to recover. ==== Follow this procedure to defragment etcd data on each etcd member. @@ -129,7 +129,7 @@ This example shows that the database size for this etcd member is now 41 MB as o .. Repeat these steps to connect to each of the other etcd members and defragment them. Always defragment the leader last. + -Wait at least one minute between defragmentation actions to allow the etcd pod to recover. Until the etcd pod recovers, the etcd member will not respond. +Wait at least one minute between defragmentation actions to allow the etcd pod to recover. Until the etcd pod recovers, the etcd member does not respond. . If any `NOSPACE` alarms were triggered due to the space quota being exceeded, clear them. diff --git a/modules/move-etcd-different-disk.adoc b/modules/move-etcd-different-disk.adoc index 9a0fa7f4a76a..c0eea1a12894 100644 --- a/modules/move-etcd-different-disk.adoc +++ b/modules/move-etcd-different-disk.adoc @@ -6,6 +6,9 @@ [id="move-etcd-different-disk_{context}"] = Moving etcd to a different disk +[role="_abstract"] +Move etcd data from a shared disk to a dedicated disk to resolve or prevent performance problems. Isolating etcd storage reduces latency from competing I/O on the control plane. + You can move etcd from a shared disk to a separate disk to prevent or resolve performance issues. The Machine Config Operator (MCO) is responsible for mounting a secondary disk for {product-title} {product-version} container storage. @@ -14,22 +17,20 @@ The Machine Config Operator (MCO) is responsible for mounting a secondary disk f ==== This encoded script only supports device names for the following device types: -SCSI or SATA:: `/dev/sd*` -Virtual device:: `/dev/vd*` -NVMe:: `/dev/nvme*[0-9]\*n*` +* SCSI or SATA: `/dev/sd*` +* Virtual device: `/dev/vd*` +* NVMe: `/dev/nvme*[0-9]\*n*` ==== -.Limitations - -* When the new disk is attached to the cluster, the etcd database is part of the root mount. It is not part of the secondary disk or the intended disk when the primary node is recreated. As a result, the primary node will not create a separate `/var/lib/etcd` mount. +When the new disk is attached to the cluster, the etcd database is part of the root mount. It is not part of the secondary disk or the intended disk when the primary node is recreated. As a result, the primary node does not create a separate `/var/lib/etcd` mount. .Prerequisites * You have a backup of your cluster's etcd data. * You have installed the {oc-first}. * You have access to the cluster with `cluster-admin` privileges. -* Add additional disks before uploading the machine configuration. -* The `MachineConfigPool` must match `metadata.labels[machineconfiguration.openshift.io/role]`. This applies to a controller, worker, or a custom pool. +* You have attached additional disks before uploading the machine configuration. +* The `MachineConfigPool` matches `metadata.labels[machineconfiguration.openshift.io/role]`. This applies to a controller, worker, or a custom pool. [NOTE] ==== @@ -64,7 +65,7 @@ Note the device name of the new disk reported by the `lsblk` command. #!/bin/bash set -uo pipefail -for device in ; do # <1> +for device in ; do /usr/sbin/blkid "${device}" &> /dev/null if [ $? == 2 ]; then echo "secondary device found ${device}" @@ -78,7 +79,8 @@ done echo "Couldn't find secondary block device!" >&2 exit 77 ---- -<1> Replace `` with a shell glob for your block device type. For SCSI or SATA drives, use `/dev/sd*`; for virtual drives, use `/dev/vd*`; for NVMe drives, use `/dev/nvme*[0-9]\*n*`. ++ +Replace `` with a shell glob for your block device type. For SCSI or SATA drives, use `/dev/sd*`; for virtual drives, use `/dev/vd*`; for NVMe drives, use `/dev/nvme*[0-9]\*n*`. . Create a base64-encoded string from the `etcd-find-secondary-device.sh` script and note its contents: + @@ -87,7 +89,7 @@ exit 77 $ base64 -w0 etcd-find-secondary-device.sh ---- -. Create a `MachineConfig` YAML file named `etcd-mc.yml` with contents such as the following: +. Create a `MachineConfig` YAML file named `etcd-mc.yml` with contents such as the following example: + [source,yaml] ---- @@ -106,7 +108,7 @@ spec: - path: /etc/find-secondary-device mode: 0755 contents: - source: data:text/plain;charset=utf-8;base64, # <1> + source: data:text/plain;charset=utf-8;base64, systemd: units: - name: find-secondary-device.service @@ -180,7 +182,8 @@ spec: [Install] WantedBy=multi-user.target graphical.target ---- -<1> Replace `` with the encoded script contents that you noted. ++ +Replace `` with the encoded script contents that you noted. . Apply the created `MachineConfig` YAML file: + @@ -189,7 +192,7 @@ spec: $ oc create -f etcd-mc.yml ---- -.Verification steps +.Verification * Run the `grep /var/lib/etcd /proc/mounts` command in a debug shell for the node to ensure that the disk is mounted: + @@ -204,7 +207,6 @@ $ oc debug node/ ---- + .Example output -+ [source,terminal] ---- /dev/sdb /var/lib/etcd xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0