Cortex writes dashboard: add panels for KV activity

For HA-tracker, show which tenants are changing election.
For ingester, show how many are active, leaving, etc.

Author: bboreham

cortex-mixin/dashboards/writes.libsonnet

@@ -110,6 +110,16 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')])
       )
+      .addPanel(
+        $.panel('Elected replica changes / min') +
+        $.queryPanel([
+          'max by(exported_cluster, user)(increase(cortex_ha_tracker_elected_replica_changes_total{%s}[1m])) >0' % $.jobMatcher($._config.job_names.distributor),
+        ], [
+          '{{user}}/{{exported_cluster}}',
+        ]) +
+        $.stack + {
+          yaxes: $.yaxes('cpm'),
+        },
       )
     )
     .addRow(
@@ -138,8 +148,15 @@ local utils = import 'mixin-utils/utils.libsonnet';
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester)+ [utils.selector.eq('kv_name', 'ingester-lifecycler')])
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.eq('kv_name', 'ingester-lifecycler')])
       )
+      .addPanel(
+        $.panel('Ingester status') +
+        $.queryPanel([
+          'max by (state)(cortex_ring_members{%s}) >0' % $.jobMatcher($._config.job_names.distributor),
+        ], [
+          '{{state}}',
+        ])
       )
     )
     .addRowIf(