Skip to content

PHOENIX-7966 :- Fix flaky HA IT transitClusterRole: expect the settled peer role #2583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lokiore wants to merge 1 commit into
base: PHOENIX-7562-feature-new
Choose a base branch
from
Open

PHOENIX-7966 :- Fix flaky HA IT transitClusterRole: expect the settled peer role #2583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,10 @@ public void testConnectionCreationFailsIfNoActiveCluster() throws Exception {
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
}

CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY);
// role2 = the SETTLED peer role: local ACTIVE_TO_STANDBY drives the peer
// STANDBY -> STANDBY_TO_ACTIVE, so the waitFor must expect the settled state.
CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY,
ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.STANDBY);

try {
Expand Down Expand Up @@ -565,7 +568,10 @@ public void testFailoverConnectionExplicitlyTimeout() throws Exception {
Connection conn = createFailoverConnection();
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);

CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY);
// role2 = the SETTLED peer role: local ACTIVE_TO_STANDBY drives the peer
// STANDBY -> STANDBY_TO_ACTIVE, so the waitFor must expect the settled state.
CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY,
ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.STANDBY);

try {
Expand Down
10 changes: 8 additions & 2 deletions phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupMetricsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,10 @@ public void testStaleCrrDetectedCount() throws Exception {
try (Connection conn = DriverManager.getConnection(CLUSTERS.getJdbcHAUrl(), clientProperties)) {
conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (2, 2)");
conn.commit();
CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY);
// role2 = the SETTLED peer role: local ACTIVE_TO_STANDBY drives the peer
// STANDBY -> STANDBY_TO_ACTIVE, so the waitFor must expect the settled state.
CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY,
ClusterRole.STANDBY_TO_ACTIVE);
// doRefreshHAGroup=false: keep haGroup's CRR snapshot stale on purpose so that
// the next mutation drives StaleClusterRoleRecordException through wrapActionDuringFailover,
// exercising the GLOBAL_HA_STALE_CRR_DETECTED_COUNT increment path.
Expand Down Expand Up @@ -250,7 +253,10 @@ public void testPollerTickCount() throws Exception {
long beforeTicks = GLOBAL_HA_POLLER_TICK_COUNT.getMetric().getValue();
long beforeFailures = GLOBAL_HA_POLLER_TICK_FAILURES.getMetric().getValue();

CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY);
// role2 = the SETTLED peer role: local ACTIVE_TO_STANDBY drives the peer
// STANDBY -> STANDBY_TO_ACTIVE, so the waitFor must expect the settled state.
CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY,
ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.STANDBY);

// Allow at least 2 poller ticks at default interval to land.
Expand Down
17 changes: 15 additions & 2 deletions phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,9 +423,13 @@ public void transitClusterRole(HighAvailabilityGroup haGroup, ClusterRole role1,
* Update HAGroup's clusterRoleRecord info to new roles (Update HAGroupStoreRecord and SYSTEM
* Tables) and then do the refresh of the clusterRoleRecord in HAGroup based on the param
* provided.
* <p>
* Pass the role each cluster SETTLES at, not a transient pre-advance role: local
* ACTIVE_TO_STANDBY drives the peer's listener to advance a STANDBY peer to STANDBY_TO_ACTIVE,
* so naming the pre-advance role makes the equality wait race the autonomous transition.
* @param haGroup the HA group name
* @param role1 cluster role for the first cluster in the group
* @param role2 cluster role for the second cluster in the group
* @param role1 settled cluster role for the first cluster in the group
* @param role2 settled cluster role for the second cluster in the group
*/
public void transitClusterRole(HighAvailabilityGroup haGroup, ClusterRole role1,
ClusterRole role2, boolean doRefreshHAGroup, HighAvailabilityPolicy policy) throws Exception {
Expand Down Expand Up @@ -487,6 +491,9 @@ public void invalidateHAGroupStoreClientForCluster(String haGroupName, Configura

public void transitClusterRoleWithCluster1Down(HighAvailabilityGroup haGroup, ClusterRole role1,
ClusterRole role2) throws Exception {
// Immune to the peer-advance race: the down cluster is client-invalidated (role
// UNKNOWN), so no live peer listener can advance the survivor past the expected
// record; no settled-role naming needed.
String haGroupName = haGroup.getGroupInfo().getName();
final Pair<HAGroupStoreRecord, Stat> currentRecord2 =
haAdmin2.getHAGroupStoreRecordInZooKeeper(haGroupName);
Expand Down Expand Up @@ -519,6 +526,9 @@ public void transitClusterRoleWithCluster1Down(HighAvailabilityGroup haGroup, Cl

public void transitClusterRoleWithCluster2Down(HighAvailabilityGroup haGroup, ClusterRole role1,
ClusterRole role2) throws Exception {
// Immune to the peer-advance race: the down cluster is client-invalidated (role
// UNKNOWN), so no live peer listener can advance the survivor past the expected
// record; no settled-role naming needed.
String haGroupName = haGroup.getGroupInfo().getName();
final Pair<HAGroupStoreRecord, Stat> currentRecord1 =
haAdmin1.getHAGroupStoreRecordInZooKeeper(haGroupName);
Expand Down Expand Up @@ -550,6 +560,9 @@ public void transitClusterRoleWithCluster2Down(HighAvailabilityGroup haGroup, Cl

public void refreshClusterRoleRecordAfterClusterRestart(HighAvailabilityGroup haGroup,
ClusterRole role1, ClusterRole role2, int clusterIndex) throws Exception {
// Both clusters live, so this shares transitClusterRole's peer-advance exposure: a
// transitional role pair must name the SETTLED peer role. Current callers pass steady
// pairs, so they are safe.
String haGroupName = haGroup.getGroupInfo().getName();
// TODO:- Inducing version change, but how would we know that a URL is changed
// as we don't store url in ZNodes so it doesn't increase version for that change
Expand Down
12 changes: 7 additions & 5 deletions phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,14 +421,16 @@ public void testOneClusterATSRoleWithActive() throws Exception {
*/
@Test
public void testOneClusterATSRoleWithStandby() throws Exception {
testOneClusterATSRole(ClusterRole.STANDBY);
}

private void testOneClusterATSRole(ClusterRole otherRole) throws Exception {
testOneClusterATSRole(otherRole, true);
testOneClusterATSRole(ClusterRole.STANDBY, false);
}

private void testOneClusterATSRole(ClusterRole otherRole, boolean doRefresh) throws Exception {
// doRefresh=false: {ACTIVE_TO_STANDBY, STANDBY} (and {ACTIVE_TO_STANDBY, ACTIVE}) is not a
// stable role pair with both clusters live -- a live peer under a local ACTIVE_TO_STANDBY
// autonomously advances (STANDBY -> STANDBY_TO_ACTIVE; ACTIVE keeps the local draining), so
// an equality waitFor against the named transient pair races that advance and intermittently
// times out. Skipping the settle-wait avoids the race; the PARALLEL connection is still
// exercised against the ACTIVE_TO_STANDBY cluster pair and must complete without error.
CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, otherRole, doRefresh,
PARALLEL);
try (Connection conn = getParallelConnection()) {
Expand Down