From 428977f365ee711b0531f43ee25f3b9c24bdbee9 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 26 May 2026 23:10:18 +0800 Subject: [PATCH 1/4] HBASE-30175 Upgrade hbase-server to use junit5 Part19 --- .../replication/SyncReplicationTestBase.java | 4 + .../replication/TestReplicationBase.java | 414 +---------------- .../TestReplicationBaseNoBeforeAll.java | 435 ++++++++++++++++++ .../regionserver/TestBasicWALEntryStream.java | 103 ++--- .../TestBasicWALEntryStreamAsyncFSWAL.java | 31 +- .../TestBasicWALEntryStreamFSHLog.java | 31 +- .../TestDrainReplicationQueuesForStandBy.java | 15 +- .../TestDumpReplicationQueues.java | 54 ++- .../TestGlobalReplicationThrottler.java | 37 +- ...ClusterReplicationEndpointFilterEdits.java | 25 +- .../TestMetaRegionReplicaReplication.java | 60 +-- ...TestRaceWhenCreatingReplicationSource.java | 27 +- ...tRefreshPeerWhileRegionServerRestarts.java | 15 +- .../TestRefreshRecoveredReplication.java | 75 ++- .../TestRegionReplicaReplication.java | 42 +- .../TestReplicationCompressedWAL.java | 32 +- .../regionserver/TestReplicationMarker.java | 36 +- .../regionserver/TestReplicationSink.java | 48 +- .../regionserver/TestReplicationSource.java | 31 +- .../TestReplicationSourceLogQueue.java | 17 +- .../TestReplicationSourceManager.java | 75 ++- .../TestReplicationSourceManagerJoin.java | 37 +- .../TestReplicationThrottler.java | 17 +- .../TestReplicationValueCompressedWAL.java | 31 +- .../regionserver/TestReplicationWALEdits.java | 31 +- .../regionserver/TestReplicator.java | 43 +- .../TestSerialReplicationChecker.java | 40 +- .../TestSerialReplicationEndpoint.java | 38 +- .../TestSyncReplicationShipperQuit.java | 17 +- .../regionserver/TestWALEntrySinkFilter.java | 30 +- .../TestWALEntryStreamCompressionReset.java | 25 +- .../TestWALEntryStreamDifferentCounts.java | 43 +- ...LEntryStreamDifferentCountsAsyncFSWAL.java | 40 +- ...stWALEntryStreamDifferentCountsFSHLog.java | 40 +- .../regionserver/WALEntryStreamTestBase.java | 23 +- 35 files changed, 975 insertions(+), 1087 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBaseNoBeforeAll.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java index f0caa7a02ba8..be1af5b18a5c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java @@ -57,6 +57,8 @@ import org.apache.hadoop.ipc.RemoteException; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; @@ -102,6 +104,7 @@ protected static void initTestingUtility(HBaseTestingUtil util, String zkParent) conf.setBoolean("replication.source.eof.autorecovery", true); } + @BeforeAll @BeforeClass public static void setUp() throws Exception { ZK_UTIL.startMiniZKCluster(); @@ -154,6 +157,7 @@ private static void shutdown(HBaseTestingUtil util) throws Exception { util.shutdownMiniCluster(); } + @AfterAll @AfterClass public static void tearDown() throws Exception { shutdown(UTIL1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java index 5ab593b00ace..02924dcaa616 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java @@ -17,264 +17,15 @@ */ package org.apache.hadoop.hbase.replication; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseTestingUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -import org.apache.hbase.thirdparty.com.google.common.io.Closeables; /** * This class is only a base for other integration-level replication tests. Do not add tests here. * TestReplicationSmallTests is where tests that don't require bring machines up/down should go All * other tests should have their own classes and extend this one */ -public class TestReplicationBase { - private static final Logger LOG = LoggerFactory.getLogger(TestReplicationBase.class); - protected static Connection connection1; - protected static Connection connection2; - protected static Configuration CONF_WITH_LOCALFS; - - protected static Admin hbaseAdmin; - - protected static Table htable1; - protected static Table htable2; - - protected static final HBaseTestingUtil UTIL1 = new HBaseTestingUtil(); - protected static final HBaseTestingUtil UTIL2 = new HBaseTestingUtil(); - protected static Configuration CONF1 = UTIL1.getConfiguration(); - protected static Configuration CONF2 = UTIL2.getConfiguration(); - - protected static int NUM_SLAVES1 = 1; - protected static int NUM_SLAVES2 = 1; - protected static final int NB_ROWS_IN_BATCH = 100; - protected static final int NB_ROWS_IN_BIG_BATCH = NB_ROWS_IN_BATCH * 10; - protected static final long SLEEP_TIME = 500; - protected static final int NB_RETRIES = 50; - protected static AtomicInteger replicateCount = new AtomicInteger(); - protected static volatile List replicatedEntries = Lists.newArrayList(); - - protected static final TableName tableName = TableName.valueOf("test"); - protected static final byte[] famName = Bytes.toBytes("f"); - protected static final byte[] row = Bytes.toBytes("row"); - protected static final byte[] noRepfamName = Bytes.toBytes("norep"); - protected static final String PEER_ID2 = "2"; - - protected boolean isSerialPeer() { - return false; - } - - protected boolean isSyncPeer() { - return false; - } - - protected final void cleanUp() throws IOException, InterruptedException { - // Starting and stopping replication can make us miss new logs, - // rolling like this makes sure the most recent one gets added to the queue - for (JVMClusterUtil.RegionServerThread r : UTIL1.getHBaseCluster().getRegionServerThreads()) { - UTIL1.getAdmin().rollWALWriter(r.getRegionServer().getServerName()); - } - int rowCount = UTIL1.countRows(tableName); - UTIL1.deleteTableData(tableName); - // truncating the table will send one Delete per row to the slave cluster - // in an async fashion, which is why we cannot just call deleteTableData on - // utility2 since late writes could make it to the slave in some way. - // Instead, we truncate the first table and wait for all the Deletes to - // make it to the slave. - Scan scan = new Scan(); - int lastCount = 0; - for (int i = 0; i < NB_RETRIES; i++) { - if (i == NB_RETRIES - 1) { - fail("Waited too much time for truncate"); - } - ResultScanner scanner = htable2.getScanner(scan); - Result[] res = scanner.next(rowCount); - scanner.close(); - if (res.length != 0) { - if (res.length < lastCount) { - i--; // Don't increment timeout if we make progress - } - lastCount = res.length; - LOG.info("Still got " + res.length + " rows"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - } - - protected static void waitForReplication(int expectedRows, int retries) - throws IOException, InterruptedException { - waitForReplication(htable2, expectedRows, retries); - } - - protected static void waitForReplication(Table table, int expectedRows, int retries) - throws IOException, InterruptedException { - Scan scan; - for (int i = 0; i < retries; i++) { - scan = new Scan(); - if (i == retries - 1) { - fail("Waited too much time for normal batch replication"); - } - int count = 0; - try (ResultScanner scanner = table.getScanner(scan)) { - while (scanner.next() != null) { - count++; - } - } - if (count != expectedRows) { - LOG.info("Only got " + count + " rows"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - } - - protected static void loadData(String prefix, byte[] row) throws IOException { - loadData(prefix, row, famName); - } - - protected static void loadData(String prefix, byte[] row, byte[] familyName) throws IOException { - List puts = new ArrayList<>(NB_ROWS_IN_BATCH); - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - Put put = new Put(Bytes.toBytes(prefix + Integer.toString(i))); - put.addColumn(familyName, row, row); - puts.add(put); - } - htable1.put(puts); - } - - protected static void setupConfig(HBaseTestingUtil util, String znodeParent) { - Configuration conf = util.getConfiguration(); - conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodeParent); - // We don't want too many edits per batch sent to the ReplicationEndpoint to trigger - // sufficient number of events. But we don't want to go too low because - // HBaseInterClusterReplicationEndpoint partitions entries into batches and we want - // more than one batch sent to the peer cluster for better testing. - conf.setInt("replication.source.size.capacity", 102400); - conf.setLong("replication.source.sleepforretries", 100); - conf.setInt("hbase.regionserver.maxlogs", 10); - conf.setLong("hbase.master.logcleaner.ttl", 10); - conf.setInt("zookeeper.recovery.retry", 1); - conf.setInt("zookeeper.recovery.retry.intervalmill", 10); - conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); - conf.setInt("replication.stats.thread.period.seconds", 5); - conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); - conf.setLong("replication.sleep.before.failover", 2000); - conf.setInt("replication.source.maxretriesmultiplier", 10); - conf.setFloat("replication.source.ratio", 1.0f); - conf.setBoolean("replication.source.eof.autorecovery", true); - conf.setLong("hbase.serial.replication.waiting.ms", 100); - } - - static void configureClusters(HBaseTestingUtil util1, HBaseTestingUtil util2) { - setupConfig(util1, "/1"); - setupConfig(util2, "/2"); - - Configuration conf2 = util2.getConfiguration(); - conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); - conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); - conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false); - } - - protected static void restartSourceCluster(int numSlaves) throws Exception { - Closeables.close(hbaseAdmin, true); - Closeables.close(htable1, true); - UTIL1.shutdownMiniHBaseCluster(); - UTIL1.restartHBaseCluster(numSlaves); - // Invalidate the cached connection state. - CONF1 = UTIL1.getConfiguration(); - hbaseAdmin = UTIL1.getAdmin(); - Connection connection1 = UTIL1.getConnection(); - htable1 = connection1.getTable(tableName); - } - - static void restartTargetHBaseCluster(int numSlaves) throws Exception { - Closeables.close(htable2, true); - UTIL2.restartHBaseCluster(numSlaves); - // Invalidate the cached connection state - CONF2 = UTIL2.getConfiguration(); - htable2 = UTIL2.getConnection().getTable(tableName); - } - - protected static void createTable(TableName tableName) throws IOException { - TableDescriptor table = TableDescriptorBuilder.newBuilder(tableName) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName).setMaxVersions(100) - .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build(); - UTIL1.createTable(table, HBaseTestingUtil.KEYS_FOR_HBA_CREATE_TABLE); - UTIL2.createTable(table, HBaseTestingUtil.KEYS_FOR_HBA_CREATE_TABLE); - UTIL1.waitUntilAllRegionsAssigned(tableName); - UTIL2.waitUntilAllRegionsAssigned(tableName); - } - - private static void startClusters() throws Exception { - UTIL1.startMiniZKCluster(); - MiniZooKeeperCluster miniZK = UTIL1.getZkCluster(); - LOG.info("Setup first Zk"); - - UTIL2.setZkCluster(miniZK); - LOG.info("Setup second Zk"); - - CONF_WITH_LOCALFS = HBaseConfiguration.create(CONF1); - UTIL1.startMiniCluster(NUM_SLAVES1); - // Have a bunch of slave servers, because inter-cluster shipping logic uses number of sinks - // as a component in deciding maximum number of parallel batches to send to the peer cluster. - UTIL2.startMiniCluster(NUM_SLAVES2); - - connection1 = ConnectionFactory.createConnection(CONF1); - connection2 = ConnectionFactory.createConnection(CONF2); - hbaseAdmin = connection1.getAdmin(); - - createTable(tableName); - htable1 = connection1.getTable(tableName); - htable2 = connection2.getTable(tableName); - } +public class TestReplicationBase extends TestReplicationBaseNoBeforeAll { @BeforeAll @BeforeClass @@ -282,167 +33,4 @@ public static void setUpBeforeClass() throws Exception { configureClusters(UTIL1, UTIL2); startClusters(); } - - private boolean peerExist(String peerId) throws IOException { - return peerExist(peerId, UTIL1); - } - - private boolean peerExist(String peerId, HBaseTestingUtil util) throws IOException { - return util.getAdmin().listReplicationPeers().stream() - .anyMatch(p -> peerId.equals(p.getPeerId())); - } - - // can be override in tests, in case you need to use zk based uri, or the old style uri - protected String getClusterKey(HBaseTestingUtil util) throws Exception { - return util.getRpcConnnectionURI(); - } - - protected final void addPeer(String peerId, TableName tableName) throws Exception { - addPeer(peerId, tableName, UTIL1, UTIL2); - } - - protected final void addPeer(String peerId, TableName tableName, HBaseTestingUtil source, - HBaseTestingUtil target) throws Exception { - if (peerExist(peerId, source)) { - return; - } - ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder() - .setClusterKey(getClusterKey(target)).setSerial(isSerialPeer()) - .setReplicationEndpointImpl(ReplicationEndpointTest.class.getName()); - if (isSyncPeer()) { - FileSystem fs2 = target.getTestFileSystem(); - // The remote wal dir is not important as we do not use it in DA state, here we only need to - // confirm that a sync peer in DA state can still replicate data to remote cluster - // asynchronously. - builder.setReplicateAllUserTables(false) - .setTableCFsMap(ImmutableMap.of(tableName, ImmutableList.of())) - .setRemoteWALDir(new Path("/RemoteWAL") - .makeQualified(fs2.getUri(), fs2.getWorkingDirectory()).toUri().toString()); - } - source.getAdmin().addReplicationPeer(peerId, builder.build()); - } - - @Before - @BeforeEach - public void setUpBase() throws Exception { - addPeer(PEER_ID2, tableName); - } - - protected final void removePeer(String peerId) throws Exception { - removePeer(peerId, UTIL1); - } - - protected final void removePeer(String peerId, HBaseTestingUtil util) throws Exception { - if (peerExist(peerId, util)) { - util.getAdmin().removeReplicationPeer(peerId); - } - } - - @After - @AfterEach - public void tearDownBase() throws Exception { - removePeer(PEER_ID2); - } - - protected static void runSimplePutDeleteTest() throws IOException, InterruptedException { - Put put = new Put(row); - put.addColumn(famName, row, row); - - htable1 = UTIL1.getConnection().getTable(tableName); - htable1.put(put); - - Get get = new Get(row); - for (int i = 0; i < NB_RETRIES; i++) { - if (i == NB_RETRIES - 1) { - fail("Waited too much time for put replication"); - } - Result res = htable2.get(get); - if (res.isEmpty()) { - LOG.info("Row not available"); - Thread.sleep(SLEEP_TIME); - } else { - assertArrayEquals(row, res.value()); - break; - } - } - - Delete del = new Delete(row); - htable1.delete(del); - - get = new Get(row); - for (int i = 0; i < NB_RETRIES; i++) { - if (i == NB_RETRIES - 1) { - fail("Waited too much time for del replication"); - } - Result res = htable2.get(get); - if (res.size() >= 1) { - LOG.info("Row not deleted"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - } - - protected static void runSmallBatchTest() throws IOException, InterruptedException { - // normal Batch tests - loadData("", row); - - Scan scan = new Scan(); - - ResultScanner scanner1 = htable1.getScanner(scan); - Result[] res1 = scanner1.next(NB_ROWS_IN_BATCH); - scanner1.close(); - assertEquals(NB_ROWS_IN_BATCH, res1.length); - - waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES); - } - - protected static void stopAllRegionServers(HBaseTestingUtil util) throws IOException { - List rses = util.getMiniHBaseCluster().getRegionServerThreads().stream() - .map(t -> t.getRegionServer().getServerName()).collect(Collectors.toList()); - for (ServerName rs : rses) { - util.getMiniHBaseCluster().stopRegionServer(rs); - } - } - - @AfterAll - @AfterClass - public static void tearDownAfterClass() throws Exception { - if (htable2 != null) { - htable2.close(); - } - if (htable1 != null) { - htable1.close(); - } - if (hbaseAdmin != null) { - hbaseAdmin.close(); - } - - if (connection2 != null) { - connection2.close(); - } - if (connection1 != null) { - connection1.close(); - } - UTIL2.shutdownMiniCluster(); - UTIL1.shutdownMiniCluster(); - } - - /** - * Custom replication endpoint to keep track of replication status for tests. - */ - public static class ReplicationEndpointTest extends HBaseInterClusterReplicationEndpoint { - public ReplicationEndpointTest() { - replicateCount.set(0); - } - - @Override - public boolean replicate(ReplicateContext replicateContext) { - replicateCount.incrementAndGet(); - replicatedEntries.addAll(replicateContext.getEntries()); - - return super.replicate(replicateContext); - } - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBaseNoBeforeAll.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBaseNoBeforeAll.java new file mode 100644 index 000000000000..fb5ebe5c9193 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBaseNoBeforeAll.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.apache.hbase.thirdparty.com.google.common.io.Closeables; + +/** + * Replication test base class without BeforeAll method, as in some tests we need to do some changes + * before starting clusters. + * @see TestReplicationBase + */ +public class TestReplicationBaseNoBeforeAll { + private static final Logger LOG = LoggerFactory.getLogger(TestReplicationBaseNoBeforeAll.class); + protected static Connection connection1; + protected static Connection connection2; + protected static Configuration CONF_WITH_LOCALFS; + + protected static Admin hbaseAdmin; + + protected static Table htable1; + protected static Table htable2; + + protected static final HBaseTestingUtil UTIL1 = new HBaseTestingUtil(); + protected static final HBaseTestingUtil UTIL2 = new HBaseTestingUtil(); + protected static Configuration CONF1 = UTIL1.getConfiguration(); + protected static Configuration CONF2 = UTIL2.getConfiguration(); + + protected static int NUM_SLAVES1 = 1; + protected static int NUM_SLAVES2 = 1; + protected static final int NB_ROWS_IN_BATCH = 100; + protected static final int NB_ROWS_IN_BIG_BATCH = NB_ROWS_IN_BATCH * 10; + protected static final long SLEEP_TIME = 500; + protected static final int NB_RETRIES = 50; + protected static AtomicInteger replicateCount = new AtomicInteger(); + protected static volatile List replicatedEntries = Lists.newArrayList(); + + protected static final TableName tableName = TableName.valueOf("test"); + protected static final byte[] famName = Bytes.toBytes("f"); + protected static final byte[] row = Bytes.toBytes("row"); + protected static final byte[] noRepfamName = Bytes.toBytes("norep"); + protected static final String PEER_ID2 = "2"; + + protected boolean isSerialPeer() { + return false; + } + + protected boolean isSyncPeer() { + return false; + } + + protected final void cleanUp() throws IOException, InterruptedException { + // Starting and stopping replication can make us miss new logs, + // rolling like this makes sure the most recent one gets added to the queue + for (JVMClusterUtil.RegionServerThread r : UTIL1.getHBaseCluster().getRegionServerThreads()) { + UTIL1.getAdmin().rollWALWriter(r.getRegionServer().getServerName()); + } + int rowCount = UTIL1.countRows(tableName); + UTIL1.deleteTableData(tableName); + // truncating the table will send one Delete per row to the slave cluster + // in an async fashion, which is why we cannot just call deleteTableData on + // utility2 since late writes could make it to the slave in some way. + // Instead, we truncate the first table and wait for all the Deletes to + // make it to the slave. + Scan scan = new Scan(); + int lastCount = 0; + for (int i = 0; i < NB_RETRIES; i++) { + if (i == NB_RETRIES - 1) { + fail("Waited too much time for truncate"); + } + ResultScanner scanner = htable2.getScanner(scan); + Result[] res = scanner.next(rowCount); + scanner.close(); + if (res.length != 0) { + if (res.length < lastCount) { + i--; // Don't increment timeout if we make progress + } + lastCount = res.length; + LOG.info("Still got " + res.length + " rows"); + Thread.sleep(SLEEP_TIME); + } else { + break; + } + } + } + + protected static void waitForReplication(int expectedRows, int retries) + throws IOException, InterruptedException { + waitForReplication(htable2, expectedRows, retries); + } + + protected static void waitForReplication(Table table, int expectedRows, int retries) + throws IOException, InterruptedException { + Scan scan; + for (int i = 0; i < retries; i++) { + scan = new Scan(); + if (i == retries - 1) { + fail("Waited too much time for normal batch replication"); + } + int count = 0; + try (ResultScanner scanner = table.getScanner(scan)) { + while (scanner.next() != null) { + count++; + } + } + if (count != expectedRows) { + LOG.info("Only got " + count + " rows"); + Thread.sleep(SLEEP_TIME); + } else { + break; + } + } + } + + protected static void loadData(String prefix, byte[] row) throws IOException { + loadData(prefix, row, famName); + } + + protected static void loadData(String prefix, byte[] row, byte[] familyName) throws IOException { + List puts = new ArrayList<>(NB_ROWS_IN_BATCH); + for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { + Put put = new Put(Bytes.toBytes(prefix + Integer.toString(i))); + put.addColumn(familyName, row, row); + puts.add(put); + } + htable1.put(puts); + } + + protected static void setupConfig(HBaseTestingUtil util, String znodeParent) { + Configuration conf = util.getConfiguration(); + conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodeParent); + // We don't want too many edits per batch sent to the ReplicationEndpoint to trigger + // sufficient number of events. But we don't want to go too low because + // HBaseInterClusterReplicationEndpoint partitions entries into batches and we want + // more than one batch sent to the peer cluster for better testing. + conf.setInt("replication.source.size.capacity", 102400); + conf.setLong("replication.source.sleepforretries", 100); + conf.setInt("hbase.regionserver.maxlogs", 10); + conf.setLong("hbase.master.logcleaner.ttl", 10); + conf.setInt("zookeeper.recovery.retry", 1); + conf.setInt("zookeeper.recovery.retry.intervalmill", 10); + conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); + conf.setInt("replication.stats.thread.period.seconds", 5); + conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); + conf.setLong("replication.sleep.before.failover", 2000); + conf.setInt("replication.source.maxretriesmultiplier", 10); + conf.setFloat("replication.source.ratio", 1.0f); + conf.setBoolean("replication.source.eof.autorecovery", true); + conf.setLong("hbase.serial.replication.waiting.ms", 100); + } + + protected static void configureClusters(HBaseTestingUtil util1, HBaseTestingUtil util2) { + setupConfig(util1, "/1"); + setupConfig(util2, "/2"); + + Configuration conf2 = util2.getConfiguration(); + conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); + conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); + conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false); + } + + protected static void restartSourceCluster(int numSlaves) throws Exception { + Closeables.close(hbaseAdmin, true); + Closeables.close(htable1, true); + UTIL1.shutdownMiniHBaseCluster(); + UTIL1.restartHBaseCluster(numSlaves); + // Invalidate the cached connection state. + CONF1 = UTIL1.getConfiguration(); + hbaseAdmin = UTIL1.getAdmin(); + Connection connection1 = UTIL1.getConnection(); + htable1 = connection1.getTable(tableName); + } + + static void restartTargetHBaseCluster(int numSlaves) throws Exception { + Closeables.close(htable2, true); + UTIL2.restartHBaseCluster(numSlaves); + // Invalidate the cached connection state + CONF2 = UTIL2.getConfiguration(); + htable2 = UTIL2.getConnection().getTable(tableName); + } + + protected static void createTable(TableName tableName) throws IOException { + TableDescriptor table = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName).setMaxVersions(100) + .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build(); + UTIL1.createTable(table, HBaseTestingUtil.KEYS_FOR_HBA_CREATE_TABLE); + UTIL2.createTable(table, HBaseTestingUtil.KEYS_FOR_HBA_CREATE_TABLE); + UTIL1.waitUntilAllRegionsAssigned(tableName); + UTIL2.waitUntilAllRegionsAssigned(tableName); + } + + protected static void startClusters() throws Exception { + UTIL1.startMiniZKCluster(); + MiniZooKeeperCluster miniZK = UTIL1.getZkCluster(); + LOG.info("Setup first Zk"); + + UTIL2.setZkCluster(miniZK); + LOG.info("Setup second Zk"); + + CONF_WITH_LOCALFS = HBaseConfiguration.create(CONF1); + UTIL1.startMiniCluster(NUM_SLAVES1); + // Have a bunch of slave servers, because inter-cluster shipping logic uses number of sinks + // as a component in deciding maximum number of parallel batches to send to the peer cluster. + UTIL2.startMiniCluster(NUM_SLAVES2); + + connection1 = ConnectionFactory.createConnection(CONF1); + connection2 = ConnectionFactory.createConnection(CONF2); + hbaseAdmin = connection1.getAdmin(); + + createTable(tableName); + htable1 = connection1.getTable(tableName); + htable2 = connection2.getTable(tableName); + } + + private boolean peerExist(String peerId, HBaseTestingUtil util) throws IOException { + return util.getAdmin().listReplicationPeers().stream() + .anyMatch(p -> peerId.equals(p.getPeerId())); + } + + // can be override in tests, in case you need to use zk based uri, or the old style uri + protected String getClusterKey(HBaseTestingUtil util) throws Exception { + return util.getRpcConnnectionURI(); + } + + protected final void addPeer(String peerId, TableName tableName) throws Exception { + addPeer(peerId, tableName, UTIL1, UTIL2); + } + + protected final void addPeer(String peerId, TableName tableName, HBaseTestingUtil source, + HBaseTestingUtil target) throws Exception { + if (peerExist(peerId, source)) { + return; + } + ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder() + .setClusterKey(getClusterKey(target)).setSerial(isSerialPeer()) + .setReplicationEndpointImpl(ReplicationEndpointTest.class.getName()); + if (isSyncPeer()) { + FileSystem fs2 = target.getTestFileSystem(); + // The remote wal dir is not important as we do not use it in DA state, here we only need to + // confirm that a sync peer in DA state can still replicate data to remote cluster + // asynchronously. + builder.setReplicateAllUserTables(false) + .setTableCFsMap(ImmutableMap.of(tableName, ImmutableList.of())) + .setRemoteWALDir(new Path("/RemoteWAL") + .makeQualified(fs2.getUri(), fs2.getWorkingDirectory()).toUri().toString()); + } + source.getAdmin().addReplicationPeer(peerId, builder.build()); + } + + @Before + @BeforeEach + public void setUpBase() throws Exception { + addPeer(PEER_ID2, tableName); + } + + protected final void removePeer(String peerId) throws Exception { + removePeer(peerId, UTIL1); + } + + protected final void removePeer(String peerId, HBaseTestingUtil util) throws Exception { + if (peerExist(peerId, util)) { + util.getAdmin().removeReplicationPeer(peerId); + } + } + + @After + @AfterEach + public void tearDownBase() throws Exception { + removePeer(PEER_ID2); + } + + protected static void runSimplePutDeleteTest() throws IOException, InterruptedException { + Put put = new Put(row); + put.addColumn(famName, row, row); + + htable1 = UTIL1.getConnection().getTable(tableName); + htable1.put(put); + + Get get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i == NB_RETRIES - 1) { + fail("Waited too much time for put replication"); + } + Result res = htable2.get(get); + if (res.isEmpty()) { + LOG.info("Row not available"); + Thread.sleep(SLEEP_TIME); + } else { + assertArrayEquals(row, res.value()); + break; + } + } + + Delete del = new Delete(row); + htable1.delete(del); + + get = new Get(row); + for (int i = 0; i < NB_RETRIES; i++) { + if (i == NB_RETRIES - 1) { + fail("Waited too much time for del replication"); + } + Result res = htable2.get(get); + if (res.size() >= 1) { + LOG.info("Row not deleted"); + Thread.sleep(SLEEP_TIME); + } else { + break; + } + } + } + + protected static void runSmallBatchTest() throws IOException, InterruptedException { + // normal Batch tests + loadData("", row); + + Scan scan = new Scan(); + + ResultScanner scanner1 = htable1.getScanner(scan); + Result[] res1 = scanner1.next(NB_ROWS_IN_BATCH); + scanner1.close(); + assertEquals(NB_ROWS_IN_BATCH, res1.length); + + waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES); + } + + protected static void stopAllRegionServers(HBaseTestingUtil util) throws IOException { + List rses = util.getMiniHBaseCluster().getRegionServerThreads().stream() + .map(t -> t.getRegionServer().getServerName()).collect(Collectors.toList()); + for (ServerName rs : rses) { + util.getMiniHBaseCluster().stopRegionServer(rs); + } + } + + @AfterAll + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (htable2 != null) { + htable2.close(); + } + if (htable1 != null) { + htable1.close(); + } + if (hbaseAdmin != null) { + hbaseAdmin.close(); + } + + if (connection2 != null) { + connection2.close(); + } + if (connection1 != null) { + connection1.close(); + } + UTIL2.shutdownMiniCluster(); + UTIL1.shutdownMiniCluster(); + } + + /** + * Custom replication endpoint to keep track of replication status for tests. + */ + public static class ReplicationEndpointTest extends HBaseInterClusterReplicationEndpoint { + public ReplicationEndpointTest() { + replicateCount.set(0); + } + + @Override + public boolean replicate(ReplicateContext replicateContext) { + replicateCount.incrementAndGet(); + replicatedEntries.addAll(replicateContext.getEntries()); + + return super.replicate(replicateContext); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java index 6d73bc2c79a3..c72f7abf5079 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java @@ -17,14 +17,15 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -32,7 +33,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.NavigableMap; @@ -72,11 +72,8 @@ import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; import org.mockito.Mockito; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; @@ -84,15 +81,13 @@ public abstract class TestBasicWALEntryStream extends WALEntryStreamTestBase { - @Parameter - public boolean isCompressionEnabled; + protected boolean isCompressionEnabled; - @Parameters(name = "{index}: isCompressionEnabled={0}") - public static Iterable data() { - return Arrays.asList(new Object[] { false }, new Object[] { true }); + protected TestBasicWALEntryStream(boolean isCompressionEnabled) { + this.isCompressionEnabled = isCompressionEnabled; } - @Before + @BeforeEach public void setUp() throws Exception { CONF.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, isCompressionEnabled); initWAL(); @@ -106,7 +101,7 @@ private WAL.Entry next(WALEntryStream entryStream) { /** * Tests basic reading of log appends */ - @Test + @TestTemplate public void testAppendsWithRolls() throws Exception { appendToLogAndSync(); long oldPos; @@ -161,7 +156,7 @@ oldPos, log, new MetricsSource("1"), fakeWalGroupId)) { * Tests that if after a stream is opened, more entries come in and then the log is rolled, we * don't mistakenly dequeue the current log thinking we're done with it */ - @Test + @TestTemplate public void testLogRollWhileStreaming() throws Exception { appendToLog("1"); // 2 @@ -198,7 +193,7 @@ public void testLogRollWhileStreaming() throws Exception { * Tests that if writes come in while we have a stream open, we shouldn't miss them */ - @Test + @TestTemplate public void testNewEntriesWhileStreaming() throws Exception { appendToLog("1"); try (WALEntryStream entryStream = @@ -220,7 +215,7 @@ public void testNewEntriesWhileStreaming() throws Exception { } } - @Test + @TestTemplate public void testResumeStreamingFromPosition() throws Exception { long lastPosition = 0; appendToLog("1"); @@ -245,7 +240,7 @@ public void testResumeStreamingFromPosition() throws Exception { * Tests that if we stop before hitting the end of a stream, we can continue where we left off * using the last position */ - @Test + @TestTemplate public void testPosition() throws Exception { long lastPosition = 0; appendEntriesToLogAndSync(3); @@ -264,7 +259,7 @@ public void testPosition() throws Exception { } } - @Test + @TestTemplate public void testEmptyStream() throws Exception { try (WALEntryStream entryStream = new WALEntryStream(logQueue, fs, CONF, 0, log, new MetricsSource("1"), fakeWalGroupId)) { @@ -272,7 +267,7 @@ public void testEmptyStream() throws Exception { } } - @Test + @TestTemplate public void testWALKeySerialization() throws Exception { Map attributes = new HashMap(); attributes.put("foo", Bytes.toBytes("foo-value")); @@ -280,7 +275,7 @@ public void testWALKeySerialization() throws Exception { WALKeyImpl key = new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, EnvironmentEdgeManager.currentTime(), new ArrayList(), 0L, 0L, mvcc, scopes, attributes); - Assert.assertEquals(attributes, key.getExtendedAttributes()); + assertEquals(attributes, key.getExtendedAttributes()); WALProtos.WALKey.Builder builder = key.getBuilder(WALCellCodec.getNoneCompressor()); WALProtos.WALKey serializedKey = builder.build(); @@ -289,14 +284,14 @@ public void testWALKeySerialization() throws Exception { deserializedKey.readFieldsFromPb(serializedKey, WALCellCodec.getNoneUncompressor()); // equals() only checks region name, sequence id and write time - Assert.assertEquals(key, deserializedKey); + assertEquals(key, deserializedKey); // can't use Map.equals() because byte arrays use reference equality - Assert.assertEquals(key.getExtendedAttributes().keySet(), + assertEquals(key.getExtendedAttributes().keySet(), deserializedKey.getExtendedAttributes().keySet()); for (Map.Entry entry : deserializedKey.getExtendedAttributes().entrySet()) { - Assert.assertArrayEquals(key.getExtendedAttribute(entry.getKey()), entry.getValue()); + assertArrayEquals(key.getExtendedAttribute(entry.getKey()), entry.getValue()); } - Assert.assertEquals(key.getReplicationScopes(), deserializedKey.getReplicationScopes()); + assertEquals(key.getReplicationScopes(), deserializedKey.getReplicationScopes()); } private ReplicationSource mockReplicationSource(boolean recovered, Configuration conf) @@ -346,7 +341,7 @@ private ReplicationSourceWALReader createReaderWithBadReplicationFilter(int numF return reader; } - @Test + @TestTemplate public void testReplicationSourceWALReader() throws Exception { appendEntriesToLogAndSync(3); // get ending position @@ -377,7 +372,7 @@ public void testReplicationSourceWALReader() throws Exception { assertEquals("foo", getRow(entryBatch.getWalEntries().get(0))); } - @Test + @TestTemplate public void testReplicationSourceWALReaderWithFailingFilter() throws Exception { appendEntriesToLogAndSync(3); // get ending position @@ -406,7 +401,7 @@ public void testReplicationSourceWALReaderWithFailingFilter() throws Exception { assertEquals(3, entryBatch.getNbRowKeys()); } - @Test + @TestTemplate public void testReplicationSourceWALReaderRecovered() throws Exception { appendEntriesToLogAndSync(10); Path walPath = getQueue().peek(); @@ -439,7 +434,7 @@ public void testReplicationSourceWALReaderRecovered() throws Exception { } // Testcase for HBASE-20206 - @Test + @TestTemplate public void testReplicationSourceWALReaderWrongPosition() throws Exception { appendEntriesToLogAndSync(1); Path walPath = getQueue().peek(); @@ -466,8 +461,8 @@ public String explainFailure() throws Exception { assertEquals(walPath, entryBatch.getLastWalPath()); long walLength = fs.getFileStatus(walPath).getLen(); - assertTrue("Position " + entryBatch.getLastWalPosition() + " is out of range, file length is " - + walLength, entryBatch.getLastWalPosition() <= walLength); + assertTrue(entryBatch.getLastWalPosition() <= walLength, "Position " + + entryBatch.getLastWalPosition() + " is out of range, file length is " + walLength); assertEquals(1, entryBatch.getNbEntries()); assertTrue(entryBatch.isEndOfFile()); @@ -491,7 +486,7 @@ public String explainFailure() throws Exception { assertFalse(entryBatch.isEndOfFile()); } - @Test + @TestTemplate public void testReplicationSourceWALReaderDisabled() throws IOException, InterruptedException, ExecutionException { appendEntriesToLogAndSync(3); @@ -599,7 +594,7 @@ public static int numFailures() { } } - @Test + @TestTemplate public void testReadBeyondCommittedLength() throws IOException, InterruptedException { appendToLog("1"); appendToLog("2"); @@ -626,7 +621,7 @@ public void testReadBeyondCommittedLength() throws IOException, InterruptedExcep * Test removal of 0 length log from logQueue if the source is a recovered source and size of * logQueue is only 1. */ - @Test + @TestTemplate public void testEOFExceptionForRecoveredQueue() throws Exception { // Create a 0 length log. Path emptyLog = new Path("emptyLog"); @@ -657,7 +652,7 @@ public void testEOFExceptionForRecoveredQueue() throws Exception { assertEquals(0, localLogQueue.getQueueSize(fakeWalGroupId)); } - @Test + @TestTemplate public void testEOFExceptionForRecoveredQueueWithMultipleLogs() throws Exception { Configuration conf = new Configuration(CONF); MetricsSource metrics = mock(MetricsSource.class); @@ -682,14 +677,14 @@ public void testEOFExceptionForRecoveredQueueWithMultipleLogs() throws Exception // Create a reader thread. ReplicationSourceWALReader reader = new ReplicationSourceWALReader(fs, conf, localLogQueue, 0, getDummyFilter(), source, fakeWalGroupId); - assertEquals("Initial log queue size is not correct", 2, - localLogQueue.getQueueSize(fakeWalGroupId)); + assertEquals(2, localLogQueue.getQueueSize(fakeWalGroupId), + "Initial log queue size is not correct"); reader.start(); reader.join(); // remove empty log from logQueue. assertEquals(0, localLogQueue.getQueueSize(fakeWalGroupId)); - assertEquals("Log queue should be empty", 0, localLogQueue.getQueueSize(fakeWalGroupId)); + assertEquals(0, localLogQueue.getQueueSize(fakeWalGroupId), "Log queue should be empty"); } private PriorityBlockingQueue getQueue() { @@ -714,7 +709,7 @@ private void appendEntries(WALProvider.Writer writer, int numEntries) throws IOE /*** * Tests size of log queue is incremented and decremented properly. */ - @Test + @TestTemplate public void testSizeOfLogQueue() throws Exception { // There should be always 1 log which is current wal. assertEquals(1, logQueue.getMetrics().getSizeOfLogQueue()); @@ -745,7 +740,7 @@ public void testSizeOfLogQueue() throws Exception { * Tests that wals are closed cleanly and we read the trailer when we remove wal from * WALEntryStream. */ - @Test + @TestTemplate public void testCleanClosedWALs() throws Exception { try (WALEntryStream entryStream = new WALEntryStreamWithRetries(logQueue, fs, CONF, 0, log, logQueue.getMetrics(), fakeWalGroupId)) { @@ -762,7 +757,7 @@ public void testCleanClosedWALs() throws Exception { /** * Tests that we handle EOFException properly if the wal has moved to oldWALs directory. */ - @Test + @TestTemplate public void testEOFExceptionInOldWALsDirectory() throws Exception { assertEquals(1, logQueue.getQueueSize(fakeWalGroupId)); AbstractFSWAL abstractWAL = (AbstractFSWAL) log; @@ -806,7 +801,7 @@ public void testEOFExceptionInOldWALsDirectory() throws Exception { * decreased because {@link WALEntryBatch} is not put to * {@link ReplicationSourceWALReader#entryBatchQueue}. */ - @Test + @TestTemplate public void testReplicationSourceWALReaderWithPartialWALEntryFailingFilter() throws Exception { appendEntriesToLogAndSync(3); // get ending position @@ -845,7 +840,7 @@ public void testReplicationSourceWALReaderWithPartialWALEntryFailingFilter() thr } // testcase for HBASE-28748 - @Test + @TestTemplate public void testWALEntryStreamEOFRightAfterHeader() throws Exception { assertEquals(1, logQueue.getQueueSize(fakeWalGroupId)); AbstractFSWAL abstractWAL = (AbstractFSWAL) log; @@ -888,7 +883,7 @@ public void testWALEntryStreamEOFRightAfterHeader() throws Exception { * WALEntryFilterRetryableException does not cause the new file to be opened at the old file's * position. */ - @Test + @TestTemplate public void testPositionResetOnFileSwitchWithRetryableFilter() throws Exception { appendEntriesToLogAndSync(3); log.rollWriter(); @@ -923,15 +918,15 @@ public void testPositionResetOnFileSwitchWithRetryableFilter() throws Exception long deadline = System.currentTimeMillis() + 30000; while (totalEntries < 6) { long remaining = deadline - System.currentTimeMillis(); - assertTrue("Reader appears stuck - likely position corruption. Only got " + totalEntries - + " of 6 entries", remaining > 0); + assertTrue(remaining > 0, "Reader appears stuck - likely position corruption. Only got " + + totalEntries + " of 6 entries"); WALEntryBatch batch = reader.poll(1); if (batch != null && batch != WALEntryBatch.NO_MORE_DATA) { totalEntries += batch.getNbEntries(); } } assertEquals(6, totalEntries); - assertTrue("Filter should have thrown at least once", threwOnce.get()); + assertTrue(threwOnce.get(), "Filter should have thrown at least once"); } private static class PartialWALEntryFailingWALEntryFilter implements WALEntryFilter { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStreamAsyncFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStreamAsyncFSWAL.java index 2c37e34ae4e7..d4ccfeed5dc2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStreamAsyncFSWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStreamAsyncFSWAL.java @@ -17,30 +17,31 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import java.util.stream.Stream; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.AsyncFSWALProvider; import org.apache.hadoop.hbase.wal.WALFactory; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.provider.Arguments; -/** - * TestBasicWALEntryStream with {@link AsyncFSWALProvider} as the WAL provider. - */ -@RunWith(Parameterized.class) -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: isCompressionEnabled={0}") public class TestBasicWALEntryStreamAsyncFSWAL extends TestBasicWALEntryStream { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBasicWALEntryStreamAsyncFSWAL.class); + public TestBasicWALEntryStreamAsyncFSWAL(boolean isCompressionEnabled) { + super(isCompressionEnabled); + } + + public static Stream parameters() { + return Stream.of(Arguments.of(false), Arguments.of(true)); + } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setClass(WALFactory.WAL_PROVIDER, AsyncFSWALProvider.class, AbstractFSWALProvider.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStreamFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStreamFSHLog.java index 7ab9d3fc5a88..84928f8fcbd5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStreamFSHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStreamFSHLog.java @@ -17,30 +17,31 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import java.util.stream.Stream; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.FSHLogProvider; import org.apache.hadoop.hbase.wal.WALFactory; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.provider.Arguments; -/** - * TestBasicWALEntryStream with {@link FSHLogProvider} as the WAL provider. - */ -@RunWith(Parameterized.class) -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: isCompressionEnabled={0}") public class TestBasicWALEntryStreamFSHLog extends TestBasicWALEntryStream { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBasicWALEntryStreamFSHLog.class); + public TestBasicWALEntryStreamFSHLog(boolean isCompressionEnabled) { + super(isCompressionEnabled); + } + + public static Stream parameters() { + return Stream.of(Arguments.of(false), Arguments.of(true)); + } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setClass(WALFactory.WAL_PROVIDER, FSHLogProvider.class, AbstractFSWALProvider.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java index 0189d4755754..f33a64876e36 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java @@ -17,9 +17,8 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -34,17 +33,13 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestDrainReplicationQueuesForStandBy extends SyncReplicationTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestDrainReplicationQueuesForStandBy.class); - @Test public void test() throws Exception { UTIL2.getAdmin().transitReplicationPeerSyncReplicationState(PEER_ID, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java index 3e1dc624fe7d..b5c8e37fb620 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; @@ -43,37 +42,33 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Pair; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; /** * Tests for DumpReplicationQueues tool */ -@Category({ ReplicationTests.class, SmallTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(SmallTests.TAG) public class TestDumpReplicationQueues { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestDumpReplicationQueues.class); - private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); private static Configuration CONF; private static FileSystem FS = null; private Path root; private Path logDir; - @Rule - public final TestName name = new TestName(); - @Before - public void setup() throws Exception { + private String testName; + + @BeforeEach + public void setup(TestInfo testInfo) throws Exception { + testName = testInfo.getTestMethod().get().getName(); UTIL.startMiniCluster(3); CONF = UTIL.getConfiguration(); - TableName tableName = TableName.valueOf("replication_" + name.getMethodName()); + TableName tableName = TableName.valueOf("replication_" + testName); UTIL.getAdmin() .createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName)); CONF.set(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME, tableName.getNameAsString()); @@ -119,14 +114,17 @@ public void testDumpReplication() throws Exception { assertTrue(dump.indexOf("Number of WALs in replication queue: 4") > 0); // test for 'Returns wal sorted' String[] parsedDump = dump.split("Replication position for"); - assertTrue("First wal should be rs1%2C12345%2C123.10: 123, but got: " + parsedDump[1], - parsedDump[1].indexOf("rs1%2C12345%2C123.10: 123") >= 0); - assertTrue("Second wal should be rs1%2C12345%2C123.11: 0, but got: " + parsedDump[2], - parsedDump[2].indexOf("rs1%2C12345%2C123.11: 0 (not started or nothing to replicate)") >= 0); - assertTrue("Third wal should be rs1%2C12345%2C123.12: 0, but got: " + parsedDump[3], - parsedDump[3].indexOf("rs1%2C12345%2C123.12: 0 (not started or nothing to replicate)") >= 0); - assertTrue("Fourth wal should be rs1%2C12345%2C123.15: 0, but got: " + parsedDump[4], - parsedDump[4].indexOf("rs1%2C12345%2C123.15: 0 (not started or nothing to replicate)") >= 0); + assertTrue(parsedDump[1].indexOf("rs1%2C12345%2C123.10: 123") >= 0, + "First wal should be rs1%2C12345%2C123.10: 123, but got: " + parsedDump[1]); + assertTrue( + parsedDump[2].indexOf("rs1%2C12345%2C123.11: 0 (not started or nothing to replicate)") >= 0, + "Second wal should be rs1%2C12345%2C123.11: 0, but got: " + parsedDump[2]); + assertTrue( + parsedDump[3].indexOf("rs1%2C12345%2C123.12: 0 (not started or nothing to replicate)") >= 0, + "Third wal should be rs1%2C12345%2C123.12: 0, but got: " + parsedDump[3]); + assertTrue( + parsedDump[4].indexOf("rs1%2C12345%2C123.15: 0 (not started or nothing to replicate)") >= 0, + "Fourth wal should be rs1%2C12345%2C123.15: 0, but got: " + parsedDump[4]); Path file1 = new Path("testHFile1"); Path file2 = new Path("testHFile2"); @@ -152,7 +150,7 @@ private void addPeer(String peerId, String clusterKey) throws IOException { UTIL.getAdmin().addReplicationPeer(peerId, builder.build(), true); } - @After + @AfterEach public void tearDown() throws Exception { UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestGlobalReplicationThrottler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestGlobalReplicationThrottler.java index 6b14438c8308..240880e9a061 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestGlobalReplicationThrottler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestGlobalReplicationThrottler.java @@ -17,10 +17,9 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; @@ -43,26 +42,18 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; -@Category({ ReplicationTests.class, LargeTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(LargeTests.TAG) public class TestGlobalReplicationThrottler { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestGlobalReplicationThrottler.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestGlobalReplicationThrottler.class); private static final int REPLICATION_SOURCE_QUOTA = 200; private static int numOfPeer = 0; private static Configuration conf1; @@ -76,10 +67,9 @@ public class TestGlobalReplicationThrottler { private static final byte[] ROW = Bytes.toBytes("r"); private static final byte[][] ROWS = HTestConst.makeNAscii(ROW, 100); - @Rule - public TestName name = new TestName(); + private String testName; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf1 = HBaseConfiguration.create(); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); @@ -115,7 +105,7 @@ public static void setUpBeforeClass() throws Exception { } } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { utility2.shutdownMiniCluster(); utility1.shutdownMiniCluster(); @@ -125,8 +115,9 @@ public static void tearDownAfterClass() throws Exception { private volatile boolean testQuotaNonZero = false; @Test - public void testQuota() throws Exception { - final TableName tableName = TableName.valueOf(name.getMethodName()); + public void testQuota(TestInfo testInfo) throws Exception { + testName = testInfo.getTestMethod().get().getName(); + final TableName tableName = TableName.valueOf(testName); TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder .newBuilder(famName).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestHBaseInterClusterReplicationEndpointFilterEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestHBaseInterClusterReplicationEndpointFilterEdits.java index 7b108f5ca148..817232ed0da3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestHBaseInterClusterReplicationEndpointFilterEdits.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestHBaseInterClusterReplicationEndpointFilterEdits.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.ExtendedCell; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; @@ -43,11 +42,10 @@ import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; @@ -55,13 +53,10 @@ * Tests {@link HBaseInterClusterReplicationEndpoint#filterNotExistColumnFamilyEdits(List)} and * {@link HBaseInterClusterReplicationEndpoint#filterNotExistTableEdits(List)} */ -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestHBaseInterClusterReplicationEndpointFilterEdits { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHBaseInterClusterReplicationEndpointFilterEdits.class); - private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); private static HBaseInterClusterReplicationEndpoint endpoint; @@ -75,7 +70,7 @@ public class TestHBaseInterClusterReplicationEndpointFilterEdits { private static final byte[] ROW = Bytes.toBytes("r"); private static final byte[] VALUE = Bytes.toBytes("v"); - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { UTIL.startMiniCluster(); ReplicationPeer replicationPeer = mock(ReplicationPeer.class); @@ -91,7 +86,7 @@ public static void setUpBeforeClass() throws Exception { UTIL.createTable(TABLE1, FAMILY); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java index ab7935ddad41..d9a1f9fc186e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java @@ -18,10 +18,10 @@ package org.apache.hadoop.hbase.replication.regionserver; import static org.apache.hadoop.hbase.client.RegionLocator.LOCATOR_META_REPLICAS_MODE; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.ClientMetaTableAccessor; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -52,15 +51,14 @@ import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,23 +67,21 @@ * replication replays the edits to the secondary region in various scenarios. * @see TestRegionReplicaReplication */ -@Category({ LargeTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(LargeTests.TAG) public class TestMetaRegionReplicaReplication { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetaRegionReplicaReplication.class); private static final Logger LOG = LoggerFactory.getLogger(TestMetaRegionReplicaReplication.class); private static final int NB_SERVERS = 4; private final HBaseTestingUtil HTU = new HBaseTestingUtil(); private int numOfMetaReplica = NB_SERVERS - 1; private static byte[] VALUE = Bytes.toBytes("value"); - @Rule - public TestName name = new TestName(); + private String testName; - @Before - public void before() throws Exception { + @BeforeEach + public void before(TestInfo testInfo) throws Exception { + testName = testInfo.getTestMethod().get().getName(); Configuration conf = HTU.getConfiguration(); conf.setInt("zookeeper.recovery.retry", 1); conf.setInt("zookeeper.recovery.retry.intervalmill", 10); @@ -104,7 +100,7 @@ public void before() throws Exception { >= numOfMetaReplica); } - @After + @AfterEach public void after() throws Exception { HTU.shutdownMiniCluster(); } @@ -115,14 +111,14 @@ public void after() throws Exception { */ @Test public void testHBaseMetaReplicates() throws Exception { - try (Table table = HTU.createTable(TableName.valueOf(this.name.getMethodName() + "_0"), - HConstants.CATALOG_FAMILY, - Arrays.copyOfRange(HBaseTestingUtil.KEYS, 1, HBaseTestingUtil.KEYS.length))) { + try ( + Table table = HTU.createTable(TableName.valueOf(testName + "_0"), HConstants.CATALOG_FAMILY, + Arrays.copyOfRange(HBaseTestingUtil.KEYS, 1, HBaseTestingUtil.KEYS.length))) { verifyReplication(TableName.META_TABLE_NAME, numOfMetaReplica, getMetaCells(table.getName())); } - try (Table table = HTU.createTable(TableName.valueOf(this.name.getMethodName() + "_1"), - HConstants.CATALOG_FAMILY, - Arrays.copyOfRange(HBaseTestingUtil.KEYS, 1, HBaseTestingUtil.KEYS.length))) { + try ( + Table table = HTU.createTable(TableName.valueOf(testName + "_1"), HConstants.CATALOG_FAMILY, + Arrays.copyOfRange(HBaseTestingUtil.KEYS, 1, HBaseTestingUtil.KEYS.length))) { verifyReplication(TableName.META_TABLE_NAME, numOfMetaReplica, getMetaCells(table.getName())); // Try delete. HTU.deleteTableIfAny(table.getName()); @@ -376,14 +372,6 @@ private void doNGets(final Table table, final byte[][] keys) throws Exception { } } - private void primaryNoChangeReplicaIncrease(final long[] before, final long[] after) { - assertEquals(before[RegionInfo.DEFAULT_REPLICA_ID], after[RegionInfo.DEFAULT_REPLICA_ID]); - - for (int i = 1; i < after.length; i++) { - assertTrue(after[i] > before[i]); - } - } - private void primaryIncreaseReplicaNoChange(final long[] before, final long[] after) { // There are read requests increase for primary meta replica. assertTrue(after[RegionInfo.DEFAULT_REPLICA_ID] > before[RegionInfo.DEFAULT_REPLICA_ID]); @@ -412,7 +400,7 @@ private void getMetaReplicaReadRequests(final Region[] metaRegions, final long[] @Test public void testHBaseMetaReplicaGets() throws Exception { - TableName tn = TableName.valueOf(this.name.getMethodName()); + TableName tn = TableName.valueOf(testName); final Region[] metaRegions = getAllRegions(TableName.META_TABLE_NAME, numOfMetaReplica); long[] readReqsForMetaReplicas = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterGet = new long[numOfMetaReplica]; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRaceWhenCreatingReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRaceWhenCreatingReplicationSource.java index 66f04dca36d5..8ee341ca8435 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRaceWhenCreatingReplicationSource.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRaceWhenCreatingReplicationSource.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.io.UncheckedIOException; @@ -27,7 +27,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -46,22 +45,18 @@ import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALProvider; import org.apache.hadoop.hbase.wal.WALStreamReader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Testcase for HBASE-20624. */ -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestRaceWhenCreatingReplicationSource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRaceWhenCreatingReplicationSource.class); - private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); private static String PEER_ID = "1"; @@ -82,7 +77,7 @@ public class TestRaceWhenCreatingReplicationSource { public static final class LocalReplicationEndpoint extends BaseReplicationEndpoint { - private static final UUID PEER_UUID = UTIL.getRandomUUID(); + private static final UUID PEER_UUID = HBaseTestingUtil.getRandomUUID(); @Override public UUID getPeerUUID() { @@ -134,7 +129,7 @@ public boolean canReplicateToSameCluster() { } } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { UTIL.getConfiguration().set(WALFactory.WAL_PROVIDER, "multiwal"); // make sure that we will create a new group for the table @@ -150,7 +145,7 @@ public static void setUpBeforeClass() throws Exception { true); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshPeerWhileRegionServerRestarts.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshPeerWhileRegionServerRestarts.java index e9d8e05c8818..ac0d048bf153 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshPeerWhileRegionServerRestarts.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshPeerWhileRegionServerRestarts.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.master.replication.DisablePeerProcedure; import org.apache.hadoop.hbase.procedure2.Procedure; @@ -34,9 +33,8 @@ import org.apache.hadoop.hbase.replication.TestReplicationBase; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationState; @@ -44,13 +42,10 @@ * This UT is used to make sure that we will not accidentally change the way to generate online * servers. See HBASE-25774 and HBASE-25032 for more details. */ -@Category({ MasterTests.class, MediumTests.class }) +@Tag(MasterTests.TAG) +@Tag(MediumTests.TAG) public class TestRefreshPeerWhileRegionServerRestarts extends TestReplicationBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRefreshPeerWhileRegionServerRestarts.class); - private static CountDownLatch ARRIVE; private static CountDownLatch RESUME; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshRecoveredReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshRecoveredReplication.java index 93aa7130926f..775ff9ef7cef 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshRecoveredReplication.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshRecoveredReplication.java @@ -17,39 +17,33 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.List; import java.util.Optional; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.replication.TestReplicationBase; +import org.apache.hadoop.hbase.replication.TestReplicationBaseNoBeforeAll; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,44 +52,36 @@ /** * Testcase for HBASE-24871. */ -@Category({ ReplicationTests.class, MediumTests.class }) -public class TestRefreshRecoveredReplication extends TestReplicationBase { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRefreshRecoveredReplication.class); +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) +public class TestRefreshRecoveredReplication extends TestReplicationBaseNoBeforeAll { private static final Logger LOG = LoggerFactory.getLogger(TestRefreshRecoveredReplication.class); private static final int BATCH = 50; - @Rule - public TestName name = new TestName(); - - private TableName tablename; - private Table table1; - private Table table2; - - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { // NUM_SLAVES1 is presumed 2 in below. NUM_SLAVES1 = 2; + configureClusters(UTIL1, UTIL2); // replicate slowly - Configuration conf1 = UTIL1.getConfiguration(); - conf1.setInt(HConstants.REPLICATION_SOURCE_TOTAL_BUFFER_KEY, 100); - TestReplicationBase.setUpBeforeClass(); + CONF1.setInt(HConstants.REPLICATION_SOURCE_TOTAL_BUFFER_KEY, 100); + startClusters(); } - @AfterClass - public static void tearDownAfterClass() throws Exception { - TestReplicationBase.tearDownAfterClass(); - } + private String testName; + + private TableName tablename; + private Table table1; + private Table table2; - @Before - public void setup() throws Exception { + @BeforeEach + public void setup(TestInfo testInfo) throws Exception { + testName = testInfo.getTestMethod().get().getName(); setUpBase(); - tablename = TableName.valueOf(name.getMethodName()); + tablename = TableName.valueOf(testName); TableDescriptor table = TableDescriptorBuilder.newBuilder(tablename).setColumnFamily(ColumnFamilyDescriptorBuilder .newBuilder(famName).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build(); @@ -108,7 +94,7 @@ public void setup() throws Exception { table2 = UTIL2.getConnection().getTable(tablename); } - @After + @AfterEach public void teardown() throws Exception { tearDownBase(); @@ -130,7 +116,7 @@ public void testReplicationRefreshSource() throws Exception { Optional server = rss.stream() .filter(rst -> CollectionUtils.isNotEmpty(rst.getRegionServer().getRegions(tablename))) .findAny(); - Assert.assertTrue(server.isPresent()); + assertTrue(server.isPresent()); HRegionServer otherServer = rss.get(0).getRegionServer() == server.get().getRegionServer() ? rss.get(1).getRegionServer() : rss.get(0).getRegionServer(); @@ -159,9 +145,10 @@ public void testReplicationRefreshSource() throws Exception { private int checkReplicationData() throws IOException { int count = 0; - ResultScanner results = table2.getScanner(new Scan().setCaching(BATCH)); - for (Result r : results) { - count++; + try (ResultScanner results = table2.getScanner(new Scan().setCaching(BATCH))) { + while (results.next() != null) { + count++; + } } return count; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplication.java index 777232f68e9b..33e0a15f788e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplication.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplication.java @@ -17,13 +17,12 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.IOException; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -42,13 +41,11 @@ import org.apache.hadoop.hbase.testclassification.FlakeyTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,23 +55,19 @@ * Tests region replication by setting up region replicas and verifying async wal replication * replays the edits to the secondary region in various scenarios. */ -@Category({ FlakeyTests.class, LargeTests.class }) +@Tag(FlakeyTests.TAG) +@Tag(LargeTests.TAG) public class TestRegionReplicaReplication { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionReplicaReplication.class); - private static final Logger LOG = LoggerFactory.getLogger(TestRegionReplicaReplication.class); private static final int NB_SERVERS = 2; private static final HBaseTestingUtil HTU = new HBaseTestingUtil(); - @Rule - public TestName name = new TestName(); + private String testName; - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { Configuration conf = HTU.getConfiguration(); conf.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f); @@ -93,7 +86,7 @@ public static void beforeClass() throws Exception { HTU.startMiniCluster(NB_SERVERS); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { HTU.shutdownMiniCluster(); } @@ -196,9 +189,10 @@ public void testRegionReplicaReplicationWith10Replicas() throws Exception { } @Test - public void testRegionReplicaWithoutMemstoreReplication() throws Exception { + public void testRegionReplicaWithoutMemstoreReplication(TestInfo testInfo) throws Exception { + testName = testInfo.getTestMethod().get().getName(); int regionReplication = 3; - TableDescriptor htd = HTU.createModifyableTableDescriptor(name.getMethodName()) + TableDescriptor htd = HTU.createModifyableTableDescriptor(testName) .setRegionReplication(regionReplication).setRegionMemStoreReplication(false).build(); createOrEnableTableWithRetries(htd, true); final TableName tableName = htd.getTableName(); @@ -226,14 +220,16 @@ public void testRegionReplicaWithoutMemstoreReplication() throws Exception { } @Test - public void testRegionReplicaReplicationForFlushAndCompaction() throws Exception { + public void testRegionReplicaReplicationForFlushAndCompaction(TestInfo testInfo) + throws Exception { + testName = testInfo.getTestMethod().get().getName(); // Tests a table with region replication 3. Writes some data, and causes flushes and // compactions. Verifies that the data is readable from the replicas. Note that this // does not test whether the replicas actually pick up flushed files and apply compaction // to their stores int regionReplication = 3; - TableDescriptor htd = HTU.createModifyableTableDescriptor(name.getMethodName()) - .setRegionReplication(regionReplication).build(); + TableDescriptor htd = + HTU.createModifyableTableDescriptor(testName).setRegionReplication(regionReplication).build(); createOrEnableTableWithRetries(htd, true); final TableName tableName = htd.getTableName(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationCompressedWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationCompressedWAL.java index 27c39cc0df2f..9682b202d440 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationCompressedWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationCompressedWAL.java @@ -17,44 +17,42 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.replication.TestReplicationBase; +import org.apache.hadoop.hbase.replication.TestReplicationBaseNoBeforeAll; import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(MediumTests.class) -public class TestReplicationCompressedWAL extends TestReplicationBase { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationCompressedWAL.class); +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) +public class TestReplicationCompressedWAL extends TestReplicationBaseNoBeforeAll { static final Logger LOG = LoggerFactory.getLogger(TestReplicationCompressedWAL.class); static final int NUM_BATCHES = 20; static final int NUM_ROWS_PER_BATCH = 100; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { + configureClusters(UTIL1, UTIL2); CONF1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); - TestReplicationBase.setUpBeforeClass(); + startClusters(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TestReplicationBase.tearDownAfterClass(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationMarker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationMarker.java index 1466f5f5a03e..884dd650a78c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationMarker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationMarker.java @@ -25,9 +25,9 @@ import static org.apache.hadoop.hbase.replication.master.ReplicationSinkTrackerTableCreator.WAL_NAME_COLUMN; import static org.apache.hadoop.hbase.replication.regionserver.ReplicationMarkerChore.REPLICATION_MARKER_CHORE_DURATION_KEY; import static org.apache.hadoop.hbase.replication.regionserver.ReplicationMarkerChore.REPLICATION_MARKER_ENABLED_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; @@ -53,11 +52,10 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WAL; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,11 +67,9 @@ * "hbase.regionserver.replication.sink.tracker.enabled" conf key enabled. This will persist the * marker rows coming from peer cluster to persist to REPLICATION.SINK_TRACKER table. **/ -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestReplicationMarker { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationMarker.class); private static final Logger LOG = LoggerFactory.getLogger(TestReplicationMarker.class); @@ -82,7 +78,7 @@ public class TestReplicationMarker { private static HBaseTestingUtil utility1; private static HBaseTestingUtil utility2; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf1 = HBaseConfiguration.create(); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); @@ -111,15 +107,15 @@ public static void setUpBeforeClass() throws Exception { ReplicationSourceManager manager = utility1.getHBaseCluster().getRegionServer(0) .getReplicationSourceService().getReplicationManager(); // Wait until the peer gets established. - Waiter.waitFor(conf1, 10000, (Waiter.Predicate) () -> manager.getSources().size() == 1); + Waiter.waitFor(conf1, 10000, () -> manager.getSources().size() == 1); } private static void waitForReplicationTrackerTableCreation() { - Waiter.waitFor(conf2, 10000, (Waiter.Predicate) () -> utility2.getAdmin() - .tableExists(REPLICATION_SINK_TRACKER_TABLE_NAME)); + Waiter.waitFor(conf2, 10000, + () -> utility2.getAdmin().tableExists(REPLICATION_SINK_TRACKER_TABLE_NAME)); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { utility1.shutdownMiniCluster(); utility2.shutdownMiniCluster(); @@ -131,14 +127,14 @@ public void testReplicationMarkerRow() throws Exception { // create enough sentinel rows. Thread.sleep(5000); WAL wal1 = utility1.getHBaseCluster().getRegionServer(0).getWAL(null); - String walName1ForCluster1 = ((AbstractFSWAL) wal1).getCurrentFileName().getName(); + String walName1ForCluster1 = ((AbstractFSWAL) wal1).getCurrentFileName().getName(); String rs1Name = utility1.getHBaseCluster().getRegionServer(0).getServerName().getHostname(); // Since we sync the marker edits while appending to wal, all the edits should be visible // to Replication threads immediately. assertTrue(getReplicatedEntries() >= 5); // Force log roll. wal1.rollWriter(true); - String walName2ForCluster1 = ((AbstractFSWAL) wal1).getCurrentFileName().getName(); + String walName2ForCluster1 = ((AbstractFSWAL) wal1).getCurrentFileName().getName(); Connection connection2 = utility2.getMiniHBaseCluster().getRegionServer(0).getConnection(); // Sleep for 5 more seconds to get marker rows with new wal name. Thread.sleep(5000); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java index bdb51ebe36f8..256bb0fd311f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.ArrayList; @@ -36,7 +37,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ExtendedCell; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; @@ -63,13 +63,11 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.HFileTestUtil; import org.apache.hadoop.hbase.wal.WALEditInternalHelper; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,13 +79,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALKey; -@Category({ ReplicationTests.class, LargeTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(LargeTests.TAG) public class TestReplicationSink { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationSink.class); - private static final Logger LOG = LoggerFactory.getLogger(TestReplicationSink.class); private static final int BATCH_SIZE = 10; @@ -123,10 +118,7 @@ public void stop(String why) { protected static String hfileArchiveDir; protected static String replicationClusterId; - /** - * @throws java.lang.Exception - */ - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().set("hbase.replication.source.fs.conf.provider", TestSourceFSConfigurationProvider.class.getCanonicalName()); @@ -142,19 +134,13 @@ public static void setUpBeforeClass() throws Exception { replicationClusterId = "12345"; } - /** - * @throws java.lang.Exception - */ - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { STOPPABLE.stop("Shutting down"); TEST_UTIL.shutdownMiniCluster(); } - /** - * @throws java.lang.Exception - */ - @Before + @BeforeEach public void setUp() throws Exception { table1 = TEST_UTIL.deleteTableData(TABLE_NAME1); table2 = TEST_UTIL.deleteTableData(TABLE_NAME2); @@ -320,7 +306,7 @@ public void testRethrowRetriesExhaustedException() throws Exception { try { SINK.replicateEntries(entries, PrivateCellUtil.createExtendedCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); - Assert.fail("Should re-throw TableNotFoundException."); + fail("Should re-throw TableNotFoundException."); } catch (TableNotFoundException e) { } entries.clear(); @@ -335,7 +321,7 @@ public void testRethrowRetriesExhaustedException() throws Exception { SINK.replicateEntries(entries, PrivateCellUtil.createExtendedCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); - Assert.fail("Should re-throw RetriesExhaustedWithDetailsException."); + fail("Should re-throw RetriesExhaustedWithDetailsException."); } catch (RetriesExhaustedException e) { } finally { admin.enableTable(TABLE_NAME1); @@ -441,7 +427,7 @@ public void testFailedReplicationSinkMetrics() throws IOException { try { SINK.replicateEntries(entries, PrivateCellUtil.createExtendedCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); - Assert.fail("Should re-throw ArrayIndexOutOfBoundsException."); + fail("Should re-throw ArrayIndexOutOfBoundsException."); } catch (ArrayIndexOutOfBoundsException e) { errorCount++; assertEquals(initialFailedBatches + errorCount, SINK.getSinkMetrics().getFailedBatches()); @@ -456,7 +442,7 @@ public void testFailedReplicationSinkMetrics() throws IOException { try { SINK.replicateEntries(entries, PrivateCellUtil.createExtendedCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); - Assert.fail("Should re-throw TableNotFoundException."); + fail("Should re-throw TableNotFoundException."); } catch (TableNotFoundException e) { errorCount++; assertEquals(initialFailedBatches + errorCount, SINK.getSinkMetrics().getFailedBatches()); @@ -475,7 +461,7 @@ public void testFailedReplicationSinkMetrics() throws IOException { SINK.replicateEntries(entries, PrivateCellUtil.createExtendedCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); - Assert.fail("Should re-throw IOException."); + fail("Should re-throw IOException."); } catch (IOException e) { errorCount++; assertEquals(initialFailedBatches + errorCount, SINK.getSinkMetrics().getFailedBatches()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java index 37af52eb93b9..468ff3940fcf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java @@ -18,11 +18,11 @@ package org.apache.hadoop.hbase.replication.regionserver; import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.META_WAL_PROVIDER_ID; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; @@ -71,24 +70,20 @@ import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; import org.apache.hadoop.hbase.wal.WALStreamReader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestReplicationSource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationSource.class); - private static final Logger LOG = LoggerFactory.getLogger(TestReplicationSource.class); private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); private final static HBaseTestingUtil TEST_UTIL_PEER = new HBaseTestingUtil(); @@ -97,7 +92,7 @@ public class TestReplicationSource { private static Path logDir; private static Configuration conf = TEST_UTIL.getConfiguration(); - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniDFSCluster(1); FS = TEST_UTIL.getDFSCluster().getFileSystem(); @@ -112,7 +107,7 @@ public static void setUpBeforeClass() throws Exception { } } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL_PEER.shutdownMiniHBaseCluster(); TEST_UTIL.shutdownMiniHBaseCluster(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceLogQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceLogQueue.java index 9d101b97217e..4608100b42ca 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceLogQueue.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceLogQueue.java @@ -17,30 +17,25 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; -@Category({ SmallTests.class, ReplicationTests.class }) +@Tag(SmallTests.TAG) +@Tag(ReplicationTests.TAG) public class TestReplicationSourceLogQueue { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationSourceLogQueue.class); - - /* + /** * Testing enqueue and dequeuing of wal and check age of oldest wal. */ @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java index aadf5982b684..fe954074d7c1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java @@ -20,9 +20,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasSize; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.Server; @@ -71,25 +70,20 @@ import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.hamcrest.Matchers; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestReplicationSourceManager { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationSourceManager.class); - public static final class ReplicationEndpointForTest extends DummyReplicationEndpoint { private String clusterKey; @@ -128,8 +122,7 @@ public void init(Context context) throws IOException { private static NavigableMap SCOPES; - @Rule - public final TestName name = new TestName(); + private String testName; private Path oldLogDir; @@ -143,7 +136,7 @@ public void init(Context context) throws IOException { private ReplicationSourceManager manager; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { UTIL.startMiniCluster(1); FS = UTIL.getTestFileSystem(); @@ -156,14 +149,15 @@ public static void setUpBeforeClass() throws Exception { SCOPES.put(F2, 0); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws IOException { UTIL.shutdownMiniCluster(); } - @Before - public void setUp() throws Exception { - Path rootDir = UTIL.getDataTestDirOnTestFS(name.getMethodName()); + @BeforeEach + public void setUp(TestInfo testInfo) throws Exception { + testName = testInfo.getTestMethod().get().getName(); + Path rootDir = UTIL.getDataTestDirOnTestFS(testName); CommonFSUtils.setRootDir(CONF, rootDir); server = mock(Server.class); when(server.getConfiguration()).thenReturn(CONF); @@ -177,7 +171,7 @@ public void setUp() throws Exception { FS.mkdirs(logDir); remoteLogDir = new Path(rootDir, ReplicationUtils.REMOTE_WAL_DIR_NAME); FS.mkdirs(remoteLogDir); - TableName tableName = TableName.valueOf("replication_" + name.getMethodName()); + TableName tableName = TableName.valueOf("replication_" + testName); UTIL.getAdmin() .createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName)); CONF.set(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME, tableName.getNameAsString()); @@ -188,7 +182,7 @@ public void setUp() throws Exception { manager = replication.getReplicationManager(); } - @After + @AfterEach public void tearDown() { replication.stopReplicationService(); } @@ -401,22 +395,21 @@ public void testPeerConfigurationOverridesPropagate() throws Exception { ReplicationSource source = (ReplicationSource) manager.getSources().stream() .filter(s -> s.getPeerId().equals(peerId)).findFirst().orElse(null); - assertNotNull("Source should be created for peer", source); + assertNotNull(source, "Source should be created for peer"); - assertEquals("ReplicationSource should use peer config override for sleepForRetries", - peerSleepOverride, source.getSleepForRetries()); + assertEquals(peerSleepOverride, source.getSleepForRetries(), + "ReplicationSource should use peer config override for sleepForRetries"); Map workers = source.workerThreads; if (!workers.isEmpty()) { ReplicationSourceShipper shipper = workers.values().iterator().next(); - assertEquals("ReplicationSourceShipper should use peer config override for sleepForRetries", - peerSleepOverride, shipper.getSleepForRetries()); + assertEquals(peerSleepOverride, shipper.getSleepForRetries(), + "ReplicationSourceShipper should use peer config override for sleepForRetries"); ReplicationSourceWALReader reader = shipper.entryReader; if (reader != null) { - assertEquals( - "ReplicationSourceWALReader should use peer config override for sleepForRetries", - peerSleepOverride, reader.getSleepForRetries()); + assertEquals(peerSleepOverride, reader.getSleepForRetries(), + "ReplicationSourceWALReader should use peer config override for sleepForRetries"); } } @@ -459,16 +452,16 @@ public void testPeerConfigurationIsolation() throws Exception { // Verify peer with override uses the override value ReplicationSource sourceWithOverride = (ReplicationSource) manager.getSources().stream() .filter(s -> s.getPeerId().equals(peerIdWithOverride)).findFirst().orElse(null); - assertNotNull("Source with override should be created", sourceWithOverride); - assertEquals("Peer with override should use override value", peerSleepOverride, - sourceWithOverride.getSleepForRetries()); + assertNotNull(sourceWithOverride, "Source with override should be created"); + assertEquals(peerSleepOverride, sourceWithOverride.getSleepForRetries(), + "Peer with override should use override value"); // Verify peer without override uses global config ReplicationSource sourceWithoutOverride = (ReplicationSource) manager.getSources().stream() .filter(s -> s.getPeerId().equals(peerIdWithoutOverride)).findFirst().orElse(null); - assertNotNull("Source without override should be created", sourceWithoutOverride); - assertEquals("Peer without override should use global config", globalSleepValue, - sourceWithoutOverride.getSleepForRetries()); + assertNotNull(sourceWithoutOverride, "Source without override should be created"); + assertEquals(globalSleepValue, sourceWithoutOverride.getSleepForRetries(), + "Peer without override should use global config"); removePeerAndWait(peerIdWithOverride); removePeerAndWait(peerIdWithoutOverride); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerJoin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerJoin.java index d7d23783eacb..a623787bf3e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerJoin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerJoin.java @@ -17,12 +17,11 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Optional; import java.util.stream.Stream; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; @@ -31,40 +30,36 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.replication.TestReplicationBase; +import org.apache.hadoop.hbase.replication.TestReplicationBaseNoBeforeAll; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -@Category({ ReplicationTests.class, MediumTests.class }) -public class TestReplicationSourceManagerJoin extends TestReplicationBase { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationSourceManagerJoin.class); - - @Rule - public TestName testName = new TestName(); +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) +public class TestReplicationSourceManagerJoin extends TestReplicationBaseNoBeforeAll { + @BeforeAll @BeforeClass public static void setUpBeforeClass() throws Exception { // NUM_SLAVES1 is presumed 2 in below. NUM_SLAVES1 = 2; - TestReplicationBase.setUpBeforeClass(); + configureClusters(UTIL1, UTIL2); + startClusters(); } @Test - public void testReplicationSourcesTerminate() throws Exception { + public void testReplicationSourcesTerminate(TestInfo testInfo) throws Exception { + String testName = testInfo.getTestMethod().get().getName(); // Create table in source cluster only, let TableNotFoundException block peer to avoid // recovered source end. - TableName tableName = TableName.valueOf(testName.getMethodName()); + TableName tableName = TableName.valueOf(testName); TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName) .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationThrottler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationThrottler.java index ce0f7a351e8a..078c3bc2d84b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationThrottler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationThrottler.java @@ -17,25 +17,20 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ ReplicationTests.class, SmallTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(SmallTests.TAG) public class TestReplicationThrottler { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationThrottler.class); - private static final Logger LOG = LoggerFactory.getLogger(TestReplicationThrottler.class); /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationValueCompressedWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationValueCompressedWAL.java index 03b83964dccf..436e745d2cda 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationValueCompressedWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationValueCompressedWAL.java @@ -17,38 +17,29 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.regionserver.wal.CompressionContext; -import org.apache.hadoop.hbase.replication.TestReplicationBase; +import org.apache.hadoop.hbase.replication.TestReplicationBaseNoBeforeAll; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(MediumTests.class) -public class TestReplicationValueCompressedWAL extends TestReplicationBase { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationValueCompressedWAL.class); +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) +public class TestReplicationValueCompressedWAL extends TestReplicationBaseNoBeforeAll { static final Logger LOG = LoggerFactory.getLogger(TestReplicationValueCompressedWAL.class); - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { + configureClusters(UTIL1, UTIL2); CONF1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); CONF1.setBoolean(CompressionContext.ENABLE_WAL_VALUE_COMPRESSION, true); - TestReplicationBase.setUpBeforeClass(); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TestReplicationBase.tearDownAfterClass(); + startClusters(); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java index bc885db0df54..6d83def2e6d6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.HashMap; @@ -29,7 +29,6 @@ import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -40,9 +39,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKeyImpl; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; @@ -50,13 +48,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor; -@Category({ ReplicationTests.class, SmallTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(SmallTests.TAG) public class TestReplicationWALEdits { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationWALEdits.class); - private static final Configuration CONF = HBaseConfiguration.create(); private static final TableName TABLE_NAME = TableName.valueOf("test"); @@ -118,8 +113,8 @@ public void testBulkLoadWALEditsWithoutBulkLoadReplicationEnabled() throws Excep ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, CONF); // 4. Assert that no bulk load entry scopes are added if bulk load hfile replication is disabled - assertNull("No bulk load entries scope should be added if bulk load replication is disabled.", - logKey.getReplicationScopes()); + assertNull(logKey.getReplicationScopes(), + "No bulk load entries scope should be added if bulk load replication is disabled."); } @Test @@ -138,10 +133,10 @@ public void testBulkLoadWALEdits() throws Exception { NavigableMap scopes = logKey.getReplicationScopes(); // Assert family with replication scope global is present in the key scopes - assertTrue("This family scope is set to global, should be part of replication key scopes.", - scopes.containsKey(F1)); + assertTrue(scopes.containsKey(F1), + "This family scope is set to global, should be part of replication key scopes."); // Assert family with replication scope local is not present in the key scopes - assertFalse("This family scope is set to local, should not be part of replication key scopes", - scopes.containsKey(F2)); + assertFalse(scopes.containsKey(F2), + "This family scope is set to local, should not be part of replication key scopes"); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java index 979db712ef34..c6ca1f3cbbc5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; @@ -32,35 +31,32 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -import org.apache.hadoop.hbase.replication.TestReplicationBase; +import org.apache.hadoop.hbase.replication.TestReplicationBaseNoBeforeAll; import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; -@Category(MediumTests.class) -public class TestReplicator extends TestReplicationBase { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicator.class); +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) +public class TestReplicator extends TestReplicationBaseNoBeforeAll { static final Logger LOG = LoggerFactory.getLogger(TestReplicator.class); static final int NUM_ROWS = 10; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { + configureClusters(UTIL1, UTIL2); // Set RPC size limit to 10kb (will be applied to both source and sink clusters) CONF1.setInt(RpcServer.MAX_REQUEST_SIZE, 1024 * 10); - TestReplicationBase.setUpBeforeClass(); + startClusters(); } @Test @@ -104,9 +100,10 @@ public String explainFailure() throws Exception { } }); - assertEquals("We sent an incorrect number of batches", NUM_ROWS, - ReplicationEndpointForTest.getBatchCount()); - assertEquals("We did not replicate enough rows", NUM_ROWS, UTIL2.countRows(htable2)); + assertEquals(NUM_ROWS, ReplicationEndpointForTest.getBatchCount(), + "We sent an incorrect number of batches"); + assertEquals(NUM_ROWS, HBaseTestingUtil.countRows(htable2), + "We did not replicate enough rows"); } finally { hbaseAdmin.removeReplicationPeer("testReplicatorBatching"); } @@ -154,17 +151,13 @@ public String explainFailure() throws Exception { } }); - assertEquals("We did not replicate enough rows", NUM_ROWS, UTIL2.countRows(htable2)); + assertEquals(NUM_ROWS, HBaseTestingUtil.countRows(htable2), + "We did not replicate enough rows"); } finally { hbaseAdmin.removeReplicationPeer("testReplicatorWithErrors"); } } - @AfterClass - public static void tearDownAfterClass() throws Exception { - TestReplicationBase.tearDownAfterClass(); - } - private void truncateTable(HBaseTestingUtil util, TableName tablename) throws IOException { Admin admin = util.getAdmin(); admin.disableTable(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java index 8731adbe4c2b..60ab0a41ac22 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.Cell.Type; import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.Server; @@ -56,26 +55,21 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALKeyImpl; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestSerialReplicationChecker { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSerialReplicationChecker.class); - private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); private static String PEER_ID = "1"; @@ -88,12 +82,11 @@ public class TestSerialReplicationChecker { private SerialReplicationChecker checker; - @Rule - public final TestName name = new TestName(); + private String testName; private TableName tableName; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { UTIL.startMiniCluster(1); TableName repTable = TableName.valueOf("test_serial_rep"); @@ -103,13 +96,14 @@ public static void setUpBeforeClass() throws Exception { UTIL.getConfiguration(), repTable); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { UTIL.shutdownMiniCluster(); } - @Before - public void setUp() throws IOException { + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException { + testName = testInfo.getTestMethod().get().getName(); ReplicationSource source = mock(ReplicationSource.class); when(source.getPeerId()).thenReturn(PEER_ID); when(source.getReplicationQueueStorage()).thenReturn(QUEUE_STORAGE); @@ -127,7 +121,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { when(server.getConnection()).thenReturn(conn); when(source.getServer()).thenReturn(server); checker = new SerialReplicationChecker(UTIL.getConfiguration(), source); - tableName = TableName.valueOf(name.getMethodName()); + tableName = TableName.valueOf(testName); } private Entry createEntry(RegionInfo region, long seqId) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationEndpoint.java index 5f99b88e0a45..b6d287d3ad70 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationEndpoint.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationEndpoint.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.replication.regionserver; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -26,7 +29,6 @@ import java.util.concurrent.LinkedBlockingQueue; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -45,29 +47,24 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestSerialReplicationEndpoint { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSerialReplicationEndpoint.class); - private static HBaseTestingUtil UTIL = new HBaseTestingUtil(); private static Configuration CONF; private static Connection CONN; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.startMiniCluster(); CONF = UTIL.getConfiguration(); @@ -75,7 +72,7 @@ public static void setUp() throws Exception { CONN = UTIL.getConnection(); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { Closeables.close(CONN, true); UTIL.shutdownMiniCluster(); @@ -117,7 +114,7 @@ private void testHBaseReplicationEndpoint(String tableNameStr, String peerId, bo Waiter.waitFor(CONF, 60000, () -> TestEndpoint.getEntries().size() >= cellNum); int index = 0; - Assert.assertEquals(TestEndpoint.getEntries().size(), cellNum); + assertEquals(cellNum, TestEndpoint.getEntries().size()); if (!isSerial) { Collections.sort(TestEndpoint.getEntries(), (a, b) -> { long seqA = a.getKey().getSequenceId(); @@ -126,15 +123,14 @@ private void testHBaseReplicationEndpoint(String tableNameStr, String peerId, bo }); } for (Entry entry : TestEndpoint.getEntries()) { - Assert.assertEquals(entry.getKey().getTableName(), tableName); - Assert.assertEquals(entry.getEdit().getCells().size(), 1); + assertEquals(tableName, entry.getKey().getTableName()); + assertEquals(1, entry.getEdit().getCells().size()); Cell cell = entry.getEdit().getCells().get(0); - Assert.assertArrayEquals( - Bytes.copy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()), - Bytes.toBytes(index)); + assertArrayEquals(Bytes.toBytes(index), + Bytes.copy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); index++; } - Assert.assertEquals(index, cellNum); + assertEquals(cellNum, index); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSyncReplicationShipperQuit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSyncReplicationShipperQuit.java index 5a0578a35d27..7eb587e6cd55 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSyncReplicationShipperQuit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSyncReplicationShipperQuit.java @@ -17,10 +17,9 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; @@ -30,20 +29,16 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Testcase for HBASE-20456. */ -@Category({ ReplicationTests.class, LargeTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(LargeTests.TAG) public class TestSyncReplicationShipperQuit extends SyncReplicationTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSyncReplicationShipperQuit.class); - @Test public void testShipperQuitWhenDA() throws Exception { // set to serial replication diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java index 11a0e98c5541..0cc36a892cd3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.net.SocketAddress; @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.ExtendedCellBuilder; import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; import org.apache.hadoop.hbase.ExtendedCellScanner; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -51,11 +50,9 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; @@ -64,15 +61,10 @@ /** * Simple test of sink-side wal entry filter facility. */ -@Category({ ReplicationTests.class, SmallTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(SmallTests.TAG) public class TestWALEntrySinkFilter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestWALEntrySinkFilter.class); - - @Rule - public TestName name = new TestName(); static final int BOUNDARY = 5; static final AtomicInteger UNFILTERED = new AtomicInteger(); static final AtomicInteger FILTERED = new AtomicInteger(); @@ -84,7 +76,8 @@ public class TestWALEntrySinkFilter { * our counting Table. */ @Test - public void testWALEntryFilter() throws IOException { + public void testWALEntryFilter(TestInfo testInfo) throws IOException { + String testName = testInfo.getTestMethod().get().getName(); Configuration conf = HBaseConfiguration.create(); // Make it so our filter is instantiated on construction of ReplicationSink. conf.setClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, @@ -98,8 +91,7 @@ public void testWALEntryFilter() throws IOException { List entries = new ArrayList<>(); AdminProtos.WALEntry.Builder entryBuilder = AdminProtos.WALEntry.newBuilder(); // Need a tablename. - ByteString tableName = - ByteString.copyFromUtf8(TableName.valueOf(this.name.getMethodName()).toString()); + ByteString tableName = ByteString.copyFromUtf8(TableName.valueOf(testName).toString()); // Add WALEdit Cells to Cells List. The way edits arrive at the sink is with protos // describing the edit with all Cells from all edits aggregated in a single CellScanner. final List cells = new ArrayList<>(); @@ -201,7 +193,7 @@ public CompletableFuture> batchAll(List actions) { for (Row action : actions) { // Row is the index of the loop above where we make WALEntry and Cells. int row = Bytes.toInt(action.getRow()); - assertTrue("" + row, row > BOUNDARY); + assertTrue(row > BOUNDARY, "" + row); UNFILTERED.incrementAndGet(); list.add(null); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamCompressionReset.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamCompressionReset.java index aa30027b3c98..9b182a86ae8d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamCompressionReset.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamCompressionReset.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; @@ -53,11 +52,10 @@ import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALEditInternalHelper; import org.apache.hadoop.hbase.wal.WALKeyImpl; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; @@ -67,13 +65,10 @@ * This is used to confirm that we can work well when hitting EOFException in the middle when * reading a WAL entry, when compression is enabled. See HBASE-27621 for more details. */ -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestWALEntryStreamCompressionReset { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestWALEntryStreamCompressionReset.class); - private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); private static TableName TABLE_NAME = TableName.valueOf("reset"); @@ -138,7 +133,7 @@ private static Pair generateWAL() throws Exception { return Pair.newPair(path, offset); } - @BeforeClass + @BeforeAll public static void setUp() throws Exception { Configuration conf = UTIL.getConfiguration(); FS = UTIL.getTestFileSystem(); @@ -167,7 +162,7 @@ public static void setUp() throws Exception { READER = new ReplicationSourceWALReader(FS, conf, LOG_QUEUE, 0, e -> e, SOURCE, GROUP_ID); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { READER.setReaderRunning(false); READER.join(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCounts.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCounts.java index 88f4f4539b23..41497c3f88f6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCounts.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCounts.java @@ -17,53 +17,38 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.replication.regionserver.WALEntryStream.HasNext; -import org.junit.Before; -import org.junit.Test; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; /** * Try out different combinations of row count and KeyValue count */ public abstract class TestWALEntryStreamDifferentCounts extends WALEntryStreamTestBase { - @Parameter(0) - public int nbRows; + protected int nbRows; + protected int walEditKVs; + protected boolean isCompressionEnabled; - @Parameter(1) - public int walEditKVs; - - @Parameter(2) - public boolean isCompressionEnabled; - - @Parameters(name = "{index}: nbRows={0}, walEditKVs={1}, isCompressionEnabled={2}") - public static Iterable data() { - List params = new ArrayList<>(); - for (int nbRows : new int[] { 1500, 60000 }) { - for (int walEditKVs : new int[] { 1, 100 }) { - for (boolean isCompressionEnabled : new boolean[] { false, true }) { - params.add(new Object[] { nbRows, walEditKVs, isCompressionEnabled }); - } - } - } - return params; + protected TestWALEntryStreamDifferentCounts(int nbRows, int walEditKVs, + boolean isCompressionEnabled) { + this.nbRows = nbRows; + this.walEditKVs = walEditKVs; + this.isCompressionEnabled = isCompressionEnabled; } - @Before + @BeforeEach public void setUp() throws IOException { CONF.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, isCompressionEnabled); initWAL(); } - @Test + @TestTemplate public void testDifferentCounts() throws Exception { mvcc.advanceTo(1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCountsAsyncFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCountsAsyncFSWAL.java index 9dcb8a344b6d..13bab98c1877 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCountsAsyncFSWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCountsAsyncFSWAL.java @@ -17,27 +17,43 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.AsyncFSWALProvider; import org.apache.hadoop.hbase.wal.WALFactory; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.provider.Arguments; -@RunWith(Parameterized.class) -@Category({ ReplicationTests.class, LargeTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate( + name = "{index}: nbRows={0}, walEditKVs={1}, isCompressionEnabled={2}") public class TestWALEntryStreamDifferentCountsAsyncFSWAL extends TestWALEntryStreamDifferentCounts { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestWALEntryStreamDifferentCountsAsyncFSWAL.class); + public TestWALEntryStreamDifferentCountsAsyncFSWAL(int nbRows, int walEditKVs, + boolean isCompressionEnabled) { + super(nbRows, walEditKVs, isCompressionEnabled); + } + + public static Stream parameters() { + List params = new ArrayList<>(); + for (int nbRows : new int[] { 1500, 60000 }) { + for (int walEditKVs : new int[] { 1, 100 }) { + for (boolean isCompressionEnabled : new boolean[] { false, true }) { + params.add(Arguments.of(nbRows, walEditKVs, isCompressionEnabled)); + } + } + } + return params.stream(); + } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setClass(WALFactory.WAL_PROVIDER, AsyncFSWALProvider.class, AbstractFSWALProvider.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCountsFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCountsFSHLog.java index fb7c3d555d8e..8c4f9eb46a47 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCountsFSHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStreamDifferentCountsFSHLog.java @@ -17,27 +17,43 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.FSHLogProvider; import org.apache.hadoop.hbase.wal.WALFactory; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.provider.Arguments; -@RunWith(Parameterized.class) -@Category({ ReplicationTests.class, LargeTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate( + name = "{index}: nbRows={0}, walEditKVs={1}, isCompressionEnabled={2}") public class TestWALEntryStreamDifferentCountsFSHLog extends TestWALEntryStreamDifferentCounts { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestWALEntryStreamDifferentCountsFSHLog.class); + public TestWALEntryStreamDifferentCountsFSHLog(int nbRows, int walEditKVs, + boolean isCompressionEnabled) { + super(nbRows, walEditKVs, isCompressionEnabled); + } + + public static Stream parameters() { + List params = new ArrayList<>(); + for (int nbRows : new int[] { 1500, 60000 }) { + for (int walEditKVs : new int[] { 1, 100 }) { + for (boolean isCompressionEnabled : new boolean[] { false, true }) { + params.add(Arguments.of(nbRows, walEditKVs, isCompressionEnabled)); + } + } + } + return params.stream(); + } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setClass(WALFactory.WAL_PROVIDER, FSHLogProvider.class, AbstractFSWALProvider.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStreamTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStreamTestBase.java index 3af5596b0054..b4b6af30c479 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStreamTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/WALEntryStreamTestBase.java @@ -42,10 +42,10 @@ import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Rule; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; @@ -125,11 +125,14 @@ public void preLogRoll(Path oldPath, Path newPath) { protected WAL log; protected ReplicationSourceLogQueue logQueue; protected PathWatcher pathWatcher; - - @Rule - public TestName tn = new TestName(); + protected String testName; protected final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); + @BeforeEach + public void setUp(TestInfo testInfo) { + testName = testInfo.getTestMethod().get().getName(); + } + protected static void startCluster() throws Exception { CONF = TEST_UTIL.getConfiguration(); CONF.setLong("replication.source.sleepforretries", 10); @@ -139,7 +142,7 @@ protected static void startCluster() throws Exception { fs = cluster.getFileSystem(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } @@ -151,12 +154,12 @@ protected void initWAL() throws IOException { metricsSource.clear(); logQueue = new ReplicationSourceLogQueue(CONF, metricsSource, source); pathWatcher = new PathWatcher(); - final WALFactory wals = new WALFactory(CONF, tn.getMethodName().replaceAll("[\\[:]", "_")); + final WALFactory wals = new WALFactory(CONF, testName.replaceAll("[\\[:]", "_")); wals.getWALProvider().addWALActionsListener(pathWatcher); log = wals.getWAL(info); } - @After + @AfterEach public void tearDown() throws Exception { Closeables.close(log, true); } From 37549634d22cdbeb60eb5b2015cd47dd5d786817 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 28 May 2026 00:04:24 +0800 Subject: [PATCH 2/4] fix --- .../replication/TestReplicationBaseNoBeforeAll.java | 11 +++++++---- .../regionserver/TestReplicationCompressedWAL.java | 7 ------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBaseNoBeforeAll.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBaseNoBeforeAll.java index fb5ebe5c9193..35031e3ede06 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBaseNoBeforeAll.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBaseNoBeforeAll.java @@ -223,21 +223,24 @@ protected static void configureClusters(HBaseTestingUtil util1, HBaseTestingUtil protected static void restartSourceCluster(int numSlaves) throws Exception { Closeables.close(hbaseAdmin, true); Closeables.close(htable1, true); + Closeables.close(connection1, true); UTIL1.shutdownMiniHBaseCluster(); UTIL1.restartHBaseCluster(numSlaves); // Invalidate the cached connection state. CONF1 = UTIL1.getConfiguration(); - hbaseAdmin = UTIL1.getAdmin(); - Connection connection1 = UTIL1.getConnection(); + connection1 = ConnectionFactory.createConnection(CONF1); + hbaseAdmin = connection1.getAdmin(); htable1 = connection1.getTable(tableName); } static void restartTargetHBaseCluster(int numSlaves) throws Exception { Closeables.close(htable2, true); + Closeables.close(connection2, true); UTIL2.restartHBaseCluster(numSlaves); // Invalidate the cached connection state CONF2 = UTIL2.getConfiguration(); - htable2 = UTIL2.getConnection().getTable(tableName); + connection2 = ConnectionFactory.createConnection(CONF2); + htable2 = connection2.getTable(tableName); } protected static void createTable(TableName tableName) throws IOException { @@ -279,7 +282,7 @@ private boolean peerExist(String peerId, HBaseTestingUtil util) throws IOExcepti .anyMatch(p -> peerId.equals(p.getPeerId())); } - // can be override in tests, in case you need to use zk based uri, or the old style uri + // can be overridden in tests, in case you need to use zk based uri, or the old style uri protected String getClusterKey(HBaseTestingUtil util) throws Exception { return util.getRpcConnnectionURI(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationCompressedWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationCompressedWAL.java index 9682b202d440..c57d252bd8ab 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationCompressedWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationCompressedWAL.java @@ -25,12 +25,10 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.replication.TestReplicationBase; import org.apache.hadoop.hbase.replication.TestReplicationBaseNoBeforeAll; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -52,11 +50,6 @@ public static void setUpBeforeClass() throws Exception { startClusters(); } - @AfterAll - public static void tearDownAfterClass() throws Exception { - TestReplicationBase.tearDownAfterClass(); - } - @Test public void testMultiplePuts() throws Exception { runMultiplePutTest(); From 50fee5c84aa0e3aad3d7922bc1dcd7cd97137beb Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 28 May 2026 17:32:49 +0800 Subject: [PATCH 3/4] fix --- .../TestRegisterPeerWorkerWhenRestarting.java | 12 +- ...eerSyncReplicationStateProcedureRetry.java | 7 +- .../replication/SyncReplicationTestBase.java | 278 +--------------- .../SyncReplicationTestBaseNoBeforeAll.java | 304 ++++++++++++++++++ 4 files changed, 313 insertions(+), 288 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBaseNoBeforeAll.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestRegisterPeerWorkerWhenRestarting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestRegisterPeerWorkerWhenRestarting.java index 97c4ae01d2af..bc628023b51e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestRegisterPeerWorkerWhenRestarting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestRegisterPeerWorkerWhenRestarting.java @@ -28,11 +28,10 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.replication.SyncReplicationState; -import org.apache.hadoop.hbase.replication.SyncReplicationTestBase; +import org.apache.hadoop.hbase.replication.SyncReplicationTestBaseNoBeforeAll; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -42,7 +41,7 @@ */ @Tag(MasterTests.TAG) @Tag(LargeTests.TAG) -public class TestRegisterPeerWorkerWhenRestarting extends SyncReplicationTestBase { +public class TestRegisterPeerWorkerWhenRestarting extends SyncReplicationTestBaseNoBeforeAll { private static volatile boolean FAIL = false; @@ -67,12 +66,7 @@ FAIL && getMasterProcedureExecutor() @BeforeAll public static void setUp() throws Exception { UTIL2.getConfiguration().setClass(HConstants.MASTER_IMPL, HMasterForTest.class, HMaster.class); - SyncReplicationTestBase.setUp(); - } - - @AfterAll - public static void tearDown() throws Exception { - SyncReplicationTestBase.tearDown(); + startClusters(); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestTransitPeerSyncReplicationStateProcedureRetry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestTransitPeerSyncReplicationStateProcedureRetry.java index d0b99d9bb65b..c88ffcb4f956 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestTransitPeerSyncReplicationStateProcedureRetry.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestTransitPeerSyncReplicationStateProcedureRetry.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.replication.SyncReplicationState; -import org.apache.hadoop.hbase.replication.SyncReplicationTestBase; +import org.apache.hadoop.hbase.replication.SyncReplicationTestBaseNoBeforeAll; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.junit.jupiter.api.BeforeAll; @@ -38,12 +38,13 @@ @Tag(MasterTests.TAG) @Tag(LargeTests.TAG) -public class TestTransitPeerSyncReplicationStateProcedureRetry extends SyncReplicationTestBase { +public class TestTransitPeerSyncReplicationStateProcedureRetry + extends SyncReplicationTestBaseNoBeforeAll { @BeforeAll public static void setUp() throws Exception { UTIL2.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); - SyncReplicationTestBase.setUp(); + startClusters(); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java index be1af5b18a5c..aee8edf8225a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBase.java @@ -17,291 +17,17 @@ */ package org.apache.hadoop.hbase.replication; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.regex.Pattern; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HBaseTestingUtil; -import org.apache.hadoop.hbase.HBaseZKTestingUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.StartTestingClusterOption; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.AsyncClusterConnection; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.master.MasterFileSystem; -import org.apache.hadoop.hbase.protobuf.ReplicationProtobufUtil; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FutureUtils; -import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKeyImpl; -import org.apache.hadoop.ipc.RemoteException; -import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; - /** * Base class for testing sync replication. */ -public class SyncReplicationTestBase { - - protected static final HBaseZKTestingUtil ZK_UTIL = new HBaseZKTestingUtil(); - - protected static final HBaseTestingUtil UTIL1 = new HBaseTestingUtil(); - - protected static final HBaseTestingUtil UTIL2 = new HBaseTestingUtil(); - - protected static TableName TABLE_NAME = TableName.valueOf("SyncRep"); - - protected static byte[] CF = Bytes.toBytes("cf"); - - protected static byte[] CQ = Bytes.toBytes("cq"); - - protected static String PEER_ID = "1"; - - protected static Path REMOTE_WAL_DIR1; - - protected static Path REMOTE_WAL_DIR2; - - protected static void initTestingUtility(HBaseTestingUtil util, String zkParent) { - util.setZkCluster(ZK_UTIL.getZkCluster()); - Configuration conf = util.getConfiguration(); - conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkParent); - conf.setInt("replication.source.size.capacity", 102400); - conf.setLong("replication.source.sleepforretries", 100); - conf.setInt("hbase.regionserver.maxlogs", 10); - conf.setLong("hbase.master.logcleaner.ttl", 10); - conf.setInt("zookeeper.recovery.retry", 1); - conf.setInt("zookeeper.recovery.retry.intervalmill", 10); - conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); - conf.setInt("replication.stats.thread.period.seconds", 5); - conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); - conf.setLong("replication.sleep.before.failover", 2000); - conf.setInt("replication.source.maxretriesmultiplier", 10); - conf.setFloat("replication.source.ratio", 1.0f); - conf.setBoolean("replication.source.eof.autorecovery", true); - } +public class SyncReplicationTestBase extends SyncReplicationTestBaseNoBeforeAll { @BeforeAll @BeforeClass public static void setUp() throws Exception { - ZK_UTIL.startMiniZKCluster(); - initTestingUtility(UTIL1, "/cluster1"); - initTestingUtility(UTIL2, "/cluster2"); - StartTestingClusterOption option = - StartTestingClusterOption.builder().numMasters(2).numRegionServers(3).numDataNodes(3).build(); - UTIL1.startMiniCluster(option); - UTIL2.startMiniCluster(option); - TableDescriptor td = - TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(CF).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build(); - UTIL1.getAdmin().createTable(td); - UTIL2.getAdmin().createTable(td); - FileSystem fs1 = UTIL1.getTestFileSystem(); - FileSystem fs2 = UTIL2.getTestFileSystem(); - REMOTE_WAL_DIR1 = - new Path(UTIL1.getMiniHBaseCluster().getMaster().getMasterFileSystem().getWALRootDir(), - "remoteWALs").makeQualified(fs1.getUri(), fs1.getWorkingDirectory()); - REMOTE_WAL_DIR2 = - new Path(UTIL2.getMiniHBaseCluster().getMaster().getMasterFileSystem().getWALRootDir(), - "remoteWALs").makeQualified(fs2.getUri(), fs2.getWorkingDirectory()); - UTIL1.getAdmin().addReplicationPeer(PEER_ID, - ReplicationPeerConfig.newBuilder().setClusterKey(UTIL2.getRpcConnnectionURI()) - .setReplicateAllUserTables(false) - .setTableCFsMap(ImmutableMap.of(TABLE_NAME, new ArrayList<>())) - .setRemoteWALDir(REMOTE_WAL_DIR2.toUri().toString()).build()); - UTIL2.getAdmin().addReplicationPeer(PEER_ID, - ReplicationPeerConfig.newBuilder().setClusterKey(UTIL1.getRpcConnnectionURI()) - .setReplicateAllUserTables(false) - .setTableCFsMap(ImmutableMap.of(TABLE_NAME, new ArrayList<>())) - .setRemoteWALDir(REMOTE_WAL_DIR1.toUri().toString()).build()); - } - - private static void shutdown(HBaseTestingUtil util) throws Exception { - if (util.getHBaseCluster() == null) { - return; - } - Admin admin = util.getAdmin(); - if (!admin.listReplicationPeers(Pattern.compile(PEER_ID)).isEmpty()) { - if ( - admin.getReplicationPeerSyncReplicationState(PEER_ID) - != SyncReplicationState.DOWNGRADE_ACTIVE - ) { - admin.transitReplicationPeerSyncReplicationState(PEER_ID, - SyncReplicationState.DOWNGRADE_ACTIVE); - } - admin.removeReplicationPeer(PEER_ID); - } - util.shutdownMiniCluster(); - } - - @AfterAll - @AfterClass - public static void tearDown() throws Exception { - shutdown(UTIL1); - shutdown(UTIL2); - ZK_UTIL.shutdownMiniZKCluster(); - } - - protected final void write(HBaseTestingUtil util, int start, int end) throws IOException { - try (Table table = util.getConnection().getTable(TABLE_NAME)) { - for (int i = start; i < end; i++) { - table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); - } - } - } - - protected final void verify(HBaseTestingUtil util, int start, int end) throws IOException { - try (Table table = util.getConnection().getTable(TABLE_NAME)) { - for (int i = start; i < end; i++) { - assertEquals(i, Bytes.toInt(table.get(new Get(Bytes.toBytes(i))).getValue(CF, CQ))); - } - } - } - - protected final void verifyThroughRegion(HBaseTestingUtil util, int start, int end) - throws IOException { - HRegion region = util.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0); - for (int i = start; i < end; i++) { - assertEquals(i, Bytes.toInt(region.get(new Get(Bytes.toBytes(i))).getValue(CF, CQ))); - } - } - - protected final void verifyNotReplicatedThroughRegion(HBaseTestingUtil util, int start, int end) - throws IOException { - HRegion region = util.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0); - for (int i = start; i < end; i++) { - assertTrue(region.get(new Get(Bytes.toBytes(i))).isEmpty()); - } - } - - protected final void waitUntilReplicationDone(HBaseTestingUtil util, int end) throws Exception { - // The reject check is in RSRpcService so we can still read through HRegion - HRegion region = util.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0); - util.waitFor(30000, new ExplainingPredicate() { - - @Override - public boolean evaluate() throws Exception { - return !region.get(new Get(Bytes.toBytes(end - 1))).isEmpty(); - } - - @Override - public String explainFailure() throws Exception { - return "Replication has not been catched up yet"; - } - }); - } - - protected final void writeAndVerifyReplication(HBaseTestingUtil util1, HBaseTestingUtil util2, - int start, int end) throws Exception { - write(util1, start, end); - waitUntilReplicationDone(util2, end); - verifyThroughRegion(util2, start, end); - } - - protected final Path getRemoteWALDir(MasterFileSystem mfs, String peerId) { - Path remoteWALDir = new Path(mfs.getWALRootDir(), ReplicationUtils.REMOTE_WAL_DIR_NAME); - return getRemoteWALDir(remoteWALDir, peerId); - } - - protected final Path getRemoteWALDir(Path remoteWALDir, String peerId) { - return new Path(remoteWALDir, peerId); - } - - protected final Path getReplayRemoteWALs(Path remoteWALDir, String peerId) { - return new Path(remoteWALDir, peerId + "-replay"); - } - - protected final void verifyRemovedPeer(String peerId, Path remoteWALDir, HBaseTestingUtil utility) - throws Exception { - ReplicationPeerStorage rps = ReplicationStorageFactory.getReplicationPeerStorage( - utility.getTestFileSystem(), utility.getZooKeeperWatcher(), utility.getConfiguration()); - try { - rps.getPeerSyncReplicationState(peerId); - fail("Should throw exception when get the sync replication state of a removed peer."); - } catch (ReplicationException e) { - // ignore. - } - try { - rps.getPeerNewSyncReplicationState(peerId); - fail("Should throw exception when get the new sync replication state of a removed peer"); - } catch (ReplicationException e) { - // ignore. - } - try (FileSystem fs = utility.getTestFileSystem()) { - assertFalse(fs.exists(getRemoteWALDir(remoteWALDir, peerId))); - assertFalse(fs.exists(getReplayRemoteWALs(remoteWALDir, peerId))); - } - } - - private void assertRejection(Throwable error) { - assertThat(error, instanceOf(DoNotRetryIOException.class)); - assertTrue(error.getMessage().contains("Reject to apply to sink cluster")); - assertTrue(error.getMessage().contains(TABLE_NAME.toString())); - } - - protected final void verifyReplicationRequestRejection(HBaseTestingUtil utility, - boolean expectedRejection) throws Exception { - HRegionServer regionServer = utility.getRSForFirstRegionInTable(TABLE_NAME); - AsyncClusterConnection connection = regionServer.getAsyncClusterConnection(); - Entry[] entries = new Entry[10]; - for (int i = 0; i < entries.length; i++) { - entries[i] = - new Entry(new WALKeyImpl(HConstants.EMPTY_BYTE_ARRAY, TABLE_NAME, 0), new WALEdit()); - } - if (!expectedRejection) { - FutureUtils.get(ReplicationProtobufUtil.replicateWALEntry( - connection.getRegionServerAdmin(regionServer.getServerName()), entries, null, null, null, - HConstants.REPLICATION_SOURCE_SHIPEDITS_TIMEOUT_DFAULT)); - } else { - try { - FutureUtils.get(ReplicationProtobufUtil.replicateWALEntry( - connection.getRegionServerAdmin(regionServer.getServerName()), entries, null, null, null, - HConstants.REPLICATION_SOURCE_SHIPEDITS_TIMEOUT_DFAULT)); - fail("Should throw IOException when sync-replication state is in A or DA"); - } catch (RemoteException e) { - assertRejection(e.unwrapRemoteException()); - } catch (DoNotRetryIOException e) { - assertRejection(e); - } - } - } - - protected final void waitUntilDeleted(HBaseTestingUtil util, Path remoteWAL) throws Exception { - MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem(); - util.waitFor(30000, new ExplainingPredicate() { - - @Override - public boolean evaluate() throws Exception { - return !mfs.getWALFileSystem().exists(remoteWAL); - } - - @Override - public String explainFailure() throws Exception { - return remoteWAL + " has not been deleted yet"; - } - }); + startClusters(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBaseNoBeforeAll.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBaseNoBeforeAll.java new file mode 100644 index 000000000000..13a7ebec346d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/SyncReplicationTestBaseNoBeforeAll.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.regex.Pattern; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HBaseZKTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.StartTestingClusterOption; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.AsyncClusterConnection; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.protobuf.ReplicationProtobufUtil; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FutureUtils; +import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALKeyImpl; +import org.apache.hadoop.ipc.RemoteException; +import org.junit.AfterClass; +import org.junit.jupiter.api.AfterAll; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; + +/** + * Sync replication test base without BeforeAll method. + * @see SyncReplicationTestBase + */ +public class SyncReplicationTestBaseNoBeforeAll { + + protected static final HBaseZKTestingUtil ZK_UTIL = new HBaseZKTestingUtil(); + + protected static final HBaseTestingUtil UTIL1 = new HBaseTestingUtil(); + + protected static final HBaseTestingUtil UTIL2 = new HBaseTestingUtil(); + + protected static TableName TABLE_NAME = TableName.valueOf("SyncRep"); + + protected static byte[] CF = Bytes.toBytes("cf"); + + protected static byte[] CQ = Bytes.toBytes("cq"); + + protected static String PEER_ID = "1"; + + protected static Path REMOTE_WAL_DIR1; + + protected static Path REMOTE_WAL_DIR2; + + protected static void initTestingUtility(HBaseTestingUtil util, String zkParent) { + util.setZkCluster(ZK_UTIL.getZkCluster()); + Configuration conf = util.getConfiguration(); + conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkParent); + conf.setInt("replication.source.size.capacity", 102400); + conf.setLong("replication.source.sleepforretries", 100); + conf.setInt("hbase.regionserver.maxlogs", 10); + conf.setLong("hbase.master.logcleaner.ttl", 10); + conf.setInt("zookeeper.recovery.retry", 1); + conf.setInt("zookeeper.recovery.retry.intervalmill", 10); + conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); + conf.setInt("replication.stats.thread.period.seconds", 5); + conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); + conf.setLong("replication.sleep.before.failover", 2000); + conf.setInt("replication.source.maxretriesmultiplier", 10); + conf.setFloat("replication.source.ratio", 1.0f); + conf.setBoolean("replication.source.eof.autorecovery", true); + } + + protected static void startClusters() throws Exception { + ZK_UTIL.startMiniZKCluster(); + initTestingUtility(UTIL1, "/cluster1"); + initTestingUtility(UTIL2, "/cluster2"); + StartTestingClusterOption option = + StartTestingClusterOption.builder().numMasters(2).numRegionServers(3).numDataNodes(3).build(); + UTIL1.startMiniCluster(option); + UTIL2.startMiniCluster(option); + TableDescriptor td = + TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(CF).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build(); + UTIL1.getAdmin().createTable(td); + UTIL2.getAdmin().createTable(td); + FileSystem fs1 = UTIL1.getTestFileSystem(); + FileSystem fs2 = UTIL2.getTestFileSystem(); + REMOTE_WAL_DIR1 = + new Path(UTIL1.getMiniHBaseCluster().getMaster().getMasterFileSystem().getWALRootDir(), + "remoteWALs").makeQualified(fs1.getUri(), fs1.getWorkingDirectory()); + REMOTE_WAL_DIR2 = + new Path(UTIL2.getMiniHBaseCluster().getMaster().getMasterFileSystem().getWALRootDir(), + "remoteWALs").makeQualified(fs2.getUri(), fs2.getWorkingDirectory()); + UTIL1.getAdmin().addReplicationPeer(PEER_ID, + ReplicationPeerConfig.newBuilder().setClusterKey(UTIL2.getRpcConnnectionURI()) + .setReplicateAllUserTables(false) + .setTableCFsMap(ImmutableMap.of(TABLE_NAME, new ArrayList<>())) + .setRemoteWALDir(REMOTE_WAL_DIR2.toUri().toString()).build()); + UTIL2.getAdmin().addReplicationPeer(PEER_ID, + ReplicationPeerConfig.newBuilder().setClusterKey(UTIL1.getRpcConnnectionURI()) + .setReplicateAllUserTables(false) + .setTableCFsMap(ImmutableMap.of(TABLE_NAME, new ArrayList<>())) + .setRemoteWALDir(REMOTE_WAL_DIR1.toUri().toString()).build()); + } + + private static void shutdown(HBaseTestingUtil util) throws Exception { + if (util.getHBaseCluster() == null) { + return; + } + Admin admin = util.getAdmin(); + if (!admin.listReplicationPeers(Pattern.compile(PEER_ID)).isEmpty()) { + if ( + admin.getReplicationPeerSyncReplicationState(PEER_ID) + != SyncReplicationState.DOWNGRADE_ACTIVE + ) { + admin.transitReplicationPeerSyncReplicationState(PEER_ID, + SyncReplicationState.DOWNGRADE_ACTIVE); + } + admin.removeReplicationPeer(PEER_ID); + } + util.shutdownMiniCluster(); + } + + @AfterAll + @AfterClass + public static void tearDown() throws Exception { + shutdown(UTIL1); + shutdown(UTIL2); + ZK_UTIL.shutdownMiniZKCluster(); + } + + protected final void write(HBaseTestingUtil util, int start, int end) throws IOException { + try (Table table = util.getConnection().getTable(TABLE_NAME)) { + for (int i = start; i < end; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + } + + protected final void verify(HBaseTestingUtil util, int start, int end) throws IOException { + try (Table table = util.getConnection().getTable(TABLE_NAME)) { + for (int i = start; i < end; i++) { + assertEquals(i, Bytes.toInt(table.get(new Get(Bytes.toBytes(i))).getValue(CF, CQ))); + } + } + } + + protected final void verifyThroughRegion(HBaseTestingUtil util, int start, int end) + throws IOException { + HRegion region = util.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0); + for (int i = start; i < end; i++) { + assertEquals(i, Bytes.toInt(region.get(new Get(Bytes.toBytes(i))).getValue(CF, CQ))); + } + } + + protected final void verifyNotReplicatedThroughRegion(HBaseTestingUtil util, int start, int end) + throws IOException { + HRegion region = util.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0); + for (int i = start; i < end; i++) { + assertTrue(region.get(new Get(Bytes.toBytes(i))).isEmpty()); + } + } + + protected final void waitUntilReplicationDone(HBaseTestingUtil util, int end) throws Exception { + // The reject check is in RSRpcService so we can still read through HRegion + HRegion region = util.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0); + util.waitFor(30000, new ExplainingPredicate() { + + @Override + public boolean evaluate() throws Exception { + return !region.get(new Get(Bytes.toBytes(end - 1))).isEmpty(); + } + + @Override + public String explainFailure() throws Exception { + return "Replication has not been catched up yet"; + } + }); + } + + protected final void writeAndVerifyReplication(HBaseTestingUtil util1, HBaseTestingUtil util2, + int start, int end) throws Exception { + write(util1, start, end); + waitUntilReplicationDone(util2, end); + verifyThroughRegion(util2, start, end); + } + + protected final Path getRemoteWALDir(MasterFileSystem mfs, String peerId) { + Path remoteWALDir = new Path(mfs.getWALRootDir(), ReplicationUtils.REMOTE_WAL_DIR_NAME); + return getRemoteWALDir(remoteWALDir, peerId); + } + + protected final Path getRemoteWALDir(Path remoteWALDir, String peerId) { + return new Path(remoteWALDir, peerId); + } + + protected final Path getReplayRemoteWALs(Path remoteWALDir, String peerId) { + return new Path(remoteWALDir, peerId + "-replay"); + } + + protected final void verifyRemovedPeer(String peerId, Path remoteWALDir, HBaseTestingUtil utility) + throws Exception { + ReplicationPeerStorage rps = ReplicationStorageFactory.getReplicationPeerStorage( + utility.getTestFileSystem(), utility.getZooKeeperWatcher(), utility.getConfiguration()); + try { + rps.getPeerSyncReplicationState(peerId); + fail("Should throw exception when get the sync replication state of a removed peer."); + } catch (ReplicationException e) { + // ignore. + } + try { + rps.getPeerNewSyncReplicationState(peerId); + fail("Should throw exception when get the new sync replication state of a removed peer"); + } catch (ReplicationException e) { + // ignore. + } + try (FileSystem fs = utility.getTestFileSystem()) { + assertFalse(fs.exists(getRemoteWALDir(remoteWALDir, peerId))); + assertFalse(fs.exists(getReplayRemoteWALs(remoteWALDir, peerId))); + } + } + + private void assertRejection(Throwable error) { + assertThat(error, instanceOf(DoNotRetryIOException.class)); + assertTrue(error.getMessage().contains("Reject to apply to sink cluster")); + assertTrue(error.getMessage().contains(TABLE_NAME.toString())); + } + + protected final void verifyReplicationRequestRejection(HBaseTestingUtil utility, + boolean expectedRejection) throws Exception { + HRegionServer regionServer = utility.getRSForFirstRegionInTable(TABLE_NAME); + AsyncClusterConnection connection = regionServer.getAsyncClusterConnection(); + Entry[] entries = new Entry[10]; + for (int i = 0; i < entries.length; i++) { + entries[i] = + new Entry(new WALKeyImpl(HConstants.EMPTY_BYTE_ARRAY, TABLE_NAME, 0), new WALEdit()); + } + if (!expectedRejection) { + FutureUtils.get(ReplicationProtobufUtil.replicateWALEntry( + connection.getRegionServerAdmin(regionServer.getServerName()), entries, null, null, null, + HConstants.REPLICATION_SOURCE_SHIPEDITS_TIMEOUT_DFAULT)); + } else { + try { + FutureUtils.get(ReplicationProtobufUtil.replicateWALEntry( + connection.getRegionServerAdmin(regionServer.getServerName()), entries, null, null, null, + HConstants.REPLICATION_SOURCE_SHIPEDITS_TIMEOUT_DFAULT)); + fail("Should throw IOException when sync-replication state is in A or DA"); + } catch (RemoteException e) { + assertRejection(e.unwrapRemoteException()); + } catch (DoNotRetryIOException e) { + assertRejection(e); + } + } + } + + protected final void waitUntilDeleted(HBaseTestingUtil util, Path remoteWAL) throws Exception { + MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem(); + util.waitFor(30000, new ExplainingPredicate() { + + @Override + public boolean evaluate() throws Exception { + return !mfs.getWALFileSystem().exists(remoteWAL); + } + + @Override + public String explainFailure() throws Exception { + return remoteWAL + " has not been deleted yet"; + } + }); + } +} From cc34c05eae6059451fc7e336cf233e8d38eb216e Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 29 May 2026 14:28:18 +0800 Subject: [PATCH 4/4] fix --- .../regionserver/TestReplicationSourceManagerJoin.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerJoin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerJoin.java index a623787bf3e1..3362a223fbd6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerJoin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerJoin.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.junit.BeforeClass; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -46,7 +45,6 @@ public class TestReplicationSourceManagerJoin extends TestReplicationBaseNoBeforeAll { @BeforeAll - @BeforeClass public static void setUpBeforeClass() throws Exception { // NUM_SLAVES1 is presumed 2 in below. NUM_SLAVES1 = 2;