diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index bfefc3b5f023..54e4d142b82a 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -368,7 +368,6 @@ llap.query.negative.files=\ ctas_noperm_loc.q,\ file_with_header_footer_negative.q,\ msck_repair_5.q,\ - msck_repair_6.q,\ stack_trace.q,\ table_nonprintable_negative.q,\ udf_local_resource.q diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java index a6188e1ad97e..2f316bec0d78 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java @@ -432,6 +432,30 @@ public void testPartitionsCheck() throws HiveException, System.err.println("Test completed - partition check"); } + @Test + public void testPartitionUriMismatchCheck() + throws HiveException, IOException, TException, MetastoreException { + Table table = createTestTable(false); + + List partitions = hive.getPartitions(table); + Partition part = partitions.getFirst(); + + // Change location to inject "localhost" authority to simulate HDFS HA mismatch + String originalPathStr = part.getDataLocation().toString(); + String manipulatedPathStr = originalPathStr.replace("file:/", "file://localhost/"); + + part.setLocation(manipulatedPathStr); + hive.alterPartition(catName, dbName, tableName, part, null, true); + + CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null); + + // MSCK should succeed and NOT report this partition as missing from FS or missing from DB + assertEquals(Collections.emptySet(), result.getTablesNotInMs()); + assertEquals(Collections.emptySet(), result.getTablesNotOnFs()); + assertEquals(Collections.emptySet(), result.getPartitionsNotOnFs()); + assertEquals(Collections.emptySet(), result.getPartitionsNotInMs()); + } + @Test public void testDataDeletion() throws HiveException, IOException, TException { diff --git a/ql/src/test/queries/clientnegative/msck_repair_6.q b/ql/src/test/queries/clientpositive/msck_repair_ignore_duplicate_dir.q similarity index 100% rename from ql/src/test/queries/clientnegative/msck_repair_6.q rename to ql/src/test/queries/clientpositive/msck_repair_ignore_duplicate_dir.q diff --git a/ql/src/test/results/clientnegative/msck_repair_6.q.out b/ql/src/test/results/clientpositive/llap/msck_repair_ignore_duplicate_dir.q.out similarity index 71% rename from ql/src/test/results/clientnegative/msck_repair_6.q.out rename to ql/src/test/results/clientpositive/llap/msck_repair_ignore_duplicate_dir.q.out index a6c22cc4779a..40dd8982f215 100644 --- a/ql/src/test/results/clientnegative/msck_repair_6.q.out +++ b/ql/src/test/results/clientpositive/llap/msck_repair_ignore_duplicate_dir.q.out @@ -29,4 +29,16 @@ Partitions not in metastore: repairtable:p1=a PREHOOK: query: MSCK REPAIR TABLE default.repairtable PREHOOK: type: MSCK PREHOOK: Output: default@repairtable -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. The partition 'repairtable:p1=a' already exists for tablerepairtable +POSTHOOK: query: MSCK REPAIR TABLE default.repairtable +POSTHOOK: type: MSCK +POSTHOOK: Output: default@repairtable +PREHOOK: query: DROP TABLE default.repairtable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@repairtable +PREHOOK: Output: database:default +PREHOOK: Output: default@repairtable +POSTHOOK: query: DROP TABLE default.repairtable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@repairtable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repairtable diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java index 11dc51b827ad..ccc604569712 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java @@ -457,13 +457,21 @@ void findUnknownPartitions(Table table, Set missingPartDirs, // will not return any results. pr.setPath(partPath); - // Check if partition already exists. No need to check for those partition which are present in db - // but no in fs as msck will override the partition location in db + /* + Check if partition already exists. No need to check for those partitions which are present in db + but not in fs as msck will override the partition location in db. + This edge case happens when a partition's URI in the metastore contains a port (e.g., :8020) + while the table's default FS does not (e.g., HA nameservice). The string mismatch causes + allPartDirs.remove() to fail, leaving the physical directory in the unknown list even though + it was successfully verified and added to correctPartitions. We can safely skip it. + */ if (result.getCorrectPartitions().contains(pr)) { - String msg = "The partition '" + pr.toString() + "' already exists for table" + table.getTableName(); - throw new MetastoreException(msg); + if (LOG.isDebugEnabled()) { + LOG.debug("The partition '{}' already exists in metastore for table {}", pr, table.getTableName()); + } + continue; } else if (result.getPartitionsNotInMs().contains(pr)) { - String msg = "Found two paths for same partition '" + pr.toString() + "' for table " + table.getTableName(); + String msg = "Found two paths for same partition '" + pr + "' for table " + table.getTableName(); throw new MetastoreException(msg); } if (transactionalTable) {