From 75be624cc2cfb981ba26e07f0a4f40e5844a32d5 Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Thu, 16 Jul 2026 16:50:04 +0300 Subject: [PATCH] SOLR-12239: add RESORTINDEX core-admin action to re-sort an existing index Enabling index sorting on a collection created without it currently fails on core reload with an IndexWriter.validateIndexSort error, forcing a delete + reindex from source. This adds a RESORTINDEX core-admin action that re-sorts the existing index into the requested sort order in place, using the LUCENE-9484 mechanism: each segment reader is wrapped in a SortingCodecReader and merged into a fresh sort-configured IndexWriter via addIndexes(CodecReader...), and the result is swapped in with modifyIndexProps (as RestoreCore/replication do), then the writer and searcher are reopened. Notes: - addIndexes does not auto-sort unsorted readers (LUCENE-8505 removed that), so the explicit SortingCodecReader wrap is what performs the merge-based re-sort. - Not supported in SolrCloud mode. - Indexes containing child/nested documents are rejected (re-sorting would break the parent-child blocks), matching UPGRADEINDEX's restriction. - On a failed swap the original index.properties is restored (RestoreCore's rollback). The target sort is given by the sort request parameter using the usual Solr sort syntax; the fields must have docValues. --- ...R-12239-index-sort-existing-collection.yml | 10 + .../api/endpoint/ResortCoreIndexApi.java | 43 +++ .../api/model/ResortCoreIndexRequestBody.java | 35 +++ .../api/model/ResortCoreIndexResponse.java | 31 +++ .../solr/handler/admin/CoreAdminHandler.java | 2 + .../handler/admin/CoreAdminOperation.java | 4 +- .../solr/handler/admin/ResortIndexOp.java | 53 ++++ .../handler/admin/api/ResortCoreIndex.java | 246 ++++++++++++++++++ .../handler/admin/ResortIndexActionTest.java | 211 +++++++++++++++ .../admin/ResortIndexConfiguredSortTest.java | 73 ++++++ .../admin/api/ResortCoreIndexTest.java | 97 +++++++ .../pages/coreadmin-api.adoc | 90 +++++++ .../solr/common/params/CoreAdminParams.java | 3 +- 13 files changed, 896 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml create mode 100644 solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java diff --git a/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml b/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml new file mode 100644 index 000000000000..fde7d60839bd --- /dev/null +++ b/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml @@ -0,0 +1,10 @@ +title: > + New RESORTINDEX core-admin action re-sorts an existing core's index into a configured sort order, + so index sorting can be enabled on a collection created without it, without deleting and reindexing + from source. Not supported in SolrCloud mode; indexes with child/nested documents are rejected. +type: added +authors: + - name: Serhiy Bzhezytskyy +links: + - name: SOLR-12239 + url: https://issues.apache.org/jira/browse/SOLR-12239 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java new file mode 100644 index 000000000000..6a2521f993a8 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; + +/** V2 API definition for re-sorting an existing core index (SOLR-12239). */ +@Path("/cores/{coreName}/resort") +public interface ResortCoreIndexApi { + + @POST + @Operation( + summary = + "Re-sort the existing index of the specified core into the target sort order, so index " + + "sorting can be enabled without reindexing from source.", + tags = {"cores"}) + ResortCoreIndexResponse resortCoreIndex( + @Parameter(description = "The name of the core whose index should be re-sorted") + @PathParam("coreName") + String coreName, + ResortCoreIndexRequestBody requestBody) + throws Exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java new file mode 100644 index 000000000000..4dfb2fa180b4 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class ResortCoreIndexRequestBody { + + @Schema( + description = + "The target index sort, in Solr sort syntax (e.g. 'timestamp desc'). If omitted, the " + + "sort configured for the core (via a SortingMergePolicy) is used. Fields must have " + + "docValues.") + @JsonProperty + public String sort; + + @Schema(description = "Request ID to track this action which will be processed asynchronously.") + @JsonProperty + public String async; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java new file mode 100644 index 000000000000..dce2fd337f3e --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class ResortCoreIndexResponse extends SolrJerseyResponse { + + @Schema(description = "The name of the core whose index was re-sorted.") + @JsonProperty + public String core; + + @Schema(description = "The index sort that was applied.") + @JsonProperty + public String indexSort; +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index 494e3f898414..de2d6e7c7019 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -69,6 +69,7 @@ import org.apache.solr.handler.admin.api.RequestBufferUpdatesAPI; import org.apache.solr.handler.admin.api.RequestCoreRecoveryAPI; import org.apache.solr.handler.admin.api.RequestSyncShardAPI; +import org.apache.solr.handler.admin.api.ResortCoreIndex; import org.apache.solr.handler.admin.api.RestoreCore; import org.apache.solr.handler.admin.api.SplitCoreAPI; import org.apache.solr.handler.admin.api.SwapCores; @@ -339,6 +340,7 @@ public Collection> getJerseyResources() { SwapCores.class, RenameCore.class, MergeIndexes.class, + ResortCoreIndex.class, GetNodeCommandStatus.class); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java index 2edc70aef30b..5213e3027898 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java @@ -33,6 +33,7 @@ import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTRECOVERY; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTSTATUS; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTSYNCSHARD; +import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.RESORTINDEX; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.RESTORECORE; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.SPLIT; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.STATUS; @@ -258,7 +259,8 @@ public enum CoreAdminOperation implements CoreAdminOp { V2ApiUtils.squashIntoSolrResponseWithoutHeader(it.rsp, response); }), - UPGRADEINDEX_OP(UPGRADEINDEX, new UpgradeCoreIndexOp()); + UPGRADEINDEX_OP(UPGRADEINDEX, new UpgradeCoreIndexOp()), + RESORTINDEX_OP(RESORTINDEX, new ResortIndexOp()); final CoreAdminParams.CoreAdminAction action; final CoreAdminOp fun; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java b/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java new file mode 100644 index 000000000000..3a4c0efc2651 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.api.ResortCoreIndex; +import org.apache.solr.handler.api.V2ApiUtils; + +/** + * v1 ({@code action=RESORTINDEX}) wrapper delegating to the {@link ResortCoreIndex} V2 API + * (SOLR-12239). + */ +class ResortIndexOp implements CoreAdminHandler.CoreAdminOp { + + @Override + public boolean isExpensive() { + return true; + } + + @Override + public void execute(CoreAdminHandler.CallInfo it) throws Exception { + final SolrParams params = it.req.getParams(); + final String cname = params.required().get(CoreAdminParams.CORE); + final var requestBody = new ResortCoreIndexRequestBody(); + // "async" is intentionally omitted because CoreAdminHandler has already processed it. + requestBody.sort = params.get(CommonParams.SORT); + + final CoreContainer coreContainer = it.handler.getCoreContainer(); + final ResortCoreIndex api = + new ResortCoreIndex(coreContainer, it.handler.getCoreAdminAsyncTracker(), it.req, it.rsp); + final ResortCoreIndexResponse response = api.resortCoreIndex(cname, requestBody); + V2ApiUtils.squashIntoSolrResponseWithoutHeader(it.rsp, response); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java new file mode 100644 index 000000000000..2e350072ebef --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SlowCodecReaderWrapper; +import org.apache.lucene.index.SortingCodecReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.Sort; +import org.apache.lucene.store.Directory; +import org.apache.solr.client.api.endpoint.ResortCoreIndexApi; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.IndexFetcher; +import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SortSpecParsing; +import org.apache.solr.util.RefCounted; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * V2 API implementation for re-sorting an existing core index (SOLR-12239). + * + *

Re-sorts a (possibly unsorted) core index into the target {@link Sort} using the LUCENE-9484 + * mechanism — each segment reader is wrapped in a {@link SortingCodecReader} and merged into a + * fresh sort-configured {@link IndexWriter} via {@link IndexWriter#addIndexes(CodecReader...)} — + * then swaps it in via {@code modifyIndexProps} and reopens the writer/searcher (mirroring + * RestoreCore). + * + *

Not supported in SolrCloud mode. Indexes with child/nested documents are rejected. + */ +public class ResortCoreIndex extends CoreAdminAPIBase implements ResortCoreIndexApi { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ResortCoreIndex( + CoreContainer coreContainer, + CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker, + SolrQueryRequest req, + SolrQueryResponse rsp) { + super(coreContainer, coreAdminAsyncTracker, req, rsp); + } + + @Override + public boolean isExpensive() { + return true; + } + + @Override + public ResortCoreIndexResponse resortCoreIndex( + String coreName, ResortCoreIndexRequestBody requestBody) throws Exception { + ensureRequiredParameterProvided("coreName", coreName); + if (coreContainer.isZooKeeperAware()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "resort is not supported in SolrCloud mode"); + } + final ResortCoreIndexResponse response = + instantiateJerseyResponse(ResortCoreIndexResponse.class); + final String sortParam = requestBody == null ? null : requestBody.sort; + final String async = requestBody == null ? null : requestBody.async; + + return handlePotentiallyAsynchronousTask( + response, + coreName, + async, + "resort-index", + () -> { + try (SolrCore core = coreContainer.getCore(coreName)) { + if (core == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Core not found: " + coreName); + } + final Sort indexSort = resolveTargetSort(core, sortParam); + assertNoChildDocs(core); + resortAndSwap(core, indexSort); + response.core = coreName; + response.indexSort = indexSort.toString(); + } catch (Exception e) { + throw new CoreAdminAPIBaseException(e); + } + return response; + }); + } + + private Sort resolveTargetSort(SolrCore core, String sortParam) throws IOException { + if (sortParam != null && !sortParam.isBlank()) { + final Sort sort = SortSpecParsing.parseSortSpec(sortParam, core.getLatestSchema()).getSort(); + if (sort == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Could not parse a usable index sort from sort=" + sortParam); + } + return sort; + } + final Sort configured = core.getSolrCoreState().getMergePolicySort(); + if (configured == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "No 'sort' parameter given and the core has no configured index sort " + + "(SortingMergePolicy) to fall back to"); + } + return configured; + } + + private void assertNoChildDocs(SolrCore core) throws IOException { + RefCounted ref = core.getSearcher(); + try { + SolrIndexSearcher searcher = ref.get(); + if (!searcher.getSchema().isUsableForChildDocs()) { + return; + } + String uniqueKeyField = searcher.getSchema().getUniqueKeyField().getName(); + for (LeafReaderContext leaf : searcher.getIndexReader().leaves()) { + Terms rootTerms = leaf.reader().terms(IndexSchema.ROOT_FIELD_NAME); + if (rootTerms == null) { + continue; + } + long uniqueRootValues = rootTerms.size(); + Terms idTerms = leaf.reader().terms(uniqueKeyField); + long uniqueIdValues = (idTerms != null) ? idTerms.size() : -1; + if (uniqueRootValues == -1 || uniqueIdValues == -1 || uniqueRootValues < uniqueIdValues) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "resort does not support indexes containing child/nested documents. " + + "Consider reindexing your data from the original source."); + } + } + } finally { + ref.decref(); + } + } + + private void resortAndSwap(SolrCore core, Sort indexSort) throws Exception { + final String resortIndexName = "resort." + System.nanoTime(); + final String resortIndexPath = core.getDataDir() + resortIndexName; + final String currentIndexPath = core.getIndexDir(); + final DirectoryFactory df = core.getDirectoryFactory(); + final String lockType = core.getSolrConfig().indexConfig.lockType; + + Directory currentDir = null; + Directory resortDir = null; + try { + currentDir = df.get(currentIndexPath, DirectoryFactory.DirContext.DEFAULT, lockType); + resortDir = df.get(resortIndexPath, DirectoryFactory.DirContext.DEFAULT, lockType); + + final IndexWriterConfig iwc = new IndexWriterConfig().setIndexSort(indexSort); + // Mirror SolrIndexConfig: a child-doc-capable schema records a parent field, which the source + // segments carry, so the re-sort writer must declare the same parent field or addIndexes + // fails. + if (core.getLatestSchema().isUsableForChildDocs()) { + iwc.setParentField(IndexSchema.IS_ROOT_FIELD_NAME); + } + + try (DirectoryReader reader = DirectoryReader.open(currentDir); + IndexWriter resortWriter = new IndexWriter(resortDir, iwc)) { + final List wrapped = new ArrayList<>(reader.leaves().size()); + for (LeafReaderContext ctx : reader.leaves()) { + // addIndexes does NOT auto-sort (LUCENE-8505); the SortingCodecReader wrap performs the + // (merge-based) re-sort of each segment. + wrapped.add( + SortingCodecReader.wrap(SlowCodecReaderWrapper.wrap(ctx.reader()), indexSort)); + } + resortWriter.addIndexes(wrapped.toArray(new CodecReader[0])); + resortWriter.commit(); + } + } finally { + if (currentDir != null) df.release(currentDir); + if (resortDir != null) df.release(resortDir); + } + + if (!core.modifyIndexProps(resortIndexName)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to point core " + core.getName() + " at the re-sorted index"); + } + try { + core.getUpdateHandler().newIndexWriter(false); + openNewSearcher(core); + } catch (Exception e) { + log.warn("Could not switch to re-sorted index for core {}; rolling back", core.getName(), e); + rollbackIndexProps(core, df, lockType); + core.getUpdateHandler().newIndexWriter(false); + openNewSearcher(core); + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to swap in the re-sorted index for core " + core.getName(), + e); + } + } + + private void rollbackIndexProps(SolrCore core, DirectoryFactory df, String lockType) { + Directory dataDir = null; + try { + dataDir = df.get(core.getDataDir(), DirectoryFactory.DirContext.META_DATA, lockType); + dataDir.deleteFile(IndexFetcher.INDEX_PROPERTIES); + } catch (Exception rollbackError) { + log.error("Rollback of index.properties failed for core {}", core.getName(), rollbackError); + } finally { + if (dataDir != null) { + try { + df.release(dataDir); + } catch (IOException ignored) { + } + } + } + } + + private void openNewSearcher(SolrCore core) throws Exception { + final Future[] waitSearcher = new Future[1]; + core.getSearcher(true, false, waitSearcher, true); + if (waitSearcher[0] != null) { + waitSearcher[0].get(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java new file mode 100644 index 000000000000..198d24f199b1 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import static org.hamcrest.CoreMatchers.containsString; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.util.RefCounted; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * End-to-end test of the {@code RESORTINDEX} core-admin action (SOLR-12239): re-sort an existing + * (unsorted) core index and swap it in, so the core serves a sorted index without reindexing. + */ +public class ResortIndexActionTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + // Plain config, no index sort configured -> the initial index is UNSORTED. + initCore("solrconfig.xml", "schema.xml"); + } + + @Before + public void clearIndexBefore() { + assertU(delQ("*:*")); + assertU(commit()); + } + + @Test + public void testResortIndexAction() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Build an existing unsorted index (intDvoDefault values out of order). + int[] vals = {5, 1, 4, 2, 3, 0}; + for (int v : vals) { + assertU(adoc("id", Integer.toString(v), "intDvoDefault", Integer.toString(v))); + } + assertU(commit()); + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "intDvoDefault asc"), + resp); + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + assertEquals(coreName, resp.getValues().get("core")); + } finally { + admin.shutdown(); + admin.close(); + } + + // All docs still present and queryable after the resort+swap. + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + + // The index is now physically sorted by intDvoDefault ascending (internal docid order). + RefCounted ref = core.getSearcher(); + try { + var storedFields = ref.get().getIndexReader().storedFields(); + int maxDoc = ref.get().getIndexReader().maxDoc(); + long prev = Long.MIN_VALUE; + for (int i = 0; i < maxDoc; i++) { + long id = Long.parseLong(storedFields.document(i).get("id")); + assertTrue( + "internal docids must be ascending after resort: " + prev + " then " + id, id >= prev); + prev = id; + } + assertEquals( + "smallest value sorts first", 0L, Long.parseLong(storedFields.document(0).get("id"))); + } finally { + ref.decref(); + } + } + + @Test + public void testResortIndexRequiresSortWhenNoneConfigured() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + assertU(adoc("id", "1", "intDvoDefault", "1")); + assertU(commit()); + + // No 'sort' param, and this core has no SortingMergePolicy configured -> clear error. + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp)); + assertThat(thrown.getMessage(), containsString("no configured index sort")); + } finally { + admin.shutdown(); + admin.close(); + } + } + + @Test + public void testResortIndexRejectsUnparseableSort() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + assertU(adoc("id", "1", "intDvoDefault", "1")); + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "no_such_field asc"), + resp)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, thrown.code()); + } finally { + admin.shutdown(); + admin.close(); + } + } + + @Test + public void testResortIndexRejectsChildDocs() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Index a parent+child block; re-sorting would break the block, so it must be rejected. + SolrInputDocument parent = new SolrInputDocument(); + parent.addField("id", "100"); + parent.addField("title", "parent"); + SolrInputDocument child = new SolrInputDocument(); + child.addField("id", "101"); + child.addField("title", "child"); + parent.addChildDocument(child); + try (SolrQueryRequestBase addReq = + new SolrQueryRequestBase(core, new ModifiableSolrParams()) {}) { + AddUpdateCommand cmd = new AddUpdateCommand(addReq); + cmd.solrDoc = parent; + core.getUpdateHandler().addDoc(cmd); + } + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "intDvoDefault asc"), + resp)); + assertThat(thrown.getMessage(), containsString("child/nested documents")); + } finally { + admin.shutdown(); + admin.close(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java new file mode 100644 index 000000000000..d4e32c6ec6be --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Verifies that {@code RESORTINDEX} falls back to the sort configured on the core (via a + * SortingMergePolicy) when no explicit {@code sort} parameter is given (SOLR-12239). + */ +public class ResortIndexConfiguredSortTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + // This config declares a SortingMergePolicy with sort = "timestamp_i_dvo desc". + initCore("solrconfig-sortingmergepolicyfactory.xml", "schema.xml"); + } + + @Test + public void testResortUsesConfiguredSortWhenNoSortParam() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + for (int i = 0; i < 5; i++) { + assertU(adoc("id", Integer.toString(i), "timestamp_i_dvo", Integer.toString(i))); + } + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + // No 'sort' param -> must fall back to the configured SortingMergePolicy sort. + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp); + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + assertEquals(coreName, resp.getValues().get("core")); + // The applied sort should be the one configured on the core (on timestamp_i_dvo), proving the + // fallback used the configured SortingMergePolicy sort rather than requiring a param. + String applied = String.valueOf(resp.getValues().get("indexSort")); + assertTrue( + "expected configured sort (on timestamp_i_dvo) in response, got: " + applied, + applied.contains("timestamp_i_dvo")); + } finally { + admin.shutdown(); + admin.close(); + } + + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='5']"); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java new file mode 100644 index 000000000000..8407320b2758 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Tests the {@link ResortCoreIndex} V2 API directly (SOLR-12239). */ +public class ResortCoreIndexTest extends SolrTestCaseJ4 { + + private CoreContainer coreContainer; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + } + + @Before + public void setup() { + coreContainer = h.getCoreContainer(); + assertU(delQ("*:*")); + assertU(commit()); + } + + @After + public void cleanup() { + assertU(delQ("*:*")); + assertU(commit()); + } + + @Test + public void testResortViaV2Api() throws Exception { + final String coreName = h.getCore().getName(); + int[] vals = {5, 1, 4, 2, 3, 0}; + for (int v : vals) { + assertU(adoc("id", Integer.toString(v), "intDvoDefault", Integer.toString(v))); + } + assertU(commit()); + + final SolrQueryRequest req = req(); + try { + ResortCoreIndex api = + new ResortCoreIndex( + coreContainer, + new CoreAdminHandler.CoreAdminAsyncTracker(), + req, + new SolrQueryResponse()); + ResortCoreIndexRequestBody body = new ResortCoreIndexRequestBody(); + body.sort = "intDvoDefault asc"; + + ResortCoreIndexResponse response = api.resortCoreIndex(coreName, body); + assertEquals(coreName, response.core); + assertNotNull(response.indexSort); + } finally { + req.close(); + } + + // Index is now physically sorted ascending by intDvoDefault. + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + var ref = h.getCore().getSearcher(); + try { + var storedFields = ref.get().getIndexReader().storedFields(); + long prev = Long.MIN_VALUE; + for (int i = 0; i < ref.get().getIndexReader().maxDoc(); i++) { + long id = Long.parseLong(storedFields.document(i).get("id")); + assertTrue("ascending after resort: " + prev + " then " + id, id >= prev); + prev = id; + } + assertEquals("smallest first", 0L, Long.parseLong(storedFields.document(0).get("id"))); + } finally { + ref.decref(); + } + } +} diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc index a7dff995ee38..9cb6dc69a6eb 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc @@ -898,6 +898,96 @@ Reusing the same `` value across restarts ensures continuity: D http://localhost:8983/solr/_collection_/update?commit=true ---- +[[coreadmin-resortindex]] +== RESORTINDEX + +The `RESORTINDEX` action re-sorts an existing core's index into a given xref:query-guide:common-query-parameters.adoc#sort-parameter[sort] order. +This makes it possible to enable xref:index-segments-merging.adoc[index sorting] on a collection that was created without it, without having to delete the index and reindex from source. + +Enabling an index sort on an existing (unsorted) index otherwise fails on core reload with an `IndexWriter` `validateIndexSort` error, because Lucene rejects the transition from unsorted segments to a configured sort. +`RESORTINDEX` produces a freshly sorted copy of the index (by merging each segment through the target sort) and swaps it in. + +This action is expensive and can take a while to complete on large indexes. +Consider running with the `async` option in such cases. + +Note: + +* Not supported in SolrCloud mode. +* Indexes containing child/nested documents are not supported (re-sorting would break the parent-child document blocks). +* The fields used in the sort must have docValues enabled. + +A typical workflow is to run `RESORTINDEX` on the existing index, then configure the same sort in `solrconfig.xml` (via a xref:index-segments-merging.adoc#mergepolicyfactory[`SortingMergePolicyFactory`]) so that subsequently added documents keep the index sorted. + +It is recommended to have a backup before running on production data. + +[tabs#coreadmin-resortindex-request] +====== +V1 API:: ++ +==== +[source,bash] +---- +http://localhost:8983/solr/admin/cores?action=RESORTINDEX&core=techproducts&sort=timestamp+desc +---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl -X POST http://localhost:8983/api/cores/techproducts/resort -H 'Content-Type: application/json' -d ' +{ + "sort": "timestamp desc" +} +' +---- +==== +====== + +=== RESORTINDEX Parameters + +`core`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the core whose index should be re-sorted. + +`sort`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The target index sort, in the usual Solr xref:query-guide:common-query-parameters.adoc#sort-parameter[sort syntax] (for example `timestamp desc`). +If omitted, the sort configured for the core (via a `SortingMergePolicy`) is used; if the core has no configured sort, the request fails. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be processed asynchronously. +Use <> with the provided `requestid` to poll for completion. + +=== RESORTINDEX Response + +On success, the response includes: + +`core`:: +The core name. + +`indexSort`:: +The index sort that was applied. + +On failure, an exception is thrown with error details. + [[coreadmin-requeststatus]] == REQUESTSTATUS diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java index 0401de380f1b..b3f035c8ac7b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java @@ -179,7 +179,8 @@ public enum CoreAdminAction { CREATESNAPSHOT, DELETESNAPSHOT, LISTSNAPSHOTS, - UPGRADEINDEX; + UPGRADEINDEX, + RESORTINDEX; public final boolean isRead;