Mercurial history per partes

opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java

@@ -492,6 +492,8 @@ public void store(History history, Repository repository, String tillRevision) t
          * hash map entry for the file) in a file. Skip renamed files
          * which will be handled separately below.
          */
+        LOGGER.log(Level.FINE, "Storing history for {0} files in repository ''{1}''",
+                new Object[]{map.entrySet().size(), repository.getDirectoryName()});
         final File root = env.getSourceRootFile();
         int fileHistoryCount = 0;
         for (Map.Entry<String, List<HistoryEntry>> map_entry : map.entrySet()) {
@@ -535,6 +537,8 @@ public void storeRenamed(Set<String> renamedFiles, Repository repository, String
 
         renamedFiles = renamedFiles.stream().filter(f -> new File(env.getSourceRootPath() + f).exists()).
                 collect(Collectors.toSet());
+        LOGGER.log(Level.FINE, "Storing history for {0} renamed files in repository ''{1}''",
+                new Object[]{renamedFiles.size(), repository.getDirectoryName()});
 
         // The directories for the renamed files have to be created before
         // the actual files otherwise storeFile() might be racing for

opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java

@@ -33,6 +33,7 @@
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.Iterator;
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -68,15 +69,16 @@ class MercurialHistoryParser implements Executor.StreamHandler {
      * specified one.
      *
      * @param file the file or directory to get history for
-     * @param changeset the changeset right before the first one to fetch, or
+     * @param sinceRevision the changeset right before the first one to fetch, or
      * {@code null} if all changesets should be fetched
+     * @param tillRevision end revision or {@code null}
      * @return history for the specified file or directory
      * @throws HistoryException if an error happens when parsing the history
      */
-    History parse(File file, String changeset) throws HistoryException {
+    History parse(File file, String sinceRevision, String tillRevision) throws HistoryException {
         isDir = file.isDirectory();
         try {
-            Executor executor = repository.getHistoryLogExecutor(file, changeset);
+            Executor executor = repository.getHistoryLogExecutor(file, sinceRevision, tillRevision, false);
             int status = executor.exec(true, this);
 
             if (status != 0) {
@@ -93,13 +95,28 @@ History parse(File file, String changeset) throws HistoryException {
         // from the list, since only the ones following it should be returned.
         // Also check that the specified changeset was found, otherwise throw
         // an exception.
-        if (changeset != null) {
-            repository.removeAndVerifyOldestChangeset(entries, changeset);
+        if (sinceRevision != null) {
+            repository.removeAndVerifyOldestChangeset(entries, sinceRevision);
+        }
+
+        // See getHistoryLogExecutor() for explanation.
+        if (repository.isHandleRenamedFiles() && file.isFile() && tillRevision != null) {
+            removeChangesets(entries, tillRevision);
         }
 
         return new History(entries, renamedFiles);
     }
 
+    private void removeChangesets(List<HistoryEntry> entries, String tillRevision) {
+        for (Iterator<HistoryEntry> iter = entries.listIterator(); iter.hasNext(); ) {
+            HistoryEntry entry = iter.next();
+            if (entry.getRevision().equals(tillRevision)) {
+                break;
+            }
+            iter.remove();
+        }
+    }
+
     /**
      * Process the output from the {@code hg log} command and collect
      * {@link HistoryEntry} elements.

opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java

@@ -0,0 +1,68 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ */
+package org.opengrok.indexer.history;
+
+import org.opengrok.indexer.util.Executor;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.function.Consumer;
+
+class MercurialHistoryParserRevisionsOnly implements Executor.StreamHandler {
+    private final MercurialRepository repository;
+    private final Consumer<String> visitor;
+
+    MercurialHistoryParserRevisionsOnly(MercurialRepository repository, Consumer<String> visitor) {
+        this.repository = repository;
+        this.visitor = visitor;
+    }
+
+    void parse(File file, String sinceRevision) throws HistoryException {
+        try {
+            Executor executor = repository.getHistoryLogExecutor(file, sinceRevision, null, true);
+            int status = executor.exec(true, this);
+
+            if (status != 0) {
+                throw new HistoryException(
+                        String.format("Failed to get revisions for: \"%s\" since revision %s Exit code: %d",
+                                file.getAbsolutePath(), sinceRevision, status));
+            }
+        } catch (IOException e) {
+            throw new HistoryException("Failed to get history for: \"" +
+                    file.getAbsolutePath() + "\"", e);
+        }
+    }
+
+    @Override
+    public void processStream(InputStream input) throws IOException {
+        try (BufferedReader in = new BufferedReader(new InputStreamReader(input))) {
+            String s;
+            while ((s = in.readLine()) != null) {
+                visitor.accept(s);
+            }
+        }
+    }
+}

opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java

@@ -33,6 +33,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.TreeSet;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -50,12 +51,14 @@
  * Access to a Mercurial repository.
  *
  */
-public class MercurialRepository extends Repository {
+public class MercurialRepository extends RepositoryWithPerPartesHistory {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(MercurialRepository.class);
 
     private static final long serialVersionUID = 1L;
 
+    public static final int MAX_CHANGESETS = 256;
+
     /**
      * The property name used to obtain the client command for this repository.
      */
@@ -81,8 +84,9 @@ public class MercurialRepository extends Repository {
     static final String END_OF_ENTRY
             = "mercurial_history_end_of_entry";
 
+    private static final String TEMPLATE_REVS = "{rev}:{node|short}\\n";
     private static final String TEMPLATE_STUB
-            = CHANGESET + "{rev}:{node|short}\\n"
+            = CHANGESET + TEMPLATE_REVS
             + USER + "{author}\\n" + DATE + "{date|isodate}\\n"
             + DESCRIPTION + "{desc|strip|obfuscate}\\n";
 
@@ -143,6 +147,19 @@ String determineBranch(CommandTimeoutType cmdType) throws IOException {
         return executor.getOutputString().trim();
     }
 
+    public int getPerPartesCount() {
+        return MAX_CHANGESETS;
+    }
+
+    private String getRevisionNum(String changeset) throws HistoryException {
+        String[] parts = changeset.split(":");
+        if (parts.length == 2) {
+            return parts[0];
+        } else {
+            throw new HistoryException("Don't know how to parse changeset identifier: " + changeset);
+        }
+    }
+
     /**
      * Get an executor to be used for retrieving the history log for the named
      * file or directory.
@@ -151,31 +168,43 @@ String determineBranch(CommandTimeoutType cmdType) throws IOException {
      * @param sinceRevision the oldest changeset to return from the executor, or
      *                  {@code null} if all changesets should be returned.
      *                  For files this does not apply and full history is returned.
+     * @param tillRevision end revision
+     * @param revisionsOnly get only revision numbers
      * @return An Executor ready to be started
      */
-    Executor getHistoryLogExecutor(File file, String sinceRevision)
+    Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevision, boolean revisionsOnly)
             throws HistoryException, IOException {
+
         String filename = getRepoRelativePath(file);
-        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
 
         List<String> cmd = new ArrayList<>();
         ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK);
         cmd.add(RepoCommand);
         cmd.add("log");
 
         if (file.isDirectory()) {
-            // If this is non-default branch we would like to get the changesets
-            // on that branch and also follow any changesets from the parent branch.
-            if (sinceRevision != null) {
+            // Note: assumes one of them is not null
+            if ((sinceRevision != null) || (tillRevision != null)) {
                 cmd.add("-r");
-                String[] parts = sinceRevision.split(":");
-                if (parts.length == 2) {
-                    cmd.add("reverse(" + parts[0] + "::'" + getBranch() + "')");
+                StringBuilder stringBuilder = new StringBuilder();
+                if (!revisionsOnly) {
+                    stringBuilder.append("reverse(");
+                }
+                if (sinceRevision != null) {
+                    stringBuilder.append(getRevisionNum(sinceRevision));
+                }
+                stringBuilder.append("::");
+                if (tillRevision != null) {
+                    stringBuilder.append(getRevisionNum(tillRevision));
                 } else {
-                    throw new HistoryException(
-                            "Don't know how to parse changeset identifier: "
-                            + sinceRevision);
+                    // If this is non-default branch we would like to get the changesets
+                    // on that branch and also follow any changesets from the parent branch.
+                    stringBuilder.append("'").append(getBranch()).append("'");
+                }
+                if (!revisionsOnly) {
+                    stringBuilder.append(")");
                 }
+                cmd.add(stringBuilder.toString());
             } else {
                 cmd.add("-r");
                 cmd.add("reverse(0::'" + getBranch() + "')");
@@ -191,16 +220,27 @@ Executor getHistoryLogExecutor(File file, String sinceRevision)
             // For files this does not matter since if getHistory() is called
             // for a file, the file has to be renamed so we want its complete history
             // if renamed file handling is enabled for this repository.
+            //
+            // Getting history for individual files should only be done when generating history for renamed files
+            // so the fact that filtering on sinceRevision does not work does not matter there as history
+            // from the initial changeset is needed. The tillRevision filtering works however not
+            // in combination with --follow so the filtering is done in MercurialHistoryParser.parse().
+            // Even if the revision filtering worked, this approach would be probably faster and consumed less memory.
             if (this.isHandleRenamedFiles()) {
+                // When using --follow, the returned revisions are from newest to oldest, hence no reverse() is needed.
                 cmd.add("--follow");
             }
         }
 
         cmd.add("--template");
-        if (file.isDirectory()) {
-            cmd.add(this.isHandleRenamedFiles() ? DIR_TEMPLATE_RENAMED : DIR_TEMPLATE);
+        if (revisionsOnly) {
+            cmd.add(TEMPLATE_REVS);
         } else {
-            cmd.add(FILE_TEMPLATE);
+            if (file.isDirectory()) {
+                cmd.add(this.isHandleRenamedFiles() ? DIR_TEMPLATE_RENAMED : DIR_TEMPLATE);
+            } else {
+                cmd.add(FILE_TEMPLATE);
+            }
         }
 
         if (!filename.isEmpty()) {
@@ -259,7 +299,7 @@ private HistoryRevResult getHistoryRev(
      * @param fullpath file path
      * @param full_rev_to_find revision number (in the form of
      * {rev}:{node|short})
-     * @returns original filename
+     * @return original filename
      */
     private String findOriginalName(String fullpath, String full_rev_to_find)
             throws IOException {
@@ -511,9 +551,17 @@ History getHistory(File file) throws HistoryException {
         return getHistory(file, null);
     }
 
+    public void accept(String sinceRevision, Consumer<String> visitor) throws HistoryException {
+        new MercurialHistoryParserRevisionsOnly(this, visitor).
+                parse(new File(getDirectoryName()), sinceRevision);
+    }
+
+    History getHistory(File file, String sinceRevision) throws HistoryException {
+        return getHistory(file, sinceRevision, null);
+    }
+
     @Override
-    History getHistory(File file, String sinceRevision)
-            throws HistoryException {
+    History getHistory(File file, String sinceRevision, String tillRevision) throws HistoryException {
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
         // Note that the filtering of revisions based on sinceRevision is done
         // in the history log executor by passing appropriate options to
@@ -522,9 +570,8 @@ History getHistory(File file, String sinceRevision)
         // for file, the file is renamed and its complete history is fetched
         // so no sinceRevision filter is needed.
         // See findOriginalName() code for more details.
-        History result = new MercurialHistoryParser(this).parse(file,
-                sinceRevision);
-
+        History result = new MercurialHistoryParser(this).parse(file, sinceRevision, tillRevision);
+
         // Assign tags to changesets they represent.
         // We don't need to check if this repository supports tags,
         // because we know it :-)

opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java

@@ -186,8 +186,7 @@ History getHistory(File file, String sinceRevision) throws HistoryException {
     void removeAndVerifyOldestChangeset(List<HistoryEntry> entries,
             String revision)
             throws HistoryException {
-        HistoryEntry entry
-                = entries.isEmpty() ? null : entries.remove(entries.size() - 1);
+        HistoryEntry entry = entries.isEmpty() ? null : entries.remove(entries.size() - 1);
 
         // TODO We should check more thoroughly that the changeset is the one
         // we expected it to be, since some SCMs may change the revision