From 7611aab689851a24cf49af985ab669bb19fde0fc Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 13:51:01 +0200 Subject: [PATCH 01/22] initial changes for Mercurial per partes history --- .../history/MercurialHistoryParser.java | 10 +-- .../MercurialHistoryParserRevisionsOnly.java | 67 +++++++++++++++++++ .../indexer/history/MercurialRepository.java | 42 ++++++++---- 3 files changed, 100 insertions(+), 19 deletions(-) create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java index 94549671073..a2dcdcb96a3 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java @@ -68,15 +68,15 @@ class MercurialHistoryParser implements Executor.StreamHandler { * specified one. * * @param file the file or directory to get history for - * @param changeset the changeset right before the first one to fetch, or + * @param sinceRevision the changeset right before the first one to fetch, or * {@code null} if all changesets should be fetched * @return history for the specified file or directory * @throws HistoryException if an error happens when parsing the history */ - History parse(File file, String changeset) throws HistoryException { + History parse(File file, String sinceRevision, String tillRevision) throws HistoryException { isDir = file.isDirectory(); try { - Executor executor = repository.getHistoryLogExecutor(file, changeset); + Executor executor = repository.getHistoryLogExecutor(file, sinceRevision, tillRevision,false); int status = executor.exec(true, this); if (status != 0) { @@ -93,8 +93,8 @@ History parse(File file, String changeset) throws HistoryException { // from the list, since only the ones following it should be returned. // Also check that the specified changeset was found, otherwise throw // an exception. - if (changeset != null) { - repository.removeAndVerifyOldestChangeset(entries, changeset); + if (sinceRevision != null) { + repository.removeAndVerifyOldestChangeset(entries, sinceRevision); } return new History(entries, renamedFiles); diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java new file mode 100644 index 00000000000..5cec7316fae --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java @@ -0,0 +1,67 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + */ +package org.opengrok.indexer.history; + +import org.opengrok.indexer.util.Executor; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.function.Consumer; + +class MercurialHistoryParserRevisionsOnly implements Executor.StreamHandler { + private final MercurialRepository repository; + private final Consumer visitor; + + MercurialHistoryParserRevisionsOnly(MercurialRepository repository, Consumer visitor) { + this.repository = repository; + this.visitor = visitor; + } + + void parse(File file, String sinceRevision) throws HistoryException { + try { + Executor executor = repository.getHistoryLogExecutor(file, sinceRevision, null, true); + int status = executor.exec(true, this); + + if (status != 0) { + throw new HistoryException("Failed to get revisions for: \"" + + file.getAbsolutePath() + + "\" Exit code: " + status); // TODO log sinceRevision + } + } catch (IOException e) { + throw new HistoryException("Failed to get history for: \"" + + file.getAbsolutePath() + "\"", e); + } + } + + @Override + public void processStream(InputStream input) throws IOException { + BufferedReader in = new BufferedReader(new InputStreamReader(input)); + String s; + while ((s = in.readLine()) != null) { + visitor.accept(s); + } + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index 1060433f152..e0658701e4f 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -33,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.TreeSet; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -50,7 +51,7 @@ * Access to a Mercurial repository. * */ -public class MercurialRepository extends Repository { +public class MercurialRepository extends RepositoryWithPerPartesHistory { private static final Logger LOGGER = LoggerFactory.getLogger(MercurialRepository.class); @@ -81,6 +82,7 @@ public class MercurialRepository extends Repository { static final String END_OF_ENTRY = "mercurial_history_end_of_entry"; + private static final String TEMPLATE_REVS = "{rev}\\n"; private static final String TEMPLATE_STUB = CHANGESET + "{rev}:{node|short}\\n" + USER + "{author}\\n" + DATE + "{date|isodate}\\n" @@ -151,12 +153,14 @@ String determineBranch(CommandTimeoutType cmdType) throws IOException { * @param sinceRevision the oldest changeset to return from the executor, or * {@code null} if all changesets should be returned. * For files this does not apply and full history is returned. + * @param tillRevision end revision + * @param revisionsOnly get only revision numbers * @return An Executor ready to be started */ - Executor getHistoryLogExecutor(File file, String sinceRevision) + Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevision, boolean revisionsOnly) throws HistoryException, IOException { + String filename = getRepoRelativePath(file); - RuntimeEnvironment env = RuntimeEnvironment.getInstance(); List cmd = new ArrayList<>(); ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK); @@ -166,15 +170,14 @@ Executor getHistoryLogExecutor(File file, String sinceRevision) if (file.isDirectory()) { // If this is non-default branch we would like to get the changesets // on that branch and also follow any changesets from the parent branch. + // TODO tillRevision if (sinceRevision != null) { cmd.add("-r"); String[] parts = sinceRevision.split(":"); if (parts.length == 2) { cmd.add("reverse(" + parts[0] + "::'" + getBranch() + "')"); } else { - throw new HistoryException( - "Don't know how to parse changeset identifier: " - + sinceRevision); + throw new HistoryException("Don't know how to parse changeset identifier: " + sinceRevision); } } else { cmd.add("-r"); @@ -197,10 +200,14 @@ Executor getHistoryLogExecutor(File file, String sinceRevision) } cmd.add("--template"); - if (file.isDirectory()) { - cmd.add(this.isHandleRenamedFiles() ? DIR_TEMPLATE_RENAMED : DIR_TEMPLATE); + if (revisionsOnly) { + cmd.add(TEMPLATE_REVS); } else { - cmd.add(FILE_TEMPLATE); + if (file.isDirectory()) { + cmd.add(this.isHandleRenamedFiles() ? DIR_TEMPLATE_RENAMED : DIR_TEMPLATE); + } else { + cmd.add(FILE_TEMPLATE); + } } if (!filename.isEmpty()) { @@ -259,7 +266,7 @@ private HistoryRevResult getHistoryRev( * @param fullpath file path * @param full_rev_to_find revision number (in the form of * {rev}:{node|short}) - * @returns original filename + * @return original filename */ private String findOriginalName(String fullpath, String full_rev_to_find) throws IOException { @@ -511,9 +518,17 @@ History getHistory(File file) throws HistoryException { return getHistory(file, null); } + public void accept(String sinceRevision, Consumer visitor) throws HistoryException { + new MercurialHistoryParserRevisionsOnly(this, visitor). + parse(new File(getDirectoryName()), sinceRevision); + } + + History getHistory(File file, String sinceRevision) throws HistoryException { + return getHistory(file, sinceRevision, null); + } + @Override - History getHistory(File file, String sinceRevision) - throws HistoryException { + History getHistory(File file, String sinceRevision, String tillRevision) throws HistoryException { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); // Note that the filtering of revisions based on sinceRevision is done // in the history log executor by passing appropriate options to @@ -522,8 +537,7 @@ History getHistory(File file, String sinceRevision) // for file, the file is renamed and its complete history is fetched // so no sinceRevision filter is needed. // See findOriginalName() code for more details. - History result = new MercurialHistoryParser(this).parse(file, - sinceRevision); + History result = new MercurialHistoryParser(this).parse(file, sinceRevision, tillRevision); // Assign tags to changesets they represent. // We don't need to check if this repository supports tags, From 931304cd7d7444e1137e1f69f79d4a425885f8a5 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 14:12:58 +0200 Subject: [PATCH 02/22] fix style --- .../org/opengrok/indexer/history/MercurialHistoryParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java index a2dcdcb96a3..abacd726782 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java @@ -76,7 +76,7 @@ class MercurialHistoryParser implements Executor.StreamHandler { History parse(File file, String sinceRevision, String tillRevision) throws HistoryException { isDir = file.isDirectory(); try { - Executor executor = repository.getHistoryLogExecutor(file, sinceRevision, tillRevision,false); + Executor executor = repository.getHistoryLogExecutor(file, sinceRevision, tillRevision, false); int status = executor.exec(true, this); if (status != 0) { From d139769b959abbdebf61b4d7ce8bd933e4059f64 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 14:13:03 +0200 Subject: [PATCH 03/22] add tillRevision constraint --- .../indexer/history/MercurialRepository.java | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index e0658701e4f..7c309bf1eca 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -145,6 +145,15 @@ String determineBranch(CommandTimeoutType cmdType) throws IOException { return executor.getOutputString().trim(); } + private String getRevisionNum(String changeset) throws HistoryException { + String[] parts = changeset.split(":"); + if (parts.length == 2) { + return parts[0]; + } else { + throw new HistoryException("Don't know how to parse changeset identifier: " + changeset); + } + } + /** * Get an executor to be used for retrieving the history log for the named * file or directory. @@ -168,17 +177,27 @@ Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevis cmd.add("log"); if (file.isDirectory()) { - // If this is non-default branch we would like to get the changesets - // on that branch and also follow any changesets from the parent branch. - // TODO tillRevision - if (sinceRevision != null) { - cmd.add("-r"); - String[] parts = sinceRevision.split(":"); - if (parts.length == 2) { - cmd.add("reverse(" + parts[0] + "::'" + getBranch() + "')"); + if ((sinceRevision != null) || (tillRevision != null)) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("-r"); + if (!revisionsOnly) { + stringBuilder.append("reverse("); + } + if (sinceRevision != null) { + stringBuilder.append(getRevisionNum(sinceRevision)); + } + stringBuilder.append("::"); + if (tillRevision != null) { + stringBuilder.append(getRevisionNum(tillRevision)); } else { - throw new HistoryException("Don't know how to parse changeset identifier: " + sinceRevision); + // If this is non-default branch we would like to get the changesets + // on that branch and also follow any changesets from the parent branch. + stringBuilder.append("'" + getBranch() + "'"); + } + if (!revisionsOnly) { + stringBuilder.append(")"); } + cmd.add(stringBuilder.toString()); } else { cmd.add("-r"); cmd.add("reverse(0::'" + getBranch() + "')"); From 6e9f380a26cd766f7d2d6b4080683e907f51af9b Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 14:21:27 +0200 Subject: [PATCH 04/22] log sinceRevision on error --- .../history/MercurialHistoryParserRevisionsOnly.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java index 5cec7316fae..2c1e6b0b46d 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.function.Consumer; class MercurialHistoryParserRevisionsOnly implements Executor.StreamHandler { @@ -46,9 +47,9 @@ void parse(File file, String sinceRevision) throws HistoryException { int status = executor.exec(true, this); if (status != 0) { - throw new HistoryException("Failed to get revisions for: \"" + - file.getAbsolutePath() + - "\" Exit code: " + status); // TODO log sinceRevision + throw new HistoryException( + String.format("Failed to get revisions for: \"%s\" since revision %s Exit code: %d", + file.getAbsolutePath(), sinceRevision, status)); } } catch (IOException e) { throw new HistoryException("Failed to get history for: \"" + From fc072265e60f5a12cadc4744a0c2b5c2484f2afb Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 14:21:51 +0200 Subject: [PATCH 05/22] try with resources --- .../history/MercurialHistoryParserRevisionsOnly.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java index 2c1e6b0b46d..1f0758d9a6b 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java @@ -59,10 +59,11 @@ void parse(File file, String sinceRevision) throws HistoryException { @Override public void processStream(InputStream input) throws IOException { - BufferedReader in = new BufferedReader(new InputStreamReader(input)); - String s; - while ((s = in.readLine()) != null) { - visitor.accept(s); + try (BufferedReader in = new BufferedReader(new InputStreamReader(input))) { + String s; + while ((s = in.readLine()) != null) { + visitor.accept(s); + } } } } From 8d086e530d51fb9d54fa4429d6b18683286e5382 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 14:22:11 +0200 Subject: [PATCH 06/22] remove unused import --- .../indexer/history/MercurialHistoryParserRevisionsOnly.java | 1 - 1 file changed, 1 deletion(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java index 1f0758d9a6b..f2bd2390c36 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.function.Consumer; class MercurialHistoryParserRevisionsOnly implements Executor.StreamHandler { From 8f1fb92d22331fe620c22944114e82224f6f0bce Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 14:35:28 +0200 Subject: [PATCH 07/22] chained append() --- .../java/org/opengrok/indexer/history/MercurialRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index 7c309bf1eca..b9a7679b363 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -192,7 +192,7 @@ Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevis } else { // If this is non-default branch we would like to get the changesets // on that branch and also follow any changesets from the parent branch. - stringBuilder.append("'" + getBranch() + "'"); + stringBuilder.append("'").append(getBranch()).append("'"); } if (!revisionsOnly) { stringBuilder.append(")"); From f7de4e34e627e41d7c2de7c0b9f2a2f05243d294 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 14:38:17 +0200 Subject: [PATCH 08/22] use colon --- .../java/org/opengrok/indexer/history/MercurialRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index b9a7679b363..e6796a62f4c 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -82,7 +82,7 @@ public class MercurialRepository extends RepositoryWithPerPartesHistory { static final String END_OF_ENTRY = "mercurial_history_end_of_entry"; - private static final String TEMPLATE_REVS = "{rev}\\n"; + private static final String TEMPLATE_REVS = "{rev}:\\n"; private static final String TEMPLATE_STUB = CHANGESET + "{rev}:{node|short}\\n" + USER + "{author}\\n" + DATE + "{date|isodate}\\n" From ea291dd28c070dbdc426ea4f2b65b9549a9d87a0 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 14:39:17 +0200 Subject: [PATCH 09/22] add comment --- .../java/org/opengrok/indexer/history/MercurialRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index e6796a62f4c..05585070b72 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -82,7 +82,7 @@ public class MercurialRepository extends RepositoryWithPerPartesHistory { static final String END_OF_ENTRY = "mercurial_history_end_of_entry"; - private static final String TEMPLATE_REVS = "{rev}:\\n"; + private static final String TEMPLATE_REVS = "{rev}:\\n"; // use colon so that getRevisionNum() works private static final String TEMPLATE_STUB = CHANGESET + "{rev}:{node|short}\\n" + USER + "{author}\\n" + DATE + "{date|isodate}\\n" From 7d6c1ed61432baed4268a2f09f603eb9e6fb5bac Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 15:57:55 +0200 Subject: [PATCH 10/22] add long entry to see how many files are to be processed --- .../java/org/opengrok/indexer/history/FileHistoryCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java index a8a80356692..795286c1eeb 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java @@ -492,6 +492,8 @@ public void store(History history, Repository repository, String tillRevision) t * hash map entry for the file) in a file. Skip renamed files * which will be handled separately below. */ + LOGGER.log(Level.FINE, "Storing history for {0} files in repository ''{1}''", + new Object[]{map.entrySet().size(), repository.getDirectoryName()}); final File root = env.getSourceRootFile(); int fileHistoryCount = 0; for (Map.Entry> map_entry : map.entrySet()) { From 3f5255e394a80112649977d4eb9333f8c449025c Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 15:58:16 +0200 Subject: [PATCH 11/22] add revision filtering for files --- .../indexer/history/MercurialRepository.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index 05585070b72..5f286a78f38 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -82,9 +82,9 @@ public class MercurialRepository extends RepositoryWithPerPartesHistory { static final String END_OF_ENTRY = "mercurial_history_end_of_entry"; - private static final String TEMPLATE_REVS = "{rev}:\\n"; // use colon so that getRevisionNum() works + private static final String TEMPLATE_REVS = "{rev}:{node|short}\\n"; private static final String TEMPLATE_STUB - = CHANGESET + "{rev}:{node|short}\\n" + = CHANGESET + TEMPLATE_REVS + USER + "{author}\\n" + DATE + "{date|isodate}\\n" + DESCRIPTION + "{desc|strip|obfuscate}\\n"; @@ -177,9 +177,10 @@ Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevis cmd.add("log"); if (file.isDirectory()) { + // Note: assumes one of them is not null if ((sinceRevision != null) || (tillRevision != null)) { + cmd.add("-r"); StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("-r"); if (!revisionsOnly) { stringBuilder.append("reverse("); } @@ -208,14 +209,30 @@ Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevis // when handling renamed files) // It is not needed to filter on a branch as 'hg log' will follow // the active branch. + // TODO: revisit: // Due to behavior of recent Mercurial versions, it is not possible // to filter the changesets of a file based on revision. // For files this does not matter since if getHistory() is called // for a file, the file has to be renamed so we want its complete history // if renamed file handling is enabled for this repository. + // TODO: why no reverse() ? if (this.isHandleRenamedFiles()) { cmd.add("--follow"); } + + // Note: assumes one of them is not null + if ((sinceRevision != null) || (tillRevision != null)) { + cmd.add("-r"); + StringBuilder stringBuilder = new StringBuilder(); + if (sinceRevision != null) { + stringBuilder.append(getRevisionNum(sinceRevision)); + } + stringBuilder.append(":"); + if (tillRevision != null) { + stringBuilder.append(getRevisionNum(tillRevision)); + } + cmd.add(stringBuilder.toString()); + } } cmd.add("--template"); From 0855143da12d0301d5ec020a3f868a100f6fabef Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 16:12:52 +0200 Subject: [PATCH 12/22] simplify the tests by using BeforeEach --- .../history/MercurialRepositoryTest.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java index 59f881cb85f..fac546718f2 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java @@ -24,6 +24,7 @@ package org.opengrok.indexer.history; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.opengrok.indexer.condition.EnabledForRepository; import org.opengrok.indexer.util.Executor; @@ -85,6 +86,11 @@ private void setUpTestRepository() throws IOException { repository.create(getClass().getResourceAsStream("repositories.zip")); } + @BeforeEach + public void setup() throws IOException { + setUpTestRepository(); + } + @AfterEach public void tearDown() { if (repository != null) { @@ -95,7 +101,6 @@ public void tearDown() { @Test public void testGetHistory() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); @@ -114,7 +119,6 @@ public void testGetHistory() throws Exception { @Test public void testGetHistorySubdir() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); // Add a subdirectory with some history. @@ -135,10 +139,8 @@ public void testGetHistorySubdir() throws Exception { */ @Test public void testGetHistoryPartial() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); // Get all but the oldest revision. History hist = mr.getHistory(root, REVISIONS[REVISIONS.length - 1]); List entries = hist.getHistoryEntries(); @@ -180,7 +182,6 @@ public static void runHgCommand(File reposRoot, String... args) { */ @Test public void testGetHistoryBranch() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); // Branch the repo and add one changeset. @@ -228,7 +229,6 @@ public void testGetHistoryBranch() throws Exception { */ @Test public void testGetHistoryGet() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); @@ -260,7 +260,6 @@ public void testGetHistoryGet() throws Exception { */ @Test public void testgetHistoryGetForAll() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); @@ -279,7 +278,6 @@ public void testgetHistoryGetForAll() throws Exception { */ @Test public void testGetHistoryGetRenamed() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); @@ -306,7 +304,6 @@ public void testGetHistoryGetRenamed() throws Exception { */ @Test public void testGetHistoryWithNoSuchRevision() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); From dbb2c9bdb9220f0e594afea97a127312c3688642 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 16:14:09 +0200 Subject: [PATCH 13/22] join lines for better readability --- .../indexer/history/MercurialRepositoryTest.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java index fac546718f2..5d959b1bd1e 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java @@ -102,8 +102,7 @@ public void tearDown() { @Test public void testGetHistory() throws Exception { File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); History hist = mr.getHistory(root); List entries = hist.getHistoryEntries(); assertEquals(REVISIONS.length, entries.size()); @@ -230,8 +229,7 @@ public void testGetHistoryBranch() throws Exception { @Test public void testGetHistoryGet() throws Exception { File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); String exp_str = "This will be a first novel of mine.\n" + "\n" + "Chapter 1.\n" @@ -261,8 +259,7 @@ public void testGetHistoryGet() throws Exception { @Test public void testgetHistoryGetForAll() throws Exception { File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); for (String rev : REVISIONS_novel) { InputStream input = mr.getHistoryGet(root.getCanonicalPath(), @@ -279,8 +276,7 @@ public void testgetHistoryGetForAll() throws Exception { @Test public void testGetHistoryGetRenamed() throws Exception { File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); String exp_str = "This is totally plaintext file.\n"; byte[] buffer = new byte[1024]; From e679821b76adefafc1cc68e274d2f9fb3ebec81b Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 16:35:58 +0200 Subject: [PATCH 14/22] add tests for getHistory() --- .../history/MercurialRepositoryTest.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java index 5d959b1bd1e..93b9b86fd22 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java @@ -38,6 +38,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -313,4 +314,57 @@ public void testGetHistoryWithNoSuchRevision() throws Exception { String constructedRevision = (number + 1) + ":" + hash; assertThrows(HistoryException.class, () -> mr.getHistory(root, constructedRevision)); } + + // TODO: also for (renamed) file + @Test + void testGetHistorySinceTillNullNull() throws Exception { + File root = new File(repository.getSourceRoot(), "mercurial"); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(root, null, null); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(10, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(REVISIONS), revisions); + } + + @Test + void testGetHistorySinceTillNullRev() throws Exception { + File root = new File(repository.getSourceRoot(), "mercurial"); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(root, null, REVISIONS[4]); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(6, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 4, REVISIONS.length)), revisions); + } + + @Test + void testGetHistorySinceTillRevNull() throws Exception { + File root = new File(repository.getSourceRoot(), "mercurial"); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(root, REVISIONS[3], null); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(3, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 0, 3)), revisions); + } + + @Test + void testGetHistorySinceTillRevRev() throws Exception { + File root = new File(repository.getSourceRoot(), "mercurial"); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(root, REVISIONS[7], REVISIONS[2]); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(5, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 2, 7)), revisions); + } } From 6996d82d682632dfafd22a388b331fdcbd1bdef1 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 17:08:09 +0200 Subject: [PATCH 15/22] join line --- .../org/opengrok/indexer/history/MercurialRepositoryTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java index 93b9b86fd22..d5bc3f55721 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java @@ -125,8 +125,7 @@ public void testGetHistorySubdir() throws Exception { runHgCommand(root, "import", Paths.get(getClass().getResource("/history/hg-export-subdir.txt").toURI()).toString()); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); History hist = mr.getHistory(new File(root, "subdir")); List entries = hist.getHistoryEntries(); assertEquals(1, entries.size()); From 63cfbcf44f653269b075a01c3762251e8e6206e1 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 17:08:15 +0200 Subject: [PATCH 16/22] add comments --- .../org/opengrok/indexer/history/MercurialRepository.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index 5f286a78f38..7cb5609ac01 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -209,14 +209,18 @@ Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevis // when handling renamed files) // It is not needed to filter on a branch as 'hg log' will follow // the active branch. - // TODO: revisit: // Due to behavior of recent Mercurial versions, it is not possible // to filter the changesets of a file based on revision. // For files this does not matter since if getHistory() is called // for a file, the file has to be renamed so we want its complete history // if renamed file handling is enabled for this repository. - // TODO: why no reverse() ? + // + // Getting history for individual files should only be done when generating history for renamed files + // so the fact that filtering on sinceRevision does not work does not matter there as history + // from the initial changeset is needed. The tillRevision filtering works. + if (this.isHandleRenamedFiles()) { + // When using --follow, the returned revisions are from newest to oldest, hence no reverse() is needed. cmd.add("--follow"); } From bb6f64c51c93d2ae74dfe3c28973700b188aff60 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 17:46:17 +0200 Subject: [PATCH 17/22] override max chunks (for debugging) --- .../org/opengrok/indexer/history/MercurialRepository.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index 7cb5609ac01..96c2324d205 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -57,6 +57,8 @@ public class MercurialRepository extends RepositoryWithPerPartesHistory { private static final long serialVersionUID = 1L; + public static final int MAX_CHANGESETS = 256; + /** * The property name used to obtain the client command for this repository. */ @@ -145,6 +147,10 @@ String determineBranch(CommandTimeoutType cmdType) throws IOException { return executor.getOutputString().trim(); } + public int getPerPartesCount() { + return MAX_CHANGESETS; + } + private String getRevisionNum(String changeset) throws HistoryException { String[] parts = changeset.split(":"); if (parts.length == 2) { From 5723326d2f3401183d761f8225e30c8046a0dc77 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 21:34:45 +0200 Subject: [PATCH 18/22] join the lines for readability --- .../src/main/java/org/opengrok/indexer/history/Repository.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java index 2fcbd5cd2f7..2c4dc0616c0 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java @@ -186,8 +186,7 @@ History getHistory(File file, String sinceRevision) throws HistoryException { void removeAndVerifyOldestChangeset(List entries, String revision) throws HistoryException { - HistoryEntry entry - = entries.isEmpty() ? null : entries.remove(entries.size() - 1); + HistoryEntry entry = entries.isEmpty() ? null : entries.remove(entries.size() - 1); // TODO We should check more thoroughly that the changeset is the one // we expected it to be, since some SCMs may change the revision From d0a0418224404aac31a813b3f8751ffba3eff20b Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 21:51:44 +0200 Subject: [PATCH 19/22] add test for renamed file --- .../history/MercurialRepositoryTest.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java index d5bc3f55721..7813c1d80d9 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.opengrok.indexer.condition.EnabledForRepository; +import org.opengrok.indexer.configuration.RuntimeEnvironment; import org.opengrok.indexer.util.Executor; import org.opengrok.indexer.util.TestRepository; @@ -44,6 +45,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opengrok.indexer.condition.RepositoryInstalled.Type.MERCURIAL; /** @@ -314,7 +316,6 @@ public void testGetHistoryWithNoSuchRevision() throws Exception { assertThrows(HistoryException.class, () -> mr.getHistory(root, constructedRevision)); } - // TODO: also for (renamed) file @Test void testGetHistorySinceTillNullNull() throws Exception { File root = new File(repository.getSourceRoot(), "mercurial"); @@ -366,4 +367,20 @@ void testGetHistorySinceTillRevRev() throws Exception { collect(Collectors.toList()); assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 2, 7)), revisions); } + + @Test + void testGetHistoryRenamedFileTillRev() throws Exception { + RuntimeEnvironment.getInstance().setHandleHistoryOfRenamedFiles(true); + File root = new File(repository.getSourceRoot(), "mercurial"); + File file = new File(root, "novel.txt"); + assertTrue(file.exists() && file.isFile()); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(file, null, "7:db1394c05268"); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(5, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 2, 7)), revisions); + } } From d6e7eb8c1ed42d198f51442a271aeafdf4591391 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 21:51:59 +0200 Subject: [PATCH 20/22] filter extra history entries by hand --- .../history/MercurialHistoryParser.java | 17 ++++++++++++++++ .../indexer/history/MercurialRepository.java | 20 +++---------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java index abacd726782..fa8bbdd767e 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java @@ -33,6 +33,7 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Date; +import java.util.Iterator; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -70,6 +71,7 @@ class MercurialHistoryParser implements Executor.StreamHandler { * @param file the file or directory to get history for * @param sinceRevision the changeset right before the first one to fetch, or * {@code null} if all changesets should be fetched + * @param tillRevision end revision or {@code null} * @return history for the specified file or directory * @throws HistoryException if an error happens when parsing the history */ @@ -97,9 +99,24 @@ History parse(File file, String sinceRevision, String tillRevision) throws Histo repository.removeAndVerifyOldestChangeset(entries, sinceRevision); } + // TODO: add comment + if (repository.isHandleRenamedFiles() && file.isFile() && tillRevision != null) { + removeChangesets(entries, tillRevision); + } + return new History(entries, renamedFiles); } + private void removeChangesets(List entries, String tillRevision) { + for (Iterator iter = entries.listIterator(); iter.hasNext(); ) { + HistoryEntry entry = iter.next(); + if (entry.getRevision().equals(tillRevision)) { + break; + } + iter.remove(); + } + } + /** * Process the output from the {@code hg log} command and collect * {@link HistoryEntry} elements. diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index 96c2324d205..e36dac30d54 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -223,26 +223,12 @@ Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevis // // Getting history for individual files should only be done when generating history for renamed files // so the fact that filtering on sinceRevision does not work does not matter there as history - // from the initial changeset is needed. The tillRevision filtering works. - + // from the initial changeset is needed. The tillRevision filtering works however not + // in combination with --follow. if (this.isHandleRenamedFiles()) { // When using --follow, the returned revisions are from newest to oldest, hence no reverse() is needed. cmd.add("--follow"); } - - // Note: assumes one of them is not null - if ((sinceRevision != null) || (tillRevision != null)) { - cmd.add("-r"); - StringBuilder stringBuilder = new StringBuilder(); - if (sinceRevision != null) { - stringBuilder.append(getRevisionNum(sinceRevision)); - } - stringBuilder.append(":"); - if (tillRevision != null) { - stringBuilder.append(getRevisionNum(tillRevision)); - } - cmd.add(stringBuilder.toString()); - } } cmd.add("--template"); @@ -584,7 +570,7 @@ History getHistory(File file, String sinceRevision, String tillRevision) throws // so no sinceRevision filter is needed. // See findOriginalName() code for more details. History result = new MercurialHistoryParser(this).parse(file, sinceRevision, tillRevision); - + // Assign tags to changesets they represent. // We don't need to check if this repository supports tags, // because we know it :-) From 6a4bb35cc63a9fccde414746156a8b2cd2d80ef9 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 22:03:53 +0200 Subject: [PATCH 21/22] add log before store renamed files' history --- .../java/org/opengrok/indexer/history/FileHistoryCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java index 795286c1eeb..ca4c93f551c 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java @@ -537,6 +537,8 @@ public void storeRenamed(Set renamedFiles, Repository repository, String renamedFiles = renamedFiles.stream().filter(f -> new File(env.getSourceRootPath() + f).exists()). collect(Collectors.toSet()); + LOGGER.log(Level.FINE, "Storing history for {0} renamed files in repository ''{1}''", + new Object[]{renamedFiles.size(), repository.getDirectoryName()}); // The directories for the renamed files have to be created before // the actual files otherwise storeFile() might be racing for From 5cbbe2eff39975f997fcb3dd259ec118ac3b4523 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 21 May 2021 22:07:05 +0200 Subject: [PATCH 22/22] add comments --- .../org/opengrok/indexer/history/MercurialHistoryParser.java | 2 +- .../java/org/opengrok/indexer/history/MercurialRepository.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java index fa8bbdd767e..cd0cfe8a84a 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java @@ -99,7 +99,7 @@ History parse(File file, String sinceRevision, String tillRevision) throws Histo repository.removeAndVerifyOldestChangeset(entries, sinceRevision); } - // TODO: add comment + // See getHistoryLogExecutor() for explanation. if (repository.isHandleRenamedFiles() && file.isFile() && tillRevision != null) { removeChangesets(entries, tillRevision); } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index e36dac30d54..eb61314d29a 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -224,7 +224,8 @@ Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevis // Getting history for individual files should only be done when generating history for renamed files // so the fact that filtering on sinceRevision does not work does not matter there as history // from the initial changeset is needed. The tillRevision filtering works however not - // in combination with --follow. + // in combination with --follow so the filtering is done in MercurialHistoryParser.parse(). + // Even if the revision filtering worked, this approach would be probably faster and consumed less memory. if (this.isHandleRenamedFiles()) { // When using --follow, the returned revisions are from newest to oldest, hence no reverse() is needed. cmd.add("--follow");