diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java index 2b4dbe3c498..2f912bb158b 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashSet; @@ -790,11 +791,11 @@ public void removeRepositories() { * Search through the directory for repositories and use the result to replace * the lists of repositories in both RuntimeEnvironment/Configuration and HistoryGuru. * - * @param dir the root directory to start the search in + * @param dir the directories to start the search in */ - public void setRepositories(String dir) { + public void setRepositories(String... dir) { List repos = new ArrayList<>(HistoryGuru.getInstance(). - addRepositories(new File[]{new File(dir)}, + addRepositories(Arrays.stream(dir).map(File::new).toArray(File[]::new), RuntimeEnvironment.getInstance().getIgnoredNames())); RuntimeEnvironment.getInstance().setRepositories(repos); } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index 3da18fd2e34..57ac86d4863 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -35,6 +35,7 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -118,6 +119,7 @@ public final class Indexer { private static int status = 0; private static final Set repositories = new HashSet<>(); + private static Set searchPaths = new HashSet<>(); private static final HashSet allowedSymlinks = new HashSet<>(); private static final HashSet canonicalRoots = new HashSet<>(); private static final Set defaultProjects = new TreeSet<>(); @@ -335,7 +337,16 @@ public static void main(String[] argv) { new Object[]{Info.getVersion(), Info.getRevision()}); // Create history cache first. - getInstance().prepareIndexer(env, searchRepositories, addProjects, + if (searchRepositories) { + if (searchPaths.isEmpty()) { + searchPaths.add(env.getSourceRootPath()); + } else { + searchPaths = searchPaths.stream(). + map(t -> Paths.get(env.getSourceRootPath(), t).toString()). + collect(Collectors.toSet()); + } + } + getInstance().prepareIndexer(env, searchPaths, addProjects, createDict, runIndex, subFiles, new ArrayList<>(repositories)); // prepareIndexer() populated the list of projects so now default projects can be set. @@ -417,6 +428,8 @@ public static String[] parseOptions(String[] argv) throws ParseException { }); }); + searchPaths.clear(); + // Limit usage lines to 72 characters for concise formatting. optParser = OptionParser.Do(parser -> { @@ -689,9 +702,16 @@ public static String[] parseOptions(String[] argv) throws ParseException { "are history-eligible; using --repository limits to only those specified.", "Option may be repeated.").Do(v -> repositories.add((String) v)); - parser.on("-S", "--search", - "Search for source repositories under -s,--source, and add them.").Do(v -> - searchRepositories = true); + parser.on("-S", "--search", "=[path/to/repository]", + "Search for source repositories under -s,--source, and add them." + + "Path (relative to the source root) to repository is optional. " + + "Option may be repeated.").Do(v -> { + searchRepositories = true; + String repoPath = (String) v; + if (!repoPath.isEmpty()) { + searchPaths.add(repoPath); + } + }); parser.on("-s", "--source", "=/path/to/source/root", "The root directory of the source tree."). @@ -873,8 +893,9 @@ public void prepareIndexer(RuntimeEnvironment env, List subFiles, List repositories) throws IndexerException, IOException { - prepareIndexer(env, searchRepositories, addProjects, createDict, true, - subFiles, repositories); + prepareIndexer(env, + searchRepositories ? Collections.singleton(env.getSourceRootPath()) : Collections.emptySet(), + addProjects, createDict, true, subFiles, repositories); } /** @@ -888,7 +909,7 @@ public void prepareIndexer(RuntimeEnvironment env, * for performance. We prefer clarity over performance here, so silence it. * * @param env runtime environment - * @param searchRepositories if true, search for repositories + * @param searchPaths list of paths in which to search for repositories * @param addProjects if true, add projects * @param createDict if true, create dictionary * @param createHistoryCache create history cache flag @@ -899,7 +920,7 @@ public void prepareIndexer(RuntimeEnvironment env, */ @SuppressWarnings("PMD.SimplifyStartsWith") public void prepareIndexer(RuntimeEnvironment env, - boolean searchRepositories, + Set searchPaths, boolean addProjects, boolean createDict, boolean createHistoryCache, @@ -954,10 +975,10 @@ public void prepareIndexer(RuntimeEnvironment env, } } - if (searchRepositories) { - LOGGER.log(Level.INFO, "Scanning for repositories..."); + if (!searchPaths.isEmpty()) { + LOGGER.log(Level.INFO, "Scanning for repositories in {0}...", searchPaths); Statistics stats = new Statistics(); - env.setRepositories(env.getSourceRootPath()); + env.setRepositories(searchPaths.toArray(new String[0])); stats.report(LOGGER, String.format("Done scanning for repositories, found %d repositories", env.getRepositories().size())); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerMainTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerMainTest.java new file mode 100644 index 00000000000..d74472209ed --- /dev/null +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerMainTest.java @@ -0,0 +1,96 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017-2019, Chris Fraire . + */ + +package org.opengrok.indexer.index; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.opengrok.indexer.configuration.RuntimeEnvironment; +import org.opengrok.indexer.history.HistoryGuru; +import org.opengrok.indexer.util.TestRepository; + +import java.io.IOException; + +import static org.junit.Assert.assertFalse; + +public class IndexerMainTest { + private TestRepository repository; + + @Before + public void setUp() throws IOException { + repository = new TestRepository(); + // For these tests we need Mercurial repository with renamed files. + repository.create(HistoryGuru.class.getResourceAsStream("repositories.zip")); + } + + @After + public void tearDown() { + repository.destroy(); + } + + private void checkNumberOfThreads() { + /* + * There should not be any threads in the renamed pool now. + * We need to check it like this since the test framework tears + * down the threads at the end of the test case run so any + * hangs due to executors not being shut down would not be visible. + */ + ThreadGroup mainGroup = Thread.currentThread().getThreadGroup(); + Thread[] threads = new Thread[mainGroup.activeCount()]; + mainGroup.enumerate(threads); + for (int i = 0; i < threads.length; i++) { + if (threads[i] == null || threads[i].getName() == null) { + continue; + } + assertFalse(threads[i].getName().contains("renamed-handling")); + } + } + + /** + * Test cleanup of renamed thread pool after indexing with -H. + */ + @Test + public void testMainWithH() { + System.out.println("Generate index by using command line options with -H"); + RuntimeEnvironment env = RuntimeEnvironment.getInstance(); + String[] argv = {"-S", "-H", "-s", repository.getSourceRoot(), + "-d", repository.getDataRoot(), "-v", "-c", env.getCtags()}; + Indexer.main(argv); + checkNumberOfThreads(); + } + + /** + * Test cleanup of renamed thread pool after indexing without -H. + */ + @Test + public void testMainWithoutH() { + System.out.println("Generate index by using command line options without -H"); + RuntimeEnvironment env = RuntimeEnvironment.getInstance(); + String[] argv = {"-S", "-P", "-s", repository.getSourceRoot(), + "-d", repository.getDataRoot(), "-v", "-c", env.getCtags()}; + Indexer.main(argv); + checkNumberOfThreads(); + } +} diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java index d2a704ef06a..632dd5dec1b 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java @@ -79,24 +79,6 @@ public void tearDown() { repository.destroy(); } - private void checkNumberOfThreads() { - /* - * There should not be any threads in the renamed pool now. - * We need to check it like this since the test framework tears - * down the threads at the end of the test case run so any - * hangs due to executors not being shut down would not be visible. - */ - ThreadGroup mainGroup = Thread.currentThread().getThreadGroup(); - Thread[] threads = new Thread[mainGroup.activeCount()]; - mainGroup.enumerate(threads); - for (int i = 0; i < threads.length; i++) { - if (threads[i] == null || threads[i].getName() == null) { - continue; - } - assertFalse(threads[i].getName().contains("renamed-handling")); - } - } - /** * Test it is possible to disable history per project. */ @@ -222,30 +204,4 @@ public void testSymlinks() throws IndexerException, IOException { IOUtils.removeRecursive(realSource); IOUtils.removeRecursive(sourceRoot); } - - /** - * Test cleanup of renamed thread pool after indexing with -H. - */ - @Test - public void testMainWithH() { - System.out.println("Generate index by using command line options with -H"); - RuntimeEnvironment env = RuntimeEnvironment.getInstance(); - String[] argv = {"-S", "-H", "-s", repository.getSourceRoot(), - "-d", repository.getDataRoot(), "-v", "-c", env.getCtags()}; - Indexer.main(argv); - checkNumberOfThreads(); - } - - /** - * Test cleanup of renamed thread pool after indexing without -H. - */ - @Test - public void testMainWithoutH() { - System.out.println("Generate index by using command line options without -H"); - RuntimeEnvironment env = RuntimeEnvironment.getInstance(); - String[] argv = {"-S", "-P", "-s", repository.getSourceRoot(), - "-d", repository.getDataRoot(), "-v", "-c", env.getCtags()}; - Indexer.main(argv); - checkNumberOfThreads(); - } } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java index b717f174cd2..4e94db4c6b9 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java @@ -33,6 +33,7 @@ import java.io.FileWriter; import java.io.IOException; import java.io.StringWriter; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -350,6 +351,25 @@ public void testRemoveFileOnFileChange() throws Exception { testrepo.destroy(); } + @Test + @ConditionalRun(RepositoryInstalled.MercurialInstalled.class) + @ConditionalRun(RepositoryInstalled.GitInstalled.class) + public void testSetRepositories() throws Exception { + RuntimeEnvironment env = RuntimeEnvironment.getInstance(); + + TestRepository testrepo = new TestRepository(); + testrepo.create(HistoryGuru.class.getResourceAsStream("repositories.zip")); + env.setSourceRoot(testrepo.getSourceRoot()); + + env.setRepositories(testrepo.getSourceRoot()); + assertEquals(9, env.getRepositories().size()); + + String[] repoNames = {"mercurial", "git"}; + env.setRepositories(Arrays.stream(repoNames). + map(t -> Paths.get(env.getSourceRootPath(), t).toString()).distinct().toArray(String[]::new)); + assertEquals(2, env.getRepositories().size()); + } + @Test public void testXref() throws IOException { List files = new ArrayList<>(); diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java index 1d682fc0bb5..690d8821be6 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java @@ -291,7 +291,7 @@ public void testDelete() throws Exception { filter(p -> p.getName().equals("git")). collect(Collectors.toSet()).size()); - // Run the indexer (ala 'indexpart') so that data directory is populated. + // Run the indexer so that data directory is populated. ArrayList subFiles = new ArrayList<>(); subFiles.add("/git"); subFiles.add("/mercurial"); @@ -301,7 +301,7 @@ public void testDelete() throws Exception { repos.add("/mercurial"); repos.add("/svn"); // This is necessary so that repositories in HistoryGuru get populated. - // When 'indexpart' is run, this is called from setConfiguration() because + // For per project reindex this is called from setConfiguration() because // of the -R option is present. HistoryGuru.getInstance().invalidateRepositories( env.getRepositories(), null, false);