From 63ba4f692f906441ee2cfe90426e953fc9da499f Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Mon, 17 Aug 2026 14:37:45 +0100 Subject: [PATCH] Remove Selenium module, fixes #1994 Signed-off-by: Julien Nioche --- docs/src/main/asciidoc/configuration.adoc | 9 - docs/src/main/asciidoc/internals.adoc | 42 ++--- external/selenium/README.md | 45 ----- external/selenium/pom.xml | 106 ------------ external/selenium/selenium-conf.yaml | 41 ----- .../protocol/selenium/NavigationFilter.java | 36 ---- .../protocol/selenium/NavigationFilters.java | 116 ------------- .../selenium/RemoteDriverProtocol.java | 137 --------------- .../protocol/selenium/SeleniumProtocol.java | 116 ------------- .../protocol/selenium/ProtocolTest.java | 156 ------------------ pom.xml | 1 - 11 files changed, 22 insertions(+), 783 deletions(-) delete mode 100644 external/selenium/README.md delete mode 100644 external/selenium/pom.xml delete mode 100644 external/selenium/selenium-conf.yaml delete mode 100644 external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java delete mode 100644 external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java delete mode 100644 external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java delete mode 100644 external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java delete mode 100644 external/selenium/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 3fc80f267..40dec13b8 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -263,15 +263,6 @@ header. | http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. | http.filter.ipaddress.exclude | - | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. See <>. | http.trust.everything | true | If true, trust all SSL/TLS connections. -| navigationfilters.config.file | - | JSON config for NavigationFilter (used by the Selenium protocol module). -| selenium.addresses | - | WebDriver server addresses. -| selenium.capabilities | - | Desired WebDriver capabilities -. -| selenium.delegated.protocol | - | Delegated protocol for selective Selenium usage. -| selenium.implicitlyWait | 0 | WebDriver element search timeout. -| selenium.instances.num | 1 | Number of instances per WebDriver connection. -| selenium.pageLoadTimeout | 0 | WebDriver page load timeout. -| selenium.setScriptTimeout | 0 | WebDriver script execution timeout. | topology.message.timeout.secs | -1 | OKHTTP message timeout. |=== diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc index ab5cddeb5..262e7c074 100644 --- a/docs/src/main/asciidoc/internals.adoc +++ b/docs/src/main/asciidoc/internals.adoc @@ -385,7 +385,7 @@ When handling HTTP redirects, StormCrawler offers three modes: 2. **Redirects disabled** (`redirections.allowed: false`): Redirect target URLs are not sent to the status stream. Redirects are effectively ignored. -3. **Immediate follow** (`http.allow.redirects: true`): Redirects are followed immediately in the HTTP client and the target URLs are not emitted to the status stream. This is the default behavior for browser-based protocols (Selenium and Playwright), but it is also supported by the OkHttp protocol. Note that with immediate follows, redirect targets bypass URL filtering, deduplication, and `robots.txt` checks. +3. **Immediate follow** (`http.allow.redirects: true`): Redirects are followed immediately in the HTTP client and the target URLs are not emitted to the status stream. This is the default behavior for the browser-based Playwright protocol, but it is also supported by the OkHttp protocol. Note that with immediate follows, redirect targets bypass URL filtering, deduplication, and `robots.txt` checks. ==== Network Protocols @@ -422,36 +422,38 @@ https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProt ---- * link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient] (*deprecated* — will be removed in the next major release; use OKHttp instead) -* link:https://github.com/apache/stormcrawler/blob/main/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java[Selenium] * link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java[OKHttp] +* link:https://github.com/apache/stormcrawler/blob/main/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java[Playwright] (browser-based, in the `playwright` external module) ==== Feature grid -[cols="2,1,1,1", options="header"] +[cols="2,1,1", options="header"] |=== -| Features | HTTPClient | OKHttp | Selenium - -| Basic authentication | link:https://github.com/apache/stormcrawler/pull/589[Y] | link:https://github.com/apache/stormcrawler/issues/792[Y] | N -| Proxy (w. credentials?) | Y / Y | Y / link:https://github.com/apache/stormcrawler/issues/751[Y] | ? -| Interruptible / trimmable link:https://github.com/apache/stormcrawler/issues/463[#463] | N / Y | Y / Y | Y / N -| Cookies | Y | link:https://github.com/apache/stormcrawler/issues/632[Y] | N -| Response headers | Y | Y | N -| Trust all certificates | N | link:https://github.com/apache/stormcrawler/issues/615[Y] | N -| HEAD method | link:https://github.com/apache/stormcrawler/issues/485[Y] | link:https://github.com/apache/stormcrawler/pull/923[Y] | N -| POST method | N | link:https://github.com/apache/stormcrawler/issues/641[Y] | N -| Verbatim response header | link:https://github.com/apache/stormcrawler/issues/317[Y] | link:https://github.com/apache/stormcrawler/issues/506[Y] | N -| Verbatim request header | N | link:https://github.com/apache/stormcrawler/issues/506[Y] | N -| IP address capture | N | link:https://github.com/apache/stormcrawler/pull/691[Y] | N -| Navigation and javascript | N | N | Y -| HTTP/2 | N | Y | (Y) -| Configurable connection pool | N | link:https://github.com/apache/stormcrawler/issues/918[Y] | N +| Features | HTTPClient | OKHttp + +| Basic authentication | link:https://github.com/apache/stormcrawler/pull/589[Y] | link:https://github.com/apache/stormcrawler/issues/792[Y] +| Proxy (w. credentials?) | Y / Y | Y / link:https://github.com/apache/stormcrawler/issues/751[Y] +| Interruptible / trimmable link:https://github.com/apache/stormcrawler/issues/463[#463] | N / Y | Y / Y +| Cookies | Y | link:https://github.com/apache/stormcrawler/issues/632[Y] +| Response headers | Y | Y +| Trust all certificates | N | link:https://github.com/apache/stormcrawler/issues/615[Y] +| HEAD method | link:https://github.com/apache/stormcrawler/issues/485[Y] | link:https://github.com/apache/stormcrawler/pull/923[Y] +| POST method | N | link:https://github.com/apache/stormcrawler/issues/641[Y] +| Verbatim response header | link:https://github.com/apache/stormcrawler/issues/317[Y] | link:https://github.com/apache/stormcrawler/issues/506[Y] +| Verbatim request header | N | link:https://github.com/apache/stormcrawler/issues/506[Y] +| IP address capture | N | link:https://github.com/apache/stormcrawler/pull/691[Y] +| Navigation and javascript | N | N +| HTTP/2 | N | Y +| Configurable connection pool | N | link:https://github.com/apache/stormcrawler/issues/918[Y] |=== +Neither of the HTTP clients executes JavaScript: use the browser-based link:https://github.com/apache/stormcrawler/tree/main/external/playwright[Playwright] protocol for pages that require rendering or navigation. + ==== HTTP/2 * The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java[OKHttp] protocol supports link:https://en.wikipedia.org/wiki/HTTP/2[HTTP/2] if the JDK includes link:https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation[ALPN] (Java 9 and upwards or Java 8 builds starting early/mid 2020). * link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient] does not yet support HTTP/2. -* link:https://github.com/apache/stormcrawler/blob/main/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java[Selenium]: whether HTTP/2 is used or not depends on the used driver. +* link:https://github.com/apache/stormcrawler/blob/main/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java[Playwright]: whether HTTP/2 is used or not depends on the browser. Since link:https://github.com/apache/stormcrawler/pull/829[#829], the HTTP protocol version used is configurable via `http.protocol.versions` (see also comments in link:https://github.com/apache/stormcrawler/blob/main/core/src/main/resources/crawler-default.yaml[crawler-default.yaml]). diff --git a/external/selenium/README.md b/external/selenium/README.md deleted file mode 100644 index d2cbd60af..000000000 --- a/external/selenium/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Selenium Protocol for Apache StormCrawler - -> **⚠️ Deprecated:** The Selenium module is deprecated and will be removed in the next major release of StormCrawler. - -This project provides a **[Selenium](https://www.selenium.dev/)-based protocol implementation** for Apache StormCrawler. - ---- - -## Overview - -The [Selenium](https://www.selenium.dev) protocol allows StormCrawler to interact with dynamic web pages using Selenium WebDriver. It is particularly useful for crawling JavaScript-heavy sites that require a real browser environment. - ---- - -## Configuration - -Add `selenium-conf.yaml` to your topology configuration. Below is a sample configuration: - -```yaml -# navigationfilters.config.file: "navigationfilters.json" -# selenium.addresses: "http://localhost:9515" - -# Enable or disable Selenium tracing (default: false) -selenium.tracing: false - -# Selenium timeouts (rely on Selenium defaults if set to -1) -selenium.timeouts: - script: -1 # Maximum time for scripts to run - pageLoad: -1 # Maximum time to wait for page load - implicit: -1 # Implicit wait time for finding elements - -# Selenium capabilities -# selenium.capabilities: -# browserName: "chrome" # Required: choose your browser -# phantomjs.page.settings.userAgent: "$userAgent" # Example: set custom user agent -# -# # ChromeDriver specific options -# goog:chromeOptions: -# args: -# - "--headless" # Run Chrome in headless mode -# - "--disable-gpu" # Disable GPU acceleration -# - "--mute-audio" # Mute audio output - - - diff --git a/external/selenium/pom.xml b/external/selenium/pom.xml deleted file mode 100644 index d2e5d02e5..000000000 --- a/external/selenium/pom.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - - - 4.0.0 - - - org.apache.stormcrawler - stormcrawler-external - 4.0.0-SNAPSHOT - ../pom.xml - - - stormcrawler-selenium - jar - - stormcrawler-selenium - https://github.com/apache/stormcrawler/tree/master/external/selenium - Selenium-based Protocol for StormCrawler (deprecated - will be removed in the next major release) - - - 4.46.0 - 3.13.2 - true - 1.00 - 0.69 - 0.80 - 0.62 - 0.65 - 0.55 - - - - - org.seleniumhq.selenium - selenium-remote-driver - ${selenium.version} - - - - org.seleniumhq.selenium - selenium-support - ${selenium.version} - - - - org.apache.stormcrawler - stormcrawler-core - ${project.version} - test-jar - test - - - - org.wiremock - wiremock - ${wiremock.version} - test - - - - org.awaitility - awaitility - test - - - - org.testcontainers - selenium - test - - - - org.seleniumhq.selenium - selenium-chrome-driver - ${selenium.version} - test - - - - org.testcontainers - junit-jupiter - test - - - - - diff --git a/external/selenium/selenium-conf.yaml b/external/selenium/selenium-conf.yaml deleted file mode 100644 index 83250ed74..000000000 --- a/external/selenium/selenium-conf.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -config: - # navigationfilters.config.file: "navigationfilters.json" - # selenium.addresses: "http://localhost:9515" - - selenium.tracing: false - - # rely on selenium's default values - # set to a value >= 0 to override - selenium.timeouts: - script: -1 - pageLoad: -1 - implicit: -1 - - # selenium.capabilities: - # a browser name is required - # browserName:"chrome" - # illustrates the use of the variable for user agent - # phantomjs.page.settings.userAgent: "$userAgent" - # ChromeDriver config - # goog:chromeOptions: - # args: - # - "--headless" - # - "--disable-gpu" - # - "--mute-audio" - - diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java deleted file mode 100644 index 284f9ed48..000000000 --- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.protocol.selenium; - -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.protocol.ProtocolResponse; -import org.apache.stormcrawler.util.AbstractConfigurable; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.openqa.selenium.remote.RemoteWebDriver; - -/** - * @deprecated the Selenium module is deprecated and will be removed in the next major release of - * StormCrawler. - */ -@Deprecated -public abstract class NavigationFilter extends AbstractConfigurable { - /** The end result comes from the first filter to return non-null * */ - public abstract @Nullable ProtocolResponse filter( - @NotNull RemoteWebDriver driver, @NotNull Metadata metadata); -} diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java deleted file mode 100644 index 02f1c40a6..000000000 --- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.protocol.selenium; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang3.StringUtils; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.protocol.ProtocolResponse; -import org.apache.stormcrawler.util.ConfUtils; -import org.apache.stormcrawler.util.Configurable; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.slf4j.LoggerFactory; - -/** - * Wrapper for the NavigationFilter defined in a JSON configuration. - * - * @see org.apache.stormcrawler.util.Configurable#createConfiguredInstance(Class, Class, Map, - * JsonNode) for more information. - * @deprecated the Selenium module is deprecated and will be removed in the next major release of - * StormCrawler. - */ -@Deprecated -public class NavigationFilters extends NavigationFilter { - - public static final NavigationFilters emptyNavigationFilters = new NavigationFilters(); - - private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(NavigationFilters.class); - - private NavigationFilter[] filters; - - private NavigationFilters() { - filters = new NavigationFilter[0]; - } - - /** - * loads the filters from a JSON configuration file - * - * @throws IOException - */ - public NavigationFilters(@NotNull Map stormConf, @NotNull String configFile) - throws IOException { - // load the JSON configFile - // build a JSON object out of it - JsonNode confNode; - try (InputStream confStream = getClass().getClassLoader().getResourceAsStream(configFile)) { - ObjectMapper mapper = new ObjectMapper(); - confNode = mapper.readValue(confStream, JsonNode.class); - } catch (Exception e) { - throw new IOException("Unable to build JSON object from file", e); - } - - configure(stormConf, confNode); - } - - public @Nullable ProtocolResponse filter( - @NotNull RemoteWebDriver driver, @NotNull Metadata metadata) { - for (NavigationFilter filter : filters) { - ProtocolResponse response = filter.filter(driver, metadata); - if (response != null) { - return response; - } - } - return null; - } - - /** - * Loads and configure the NavigationFilters based on the storm config if there is one otherwise - * returns an emptyNavigationFilters. - */ - public static NavigationFilters fromConf(@NotNull Map stormConf) { - String configfile = ConfUtils.getString(stormConf, "navigationfilters.config.file"); - if (StringUtils.isNotBlank(configfile)) { - try { - return new NavigationFilters(stormConf, configfile); - } catch (IOException e) { - String message = - "Exception caught while loading the NavigationFilters from " + configfile; - LOG.error(message); - throw new RuntimeException(message, e); - } - } - - return NavigationFilters.emptyNavigationFilters; - } - - @Override - public void configure(@NotNull Map stormConf, @NotNull JsonNode filtersConf) { - List filterLists = - Configurable.createConfiguredInstance( - this.getClass(), NavigationFilter.class, stormConf, filtersConf); - - filters = filterLists.toArray(new NavigationFilter[0]); - } -} diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java deleted file mode 100644 index 5ef162a12..000000000 --- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.protocol.selenium; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import org.apache.storm.Config; -import org.apache.stormcrawler.protocol.Protocol; -import org.apache.stormcrawler.util.ConfUtils; -import org.apache.stormcrawler.util.URLUtil; -import org.openqa.selenium.WebDriver.Timeouts; -import org.openqa.selenium.remote.DesiredCapabilities; -import org.openqa.selenium.remote.RemoteWebDriver; - -/** - * Delegates the requests to one or more remote selenium servers. The processes must be started / - * stopped separately. The URLs to connect to are specified with the config 'selenium.addresses'. - * - * @deprecated the Selenium module is deprecated and will be removed in the next major release of - * StormCrawler. - */ -@Deprecated -public class RemoteDriverProtocol extends SeleniumProtocol { - - private void substituteUserAgent(Map keyvals, final String userAgentString) { - if (keyvals == null) { - return; - } - - for (Entry entry : keyvals.entrySet()) { - Object val = entry.getValue(); - // substitute variable $useragent for the real value - if (val instanceof String && val.toString().contains("$useragent")) { - String newval = ((String) val).replaceAll("\\$useragent", userAgentString); - entry.setValue(newval); - } else if (val instanceof Map) { - substituteUserAgent((Map) val, userAgentString); - } else if (val instanceof List) { - List newList = new ArrayList<>(); - ((List) val) - .forEach( - s -> { - String newval = s.replaceAll("\\$useragent", userAgentString); - newList.add(newval); - }); - entry.setValue(newList); - } - } - } - - @Override - public void configure(Config conf) { - super.configure(conf); - - final String userAgentString = getAgentString(conf); - - // custom capabilities - final Map confCapabilities = - (Map) conf.get("selenium.capabilities"); - - substituteUserAgent(confCapabilities, userAgentString); - - // see https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities - final DesiredCapabilities capabilities = new DesiredCapabilities(); - - confCapabilities.forEach((k, v) -> capabilities.setCapability(k, v)); - - LOG.info("Configuring Selenium with {}", capabilities); - - // load addresses from config - List addresses = ConfUtils.loadListFromConf("selenium.addresses", conf); - if (addresses.size() == 0) { - throw new RuntimeException("No value found for selenium.addresses"); - } - - final boolean tracing = ConfUtils.getBoolean(conf, "selenium.tracing", false); - - // TEMPORARY: draw attention to config change - for (String p : new String[] {"implicitlyWait", "pageLoadTimeout", "scriptTimeout"}) { - if (conf.containsKey("selenium." + p)) { - String message = "selenium." + p + " is deprecated. Please use selenium.timeouts!"; - LOG.error(message); - throw new RuntimeException(message); - } - } - - for (String cdaddress : addresses) { - try { - RemoteWebDriver driver = - new RemoteWebDriver(URLUtil.toURL(cdaddress), capabilities, tracing); - // setting timouts - // see https://www.browserstack.com/guide/understanding-selenium-timeouts - Timeouts touts = driver.manage().timeouts(); - Map timeouts = (Map) conf.get("selenium.timeouts"); - if (timeouts != null) { - long implicitTimeout = timeouts.getOrDefault("implicit", -1).longValue(); - long pageLoadTimeout = timeouts.getOrDefault("pageLoad", -1).longValue(); - long scriptTimeout = timeouts.getOrDefault("script", -1).longValue(); - if (implicitTimeout != -1) { - touts.implicitlyWait(Duration.ofMillis(implicitTimeout)); - } - if (pageLoadTimeout != -1) { - touts.pageLoadTimeout(Duration.ofMillis(pageLoadTimeout)); - } - if (scriptTimeout != -1) { - touts.scriptTimeout(Duration.ofMillis(scriptTimeout)); - } - } - drivers.add(driver); - } catch (Exception e) { - LOG.error(e.getLocalizedMessage(), e); - } - } - } - - public static void main(String[] args) throws Exception { - Protocol.main(new RemoteDriverProtocol(), args); - } -} diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java deleted file mode 100644 index c0ce3e3e2..000000000 --- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.protocol.selenium; - -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.concurrent.LinkedBlockingQueue; -import org.apache.http.HttpHeaders; -import org.apache.storm.Config; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.protocol.AbstractHttpProtocol; -import org.apache.stormcrawler.protocol.ProtocolResponse; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.slf4j.LoggerFactory; - -/** - * @deprecated the Selenium module is deprecated and will be removed in the next major release of - * StormCrawler. - */ -@Deprecated -public abstract class SeleniumProtocol extends AbstractHttpProtocol { - - protected static final org.slf4j.Logger LOG = LoggerFactory.getLogger(SeleniumProtocol.class); - - protected LinkedBlockingQueue drivers; - - private NavigationFilters filters; - - public static final String MD_KEY_START = "selenium.protocol.start"; - public static final String MD_KEY_END = "selenium.protocol.end"; - - @Override - public void configure(Config conf) { - super.configure(conf); - drivers = new LinkedBlockingQueue<>(); - filters = NavigationFilters.fromConf(conf); - } - - public ProtocolResponse getProtocolOutput(String url, Metadata metadata) throws Exception { - RemoteWebDriver driver; - while ((driver = getDriver()) == null) { - // get there if there has been an interrupted exception - // just try again - } - try { - final Metadata outputMeta = new Metadata(); - outputMeta.addValue(MD_KEY_START, Instant.now().toString()); - - // This will block for the page load and any - // associated AJAX requests - driver.get(url); - - String u = driver.getCurrentUrl(); - - // call the filters - ProtocolResponse response = filters.filter(driver, metadata); - if (response != null) { - return response; - } - - // if the URL is different then we must have hit a redirection - if (!u.equalsIgnoreCase(url)) { - byte[] content = new byte[] {}; - Metadata m = new Metadata(); - m.addValue(HttpHeaders.LOCATION, u); - return new ProtocolResponse(content, 307, m); - } - - outputMeta.addValue(MD_KEY_END, Instant.now().toString()); - - // if no filters got triggered - byte[] content = driver.getPageSource().getBytes(StandardCharsets.UTF_8); - return new ProtocolResponse(content, 200, outputMeta); - - } finally { - // finished with this driver - return it to the queue - drivers.put(driver); - } - } - - /** Returns the first available driver * */ - private final RemoteWebDriver getDriver() { - try { - return drivers.take(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return null; - } - - @Override - public void cleanup() { - LOG.info("Cleanup called on Selenium protocol drivers"); - synchronized (drivers) { - drivers.forEach( - (d) -> { - d.close(); - }); - } - } -} diff --git a/external/selenium/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java b/external/selenium/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java deleted file mode 100644 index 5e03e6580..000000000 --- a/external/selenium/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.protocol.selenium; - -import static org.awaitility.Awaitility.await; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.mutable.MutableBoolean; -import org.apache.storm.Config; -import org.apache.storm.utils.MutableObject; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.protocol.AbstractProtocolTest; -import org.apache.stormcrawler.protocol.ProtocolResponse; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.openqa.selenium.chrome.ChromeOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.BrowserWebDriverContainer; -import org.testcontainers.containers.BrowserWebDriverContainer.VncRecordingMode; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.utility.DockerImageName; - -/** - * Tests the Selenium protocol implementation on a standalone Chrome instance and not through - * Selenium Grid. https://java.testcontainers.org/modules/webdriver_containers/#example - */ -@Testcontainers(disabledWithoutDocker = true) -class ProtocolTest extends AbstractProtocolTest { - - private static final Logger LOG = LoggerFactory.getLogger(ProtocolTest.class); - - private static final DockerImageName SELENIUM_IMAGE = - DockerImageName.parse("seleniarm/standalone-chromium:124.0") - .asCompatibleSubstituteFor("selenium/standalone-chrome"); - - private BrowserWebDriverContainer chrome = - new BrowserWebDriverContainer<>(SELENIUM_IMAGE) - .withCapabilities(new ChromeOptions()) - .withRecordingMode(VncRecordingMode.SKIP, null) - .withAccessToHost(true) - .withExtraHost("website.test", "host-gateway"); - - public RemoteDriverProtocol getProtocol() { - LOG.info( - "Configuring protocol instance to connect to {}", - chrome.getSeleniumAddress().toExternalForm()); - List l = new ArrayList<>(); - // l.add("--no-sandbox"); - // l.add("--disable-dev-shm-usage"); - // l.add("--headless"); - // l.add("--disable-gpu"); - // l.add("--remote-allow-origins=*"); - Map m = new HashMap<>(); - m.put("args", l); - // m.put("extensions", Collections.EMPTY_LIST); - Map capabilities = new HashMap<>(); - capabilities.put("browserName", "chrome"); - capabilities.put("goog:chromeOptions", m); - Config conf = new Config(); - conf.put("http.agent.name", "this_is_only_a_test"); - conf.put("selenium.addresses", chrome.getSeleniumAddress().toExternalForm()); - Map timeouts = new HashMap<>(); - timeouts.put("implicit", 10000); - timeouts.put("pageLoad", 10000); - timeouts.put("script", 10000); - conf.put("selenium.timeouts", timeouts); - conf.put("selenium.capabilities", capabilities); - RemoteDriverProtocol protocol = new RemoteDriverProtocol(); - protocol.configure(conf); - return protocol; - } - - @BeforeEach - void init() { - chrome.start(); - } - - /** - * you can configure one instance of Selenium to talk to multiple drivers but can't have a - * multiple instances of the protocol. If there is only one instance and one target, you must - * wait... - */ - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - void testBlocking() { - RemoteDriverProtocol protocol = getProtocol(); - MutableBoolean noException = new MutableBoolean(true); - MutableObject endTimeFirst = new MutableObject(); - MutableObject startTimeSecond = new MutableObject(); - await().until(() -> httpServer.isRunning()); - final String url = "http://website.test" + ":" + HTTP_PORT + "/"; - new Thread( - () -> { - try { - ProtocolResponse response = - protocol.getProtocolOutput(url, new Metadata()); - endTimeFirst.setObject( - Instant.parse( - response.getMetadata() - .getFirstValue( - SeleniumProtocol.MD_KEY_END))); - } catch (Exception e) { - noException.setValue(false); - } - }) - .start(); - new Thread( - () -> { - try { - ProtocolResponse response = - protocol.getProtocolOutput(url, new Metadata()); - startTimeSecond.setObject( - Instant.parse( - response.getMetadata() - .getFirstValue( - SeleniumProtocol.MD_KEY_START))); - } catch (Exception e) { - noException.setValue(false); - } - }) - .start(); - await().until( - () -> - endTimeFirst.getObject() != null - && startTimeSecond.getObject() != null); - Instant etf = (Instant) endTimeFirst.getObject(); - Instant sts = (Instant) startTimeSecond.getObject(); - // check that the second call started AFTER the first one finished - Assertions.assertTrue(etf.isBefore(sts)); - Assertions.assertTrue(noException.booleanValue()); - protocol.cleanup(); - } -} diff --git a/pom.xml b/pom.xml index 65f2a1e2f..2277cd275 100644 --- a/pom.xml +++ b/pom.xml @@ -724,7 +724,6 @@ under the License. external/opensearch external/opensearch-java external/playwright - external/selenium external/solr external/sql external/tika