fix(python): handle pip options, hashes, and line continuations in requirements.txt

src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java

@@ -95,7 +95,7 @@ public Content provideComponent() throws IOException {
 
   @Override
   protected Set<PackageURL> getIgnoredDependencies(String manifestContent) {
-    String[] lines = manifestContent.split(System.lineSeparator());
+    String[] lines = manifestContent.split("\\R");
     return Arrays.stream(lines)
         .filter(this::containsIgnorePattern)
         .map(PythonPipProvider::extractDepFull)

src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java

@@ -35,6 +35,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -112,10 +114,9 @@ public final List<Map<String, Object>> getDependencies(
 
   private void installingRequirementsOneByOne(String pathToRequirements) {
     try {
-      List<String> requirementsRows = Files.readAllLines(Path.of(pathToRequirements));
+      List<String> requirementsRows =
+          preprocessRequirementsLines(Files.readAllLines(Path.of(pathToRequirements)));
       requirementsRows.stream()
-          .filter((line) -> !line.trim().startsWith("#"))
-          .filter((line) -> !line.trim().isEmpty())
           .forEach(
               (dependency) -> {
                 String dependencyName = getDependencyName(dependency);
@@ -151,11 +152,7 @@ private List<Map<String, Object>> getDependenciesImpl(
     }
     List<String> linesOfRequirements;
     try {
-      linesOfRequirements =
-          Files.readAllLines(requirementsPath).stream()
-              .filter((line) -> !line.trim().startsWith("#") && !line.trim().isEmpty())
-              .map(String::trim)
-              .collect(Collectors.toList());
+      linesOfRequirements = preprocessRequirementsLines(Files.readAllLines(requirementsPath));
     } catch (IOException e) {
       log.warning(
           "Error while trying to read the requirements.txt file, will not be able to install"
@@ -377,6 +374,71 @@ protected String getDependencyNameShow(String pipShowOutput) {
     return versionToken.substring(0, endOfLine).trim();
   }
 
+  private static final Pattern INLINE_OPTION_PATTERN = Pattern.compile("\\s--");
+  private static final Pattern WINDOWS_DRIVE_PATH_PATTERN = Pattern.compile("^[a-zA-Z]:[/\\\\]");
+
+  /**
+   * Preprocesses raw requirements.txt lines by joining line continuations, stripping inline
+   * options, and filtering out pip option lines, URLs, local paths, and empty/comment lines.
+   */
+  public static List<String> preprocessRequirementsLines(List<String> rawLines) {
+    // Join line continuations (trailing backslash, possibly followed by whitespace)
+    List<String> joined = new ArrayList<>();
+    StringBuilder current = new StringBuilder();
+    for (String line : rawLines) {
+      String stripped = line.stripTrailing();
+      if (stripped.endsWith("\\")) {
+        current.append(stripped, 0, stripped.length() - 1);
+      } else {
+        current.append(line);
+        joined.add(current.toString());
+        current = new StringBuilder();
+      }
+    }
+    if (current.length() > 0) {
+      joined.add(current.toString());
+    }
+
+    List<String> result = new ArrayList<>();
+    for (String raw : joined) {
+      String line = raw.trim();
+      if (line.isEmpty() || line.startsWith("#")) {
+        continue;
+      }
+      // Filter out pip options (lines starting with -)
+      if (line.startsWith("-")) {
+        continue;
+      }
+      // Filter out local path requirements (./path, ../path, /abs/path, C:\path, C:/path)
+      if (line.startsWith("./")
+          || line.startsWith("../")
+          || line.startsWith("/")
+          || WINDOWS_DRIVE_PATH_PATTERN.matcher(line).find()) {
+        continue;
+      }
+      // Strip PEP 508 direct references (name @ url -> name) before URL check
+      int atIndex = line.indexOf(" @ ");
+      if (atIndex != -1) {
+        line = line.substring(0, atIndex).trim();
+      }
+      // Strip inline pip options (--hash=..., --config-settings=..., etc.)
+      Matcher optionMatcher = INLINE_OPTION_PATTERN.matcher(line);
+      if (optionMatcher.find()) {
+        line = line.substring(0, optionMatcher.start()).trim();
+      }
+      // Filter out bare URLs and VCS URLs — check the requirement part (before any marker)
+      // to avoid false positives from marker strings
+      String requirementPart = line.contains(";") ? line.substring(0, line.indexOf(";")) : line;
+      if (requirementPart.contains("://")) {
+        continue;
+      }
+      if (!line.isEmpty()) {
+        result.add(line);
+      }
+    }
+    return result;
+  }
+
   public static String getDependencyName(String dep) {
     int markerSeparator = dep.indexOf(";");
     String requirement = markerSeparator == -1 ? dep : dep.substring(0, markerSeparator);

src/test/java/io/github/guacsec/trustifyda/utils/PythonControllerBaseTest.java

@@ -2043,4 +2043,163 @@ void when_spliting_pip_show_dep_with_license() {
             + "Requires: \n"
             + "Required-by: cycler, gensim, gTTS, python-dateutil, tweepy\n");
   }
+
+  @Test
+  void preprocessRequirementsLines_filters_extra_index_url() {
+    List<String> input =
+        List.of(
+            "--extra-index-url https://pypi.example.com/simple",
+            "requests==2.28.0",
+            "--index-url https://pypi.org/simple",
+            "flask==2.0.3");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_filters_short_pip_options() {
+    List<String> input =
+        List.of("-r other-requirements.txt", "-c constraints.txt", "numpy==1.24.0");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("numpy==1.24.0"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_strips_hashes() {
+    List<String> input =
+        List.of("requests==2.28.0 --hash=sha256:abc123 --hash=sha256:def456", "flask==2.0.3");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_strips_config_settings() {
+    List<String> input =
+        List.of(
+            "aiohappyeyeballs==2.6.1 --config-settings=KEY=VALUE --config-settings=OTHER=VAL",
+            "flask==2.0.3");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("aiohappyeyeballs==2.6.1", "flask==2.0.3"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_joins_line_continuations() {
+    List<String> input =
+        List.of(
+            "requests==2.28.0 \\",
+            "    --hash=sha256:abc123 \\",
+            "    --hash=sha256:def456",
+            "flask==2.0.3");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_joins_continuations_with_config_settings() {
+    List<String> input =
+        List.of(
+            "aiohappyeyeballs==2.6.1 \\",
+            "    --config-settings=SAMPLE_TEXT=TEST_VALUE \\",
+            "    --config-settings=ANOTHER_KEY=ANOTHER_VALUE",
+            "async-timeout==5.0.1 ; python_full_version < '3.11'");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(
+        List.of("aiohappyeyeballs==2.6.1", "async-timeout==5.0.1 ; python_full_version < '3.11'"),
+        result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_filters_comments_and_empty_lines() {
+    List<String> input = List.of("# this is a comment", "", "  ", "requests==2.28.0");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_strips_direct_references() {
+    List<String> input =
+        List.of(
+            "pip @ https://github.com/pypa/pip/archive/22.0.2.zip",
+            "requests[security] @ https://github.com/psf/requests/archive/main.zip",
+            "flask==2.0.3");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("pip", "requests[security]", "flask==2.0.3"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_handles_trailing_whitespace_after_backslash() {
+    List<String> input =
+        List.of("requests==2.28.0 \\   ", "    --hash=sha256:abc123", "flask==2.0.3");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_filters_bare_urls() {
+    List<String> input =
+        List.of(
+            "https://example.com/packages/MyPackage-1.0.tar.gz",
+            "http://example.com/packages/other.whl",
+            "git+https://git.example.com/MyProject.git@v1.0",
+            "git+git://git.example.com/repo.git",
+            "hg+http://hg.example.com/repo",
+            "hg+ssh://hg.example.com/repo",
+            "svn+svn://svn.example.com/project/trunk",
+            "bzr+ftp://bzr.example.com/repo",
+            "requests==2.28.0");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_filters_local_paths() {
+    List<String> input =
+        List.of(
+            "./local-package",
+            "../parent-package",
+            "./downloads/MyPackage-1.0.tar.gz",
+            "/absolute/path/to/package",
+            "requests==2.28.0");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_filters_windows_paths() {
+    List<String> input =
+        List.of(
+            "C:\\Users\\dev\\my-package",
+            "c:/projects/my-lib",
+            "D:\\packages\\MyPackage-1.0.tar.gz",
+            "requests==2.28.0");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_handles_final_line_ending_with_backslash() {
+    List<String> input = List.of("flask==2.0.3", "requests==2.28.0 \\");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("flask==2.0.3", "requests==2.28.0"), result);
+  }
+
+  @Test
+  void preprocessRequirementsLines_combined_scenario() {
+    List<String> input =
+        List.of(
+            "--extra-index-url https://pypi.example.com/simple",
+            "# A comment",
+            "requests==2.28.0 \\   ",
+            "    --hash=sha256:abc123",
+            "",
+            "--trusted-host pypi.example.com",
+            "flask==2.0.3 --hash=sha256:xyz789",
+            "pip @ https://github.com/pypa/pip/archive/22.0.2.zip",
+            "https://example.com/packages/other.tar.gz",
+            "git+git://git.example.com/repo.git",
+            "./local-package",
+            "numpy>=1.24.0");
+    List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
+    assertEquals(List.of("requests==2.28.0", "flask==2.0.3", "pip", "numpy>=1.24.0"), result);
+  }
 }