[pull] master from kevoreilly:master

.prettierignore

@@ -0,0 +1 @@
+*.html

conf/copy_configs.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
-for filename in conf/default/*.conf.default; do
-    cp -vf "./$filename" "./$(echo "$filename" | sed -e 's/.default//g' | sed -e 's/default//g')";
+for filename in conf/default/*.conf.default conf/default/*.env; do
+    dest="conf/${filename#conf/default/}"
+    cp -vf "./$filename" "./${dest%.default}"
 done

conf/default/cape-processor.env

@@ -0,0 +1,26 @@
+# CAPE Processor Configuration overrides
+# Un-comment the variables you want to change
+
+# ID of the analysis to process (default: auto)
+#CAPE_ID=auto
+
+# Max amount of time spent in processing before we fail a task (default: 300)
+#CAPE_PROCESSING_TIMEOUT=900
+
+# Number of parallel threads to use (default: 1)
+#CAPE_PARALLEL=7
+
+# Max children tasks per worker (default: 7)
+#CAPE_MAXTASKSPERCHILD=7
+
+# Enable debug messages (default: false)
+#CAPE_DEBUG=true
+
+# Reprocess failed processing (default: false)
+#CAPE_FAILED_PROCESSING=true
+
+# Enable logging garbage collection related info (default: false)
+#CAPE_MEMORY_DEBUGGING=true
+
+# Disable memory limit (default: false)
+#CAPE_DISABLE_MEMORY_LIMIT=true

installer/cape2.sh

@@ -1399,8 +1399,11 @@ function install_CAPE() {
     fi
 
     cd "$CAPE_ROOT/" || return
-    # copy *.conf.default to *.conf so we have all properly updated fields, as we can't ignore old configs in repository
-    for filename in conf/default/*.conf.default; do cp -vf "./$filename" "./$(echo "$filename" | sed -e 's/.default//g' | sed -e 's/default//g')";  done
+    # copy *.conf.default and *.env to their destination so we have all properly updated fields
+    for filename in conf/default/*.conf.default conf/default/*.env; do
+        dest="conf/${filename#conf/default/}"
+        cp -vf "./$filename" "./${dest%.default}"
+    done
 
     sed -i "/connection =/cconnection = postgresql://${USER}:${PASSWD}@localhost:5432/${USER}" conf/cuckoo.conf
     # sed -i "/tor/{n;s/enabled = no/enabled = yes/g}" conf/routing.conf

lib/cuckoo/common/demux.py

@@ -5,7 +5,7 @@
 import logging
 import os
 import tempfile
-from typing import List
+from typing import Any, Dict, List, Tuple
 
 from lib.cuckoo.common.config import Config
 from lib.cuckoo.common.exceptions import CuckooDemuxError
@@ -185,17 +185,17 @@ def is_valid_package(package: str) -> bool:
 
 
 # ToDo fix return type
-def _sf_children(child: sfFile):  # -> bytes:
-    path_to_extract = ""
+def _sf_children(child: Any) -> Tuple[bytes, str, str, int]:
+    path_to_extract = b""
     filename_lower = child.filename.lower()
 
     # Skip junk files
     if any(filename_lower.endswith(ext) for ext in JUNK_EXTENSIONS):
-        return (b"", child.platform, child.magic, child.filesize)
+        return b"", child.platform, child.magic, child.filesize
     if any(name in filename_lower for name in JUNK_NAMES):
-        return (b"", child.platform, child.magic, child.filesize)
+        return b"", child.platform, child.magic, child.filesize
     if b".github/" in filename_lower or b".git/" in filename_lower:
-        return (b"", child.platform, child.magic, child.filesize)
+        return b"", child.platform, child.magic, child.filesize
 
     _, ext = os.path.splitext(child.filename)
     ext = ext.lower()
@@ -213,22 +213,29 @@ def _sf_children(child: sfFile):  # -> bytes:
         tmp_dir = tempfile.mkdtemp(dir=target_path)
         try:
             if child.contents:
-                path_to_extract = os.path.join(tmp_dir, sanitize_filename((child.filename).decode()))
+                path_to_extract = os.path.join(tmp_dir, sanitize_filename((child.filename).decode())).encode()
                 _ = path_write_file(path_to_extract, child.contents)
         except Exception as e:
             log.exception(e)
-    return (path_to_extract.encode(), child.platform, child.magic or "", child.filesize)
+    return path_to_extract, child.platform, child.magic or "", child.filesize
 
 
-# ToDo fix typing need to add str as error msg
-def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True):  # -> List[bytes]:
+def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) -> Tuple[List[Tuple[bytes, str, str, int]], str]:
     retlist = []
     # do not extract from .bin (downloaded from us)
     if os.path.splitext(filename)[1] == b".bin":
         return retlist, ""
 
     # ToDo need to introduce error msgs here
     try:
+        platform = ""
+        magic_type = ""
+        file_size = 0
+
+        # Before unpacking, ensure the file actually exists and is not empty to avoid IncorrectUsageException
+        if not path_exists(filename) or os.path.getsize(filename) == 0:
+            return [(filename, platform, magic_type, file_size)], "file not found or empty"
+
         password = options2passwd(options) or "infected"
         try:
             unpacked = unpack(filename, password=password, check_shellcode=check_shellcode)
@@ -240,7 +247,7 @@ def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True):
             magic_type = file.get_type() or ""
             platform = file.get_platform()
             file_size = file.get_size()
-            return [[filename, platform, magic_type, file_size]], ""
+            return [(filename, platform, magic_type, file_size)], ""
         if unpacked.package in blacklist_extensions:
             return [], "blacklisted package"
         for sf_child in unpacked.children:
@@ -267,7 +274,9 @@ def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True):
     return list(filter(None, retlist)), ""
 
 
-def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool = True, platform: str = ""):  # -> tuple[bytes, str]:
+def demux_sample(
+    filename: bytes, package: str, options: str, use_sflock: bool = True, platform: str = ""
+) -> Tuple[List[Tuple[bytes, str]], List[Dict[str, str]]]:
     """
     If file is a ZIP, extract its included files and return their file paths
     If file is an email, extracts its attachments and return their file paths (later we'll also extract URLs)
@@ -311,9 +320,7 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
         return retlist, error_list
 
     # handle quarantine files
-    tmp_path = unquarantine(filename)
-    if tmp_path:
-        filename = tmp_path
+    filename = unquarantine(filename)
 
     # don't try to extract from office docs
     magic = File(filename).get_type() or ""
@@ -407,3 +414,4 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
             new_retlist.append((filename, platform))
 
     return new_retlist[:demux_files_limit], error_list
+

lib/cuckoo/common/gcp.py

@@ -3,11 +3,11 @@
 import logging
 import time
 import shutil
+from typing import Any, Dict, List, Optional, Set
 
 from lib.cuckoo.common.config import Config
 from lib.cuckoo.common.path_utils import path_exists
 from lib.cuckoo.common.constants import CUCKOO_ROOT
-
 try:
     from google.api_core.exceptions import Forbidden
     from google.cloud import compute_v1
@@ -38,7 +38,7 @@ class GCSUploader:
     """Helper class to upload files to GCS."""
 
     @staticmethod
-    def parse_custom_string(custom_str):
+    def parse_custom_string(custom_str: str) -> Dict[str, str]:
         if not custom_str:
             return {}
 
@@ -52,7 +52,15 @@ def parse_custom_string(custom_str):
                 data[key] = value
         return data
 
-    def __init__(self, bucket_name=None, auth_by=None, credentials_path=None, exclude_dirs=None, exclude_files=None, mode=None):
+    def __init__(
+        self,
+        bucket_name: Optional[str] = None,
+        auth_by: Optional[str] = None,
+        credentials_path: Optional[str] = None,
+        exclude_dirs: Optional[Set[str]] = None,
+        exclude_files: Optional[Set[str]] = None,
+        mode: Optional[str] = None,
+    ):
         if not HAVE_GCP:
             raise ImportError("google-cloud-storage library is missing")
 
@@ -89,7 +97,7 @@ def __init__(self, bucket_name=None, auth_by=None, credentials_path=None, exclud
 
         self.bucket = self.storage_client.bucket(bucket_name)
 
-    def _iter_files_to_upload(self, source_directory):
+    def _iter_files_to_upload(self, source_directory: str):
         """Generator that yields files to be uploaded, skipping excluded ones."""
         for root, dirs, files in os.walk(source_directory):
             # Exclude specified directories
@@ -105,13 +113,13 @@ def _iter_files_to_upload(self, source_directory):
                 relative_path = os.path.relpath(local_path, source_directory)
                 yield local_path, relative_path
 
-    def upload(self, source_directory, analysis_id, tlp=None, metadata=None):
+    def upload(self, source_directory: str, analysis_id: int, tlp: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None):
         if self.mode == "zip":
             self.upload_zip_archive(analysis_id, source_directory, tlp=tlp, metadata=metadata)
         else:
             self.upload_files_individually(analysis_id, source_directory, tlp=tlp, metadata=metadata)
 
-    def upload_zip_archive(self, analysis_id, source_directory, tlp=None, metadata=None):
+    def upload_zip_archive(self, analysis_id: int, source_directory: str, tlp: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None):
         log.debug("Compressing and uploading files for analysis ID %s to GCS", analysis_id)
         blob_name = f"{analysis_id}_tlp_{tlp}.zip" if tlp else f"{analysis_id}.zip"
 
@@ -130,7 +138,9 @@ def upload_zip_archive(self, analysis_id, source_directory, tlp=None, metadata=N
             os.unlink(tmp_zip_file_name)
         log.info("Successfully uploaded archive for analysis %s to GCS.", analysis_id)
 
-    def upload_files_individually(self, analysis_id, source_directory, tlp=None, metadata=None):
+    def upload_files_individually(
+        self, analysis_id: int, source_directory: str, tlp: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None
+    ):
         log.debug("Uploading files for analysis ID %s to GCS", analysis_id)
         folder_name = f"{analysis_id}_tlp_{tlp}" if tlp else str(analysis_id)
 
@@ -144,7 +154,7 @@ def upload_files_individually(self, analysis_id, source_directory, tlp=None, met
 
         log.info("Successfully uploaded files for analysis %s to GCS.", analysis_id)
 
-    def check_exists(self, analysis_id):
+    def check_exists(self, analysis_id: int) -> bool:
         """Check if any blobs exist for the given analysis ID."""
         prefix = str(analysis_id)
         blobs = list(self.storage_client.list_blobs(self.bucket, prefix=prefix, max_results=1))
@@ -165,7 +175,7 @@ def check_exists(self, analysis_id):
         GCS_ENABLED = False
 
 
-def download_from_gcs(gcs_uri, destination_path, logger=None, client=None):
+def download_from_gcs(gcs_uri: str, destination_path: str, logger: Optional[Any] = None, client: Optional[storage.Client] = None) -> bool:
     """
     Downloads a file from GCS.
     gcs_uri: gs://bucket_name/object_name
@@ -174,7 +184,9 @@ def download_from_gcs(gcs_uri, destination_path, logger=None, client=None):
         logger = log
 
     if not HAVE_GCP:
-        logger.error("Google Cloud Storage dependencies not installed. Please run `poetry install --extras gcp` or `pip install google-cloud-storage`")
+        logger.error(
+            "Google Cloud Storage dependencies not installed. Please run `poetry install --extras gcp` or `pip install google-cloud-storage`"
+        )
         return False
 
     try:
@@ -220,6 +232,7 @@ def download_from_gcs(gcs_uri, destination_path, logger=None, client=None):
         logger.error("Failed to download from GCS %s: %s", gcs_uri, e)
         return False
 
+
 def check_node_up(host: str) -> bool:
     """Auxiliar function for autodiscovery of instances when cluster autoscale"""
     try:
@@ -252,7 +265,7 @@ def __init__(self) -> None:
             "Authorization": f"Bearer {self.token}",
         }
 
-    def list_instances(self) -> dict:
+    def list_instances(self) -> Dict[str, List[str]]:
         """Auto discovery of new servers"""
         servers = {}
         instance_name_pattern = "cape-server"
@@ -261,7 +274,9 @@ def list_instances(self) -> dict:
         if self.token:
             for zone in self.zones:
                 try:
-                    r = requests.get(f"{self.GCP_BASE_URL}projects/{self.project_id}/zones/{zone}/instances", headers=self.headers)
+                    r = requests.get(
+                        f"{self.GCP_BASE_URL}projects/{self.project_id}/zones/{zone}/instances", headers=self.headers
+                    )
                     for instance in r.json().get("items", []):
                         if not instance["name"].startswith(instance_name_pattern):
                             continue
@@ -329,7 +344,7 @@ def autodiscovery(self):
             time.sleep(autodiscovery_interval)
 
 
-def gcs_replay(task_range):
+def gcs_replay(task_range: str):
     if not GCS_ENABLED:
         log.error("GCS is not enabled in reporting.conf")
         return
@@ -372,15 +387,15 @@ def gcs_replay(task_range):
                     metadata["md5"] = samples[0].sample.md5
                     metadata["sha1"] = samples[0].sample.sha1
 
-                metadata["task_id"] = task_id
+                metadata["task_id"] = str(task_id)
 
                 gcs_upload_report(report_path, task_id, tlp, metadata=metadata)
 
         except Exception as e:
             log.error("Failed to replay GCS upload for task %d: %s", task_id, e)
 
 
-def gcs_upload_report(report_path, analysis_id, tlp=None, metadata=None):
+def gcs_upload_report(report_path: str, analysis_id: int, tlp: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None):
     if not GCS_ENABLED:
         return
 
@@ -399,7 +414,7 @@ def gcs_upload_report(report_path, analysis_id, tlp=None, metadata=None):
         log.error("Failed to upload report to GCS for task %d: %s", analysis_id, e)
 
 
-def gcs_sync(time_range):
+def gcs_sync(time_range: str):
     if not GCS_ENABLED:
         log.error("GCS is not enabled in reporting.conf")
         return
@@ -450,7 +465,7 @@ def gcs_sync(time_range):
     gcs_replay(",".join(map(str, sorted(missing_ids))))
 
 
-def gcs_refetch_banned(time_range, samples_bucket=None):
+def gcs_refetch_banned(time_range: str, samples_bucket: Optional[str] = None):
     if not HAVE_GCP:
         log.error("Google Cloud Storage dependencies not installed.")
         return