Fix MAP-Elites eviction (#454) & sampling (#452) bugs; harden library API; dhara CI + Frame SAST

.github/workflows/python-test.yml

@@ -37,7 +37,7 @@ jobs:
   integration-tests:
     needs: unit-tests  # Only run if unit tests pass
     runs-on: ubuntu-latest
-    timeout-minutes: 30  # Limit integration tests to 30 minutes
+    timeout-minutes: 90  # Full integration suite (incl. real-LLM tests) can be slow
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -59,32 +59,63 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[dev]"
-          pip install optillm
+          # math-verify is imported by optillm's cepo module but is NOT declared as a
+          # dependency of the optillm PyPI package, so `pip install optillm` alone
+          # leaves it missing and the server fails to start.
+          pip install optillm math-verify
+
+      - name: Cache HuggingFace models
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/huggingface
+          key: ${{ runner.os }}-hf-codelion-dhara-250m
+          restore-keys: |
+            ${{ runner.os }}-hf-
 
       - name: Start optillm server
         run: |
           echo "Starting optillm server for integration tests..."
-          OPTILLM_API_KEY=optillm HF_TOKEN=${{ secrets.HF_TOKEN }} optillm --model google/gemma-3-270m-it --port 8000 &
+          optillm --model codelion/dhara-250m --port 8000 > optillm_server.log 2>&1 &
           echo $! > server.pid
-
-          # Wait for server to be ready
-          echo "Waiting for server to start..."
-          sleep 15
-
-          # Test server health
-          curl -s http://localhost:8000/health || echo "Server health check failed"
+
+          # Poll until healthy. On a cold HuggingFace cache the model download can
+          # take a few minutes; fail fast if the server process dies (e.g. a missing
+          # dependency) instead of proceeding to tests with no server.
+          echo "Waiting for optillm server health..."
+          for i in $(seq 1 60); do
+            if curl -sf http://localhost:8000/health >/dev/null 2>&1; then
+              echo "optillm server healthy after ~$((i*10))s"
+              break
+            fi
+            if ! kill -0 "$(cat server.pid)" 2>/dev/null; then
+              echo "::error::optillm server process exited before becoming healthy"
+              cat optillm_server.log
+              exit 1
+            fi
+            sleep 10
+          done
+          if ! curl -sf http://localhost:8000/health >/dev/null 2>&1; then
+            echo "::error::optillm server did not become healthy in time"
+            tail -100 optillm_server.log
+            exit 1
+          fi
         env:
           OPTILLM_API_KEY: optillm
+          # Bound generation: dhara-250m does not reliably emit an EOS token and would
+          # otherwise ramble up to the default 4096 tokens per call (minutes each).
+          OPTILLM_MAX_TOKENS: "256"
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
-      - name: Run integration tests (excluding slow tests)
+      - name: Run integration tests (full suite, including real-LLM tests)
         env:
           OPENAI_API_KEY: optillm
           OPTILLM_API_KEY: optillm
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: |
-          # Run only fast integration tests, skip slow tests that require real LLM
-          pytest tests/integration -v --tb=short -m "not slow"
+          # Run the ENTIRE integration suite against the local dhara model server.
+          # Slow real-LLM tests are intentionally NOT skipped so contributor PRs are
+          # exercised end-to-end (evolution loop, checkpoints, migration, library API).
+          pytest tests/integration -v --tb=short
 
       - name: Stop optillm server
         if: always()

.github/workflows/security-scan.yml

@@ -0,0 +1,72 @@
+name: Security Scan (Frame SAST)
+
+# Runs the Frame neuro-symbolic SAST tool (https://github.com/lambdasec/frame)
+# on the Python files changed by a pull request. Scanning only the PR's changed
+# files surfaces issues introduced by the change without failing on pre-existing
+# findings elsewhere in the tree. The job fails only on high/critical severity.
+#
+# Mirrors the setup used in the optillm repo (no SARIF / code-scanning upload,
+# so no GitHub Advanced Security requirement).
+
+on:
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  frame-scan:
+    name: Frame SAST (changed files)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout (full history for diff)
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install Frame (pinned)
+        run: |
+          git clone https://github.com/lambdasec/frame.git /tmp/frame
+          git -C /tmp/frame checkout 75811925b0984f3d2ae3ab14b946d118e8f80617
+          pip install "/tmp/frame[scan]"
+
+      - name: Scan Python files changed in this PR
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          set -uo pipefail
+
+          # Added/copied/modified/renamed Python files in this PR (skip deletions).
+          mapfile -t FILES < <(git diff --name-only --diff-filter=ACMR "$BASE_SHA" HEAD -- '*.py')
+
+          if [ "${#FILES[@]}" -eq 0 ]; then
+            echo "No Python files changed in this PR - nothing to scan."
+            exit 0
+          fi
+
+          echo "Scanning ${#FILES[@]} changed Python file(s) (fail on high/critical):"
+          printf '  %s\n' "${FILES[@]}"
+
+          FAIL=0
+          for f in "${FILES[@]}"; do
+            # File may have been renamed away or removed in a later commit.
+            [ -f "$f" ] || continue
+            echo "::group::Frame scan $f"
+            if ! frame scan "$f" --fail-on high; then
+              FAIL=1
+              echo "::error file=$f::Frame flagged a high/critical severity issue in $f"
+            fi
+            echo "::endgroup::"
+          done
+
+          if [ "$FAIL" -ne 0 ]; then
+            echo "Frame SAST found high/critical severity issue(s) in changed files."
+            exit 1
+          fi
+          echo "No high/critical severity issues in changed files."

openevolve/_version.py

@@ -1,3 +1,3 @@
 """Version information for openevolve package."""
 
-__version__ = "0.2.27"
+__version__ = "0.3.0"

openevolve/api.py

@@ -234,6 +234,51 @@ def _prepare_program(
     return program_file
 
 
+def _extract_lambda_source(source: str) -> Optional[str]:
+    """Extract a single ``lambda ...`` expression from a source snippet.
+
+    ``inspect.getsource`` on a lambda returns the whole line it appears on, e.g.
+    ``evaluator=lambda p: {"score": 0.8},  # comment``. This isolates just the
+    ``lambda p: {"score": 0.8}`` expression using a bracket/string-aware scan so a
+    trailing comma, comment, or the enclosing call's ``)`` do not leak in.
+
+    Returns the lambda expression string, or None if no lambda is found.
+    """
+    idx = source.find("lambda")
+    if idx == -1:
+        return None
+
+    out = []
+    depth = 0
+    quote = None
+    i = idx
+    while i < len(source):
+        c = source[i]
+        if quote is not None:
+            out.append(c)
+            if c == quote and source[i - 1] != "\\":
+                quote = None
+        elif c in "\"'":
+            quote = c
+            out.append(c)
+        elif c in "([{":
+            depth += 1
+            out.append(c)
+        elif c in ")]}":
+            if depth == 0:
+                break  # closing bracket of the enclosing call -> lambda ended
+            depth -= 1
+            out.append(c)
+        elif depth == 0 and (c == "," or c == "#" or c == "\n"):
+            break  # top-level comma / comment / newline ends the lambda
+        else:
+            out.append(c)
+        i += 1
+
+    expr = "".join(out).strip()
+    return expr or None
+
+
 def _prepare_evaluator(
     evaluator: Union[str, Path, Callable], temp_dir: Optional[str], temp_files: List[str]
 ) -> str:
@@ -256,6 +301,18 @@ def _prepare_evaluator(
             func_source = textwrap.dedent(func_source)
             func_name = evaluator.__name__
 
+            if func_name == "<lambda>":
+                # A lambda has no usable name (referencing it as `<lambda>` is a
+                # syntax error). Extract the lambda expression from the source and
+                # bind it to a real name so the generated module is self-contained
+                # and works in subprocess workers.
+                lambda_src = _extract_lambda_source(func_source)
+                if lambda_src is None:
+                    # Couldn't isolate the expression; fall back to the globals path
+                    raise TypeError("cannot serialize lambda source")
+                func_name = "_user_evaluator"
+                func_source = f"{func_name} = {lambda_src}"
+
             # Build a self-contained evaluator module with the function source
             # and an evaluate() entry point that calls it
             evaluator_code = f"""

openevolve/controller.py

@@ -100,9 +100,13 @@ def __init__(
             random.seed(self.config.random_seed)
             np.random.seed(self.config.random_seed)
 
-            # Create hash-based seeds for different components
+            # Create hash-based seeds for different components. md5 is used only to
+            # derive a deterministic RNG seed from the configured seed, not for any
+            # security purpose; usedforsecurity=False documents that intent.
             base_seed = str(self.config.random_seed).encode("utf-8")
-            llm_seed = int(hashlib.md5(base_seed + b"llm").hexdigest()[:8], 16) % (2**31)
+            llm_seed = int(
+                hashlib.md5(base_seed + b"llm", usedforsecurity=False).hexdigest()[:8], 16
+            ) % (2**31)
 
             # Propagate seed to LLM configurations
             self.config.llm.random_seed = llm_seed
@@ -225,7 +229,7 @@ def _setup_manual_mode_queue(self) -> None:
         if not bool(getattr(self.config.llm, "manual_mode", False)):
             return
 
-        qdir = (Path(self.output_dir).expanduser().resolve() / "manual_tasks_queue")
+        qdir = Path(self.output_dir).expanduser().resolve() / "manual_tasks_queue"
 
         # Clear stale tasks from previous runs
         if qdir.exists():
@@ -303,6 +307,12 @@ async def run(
 
             self.database.add(initial_program)
 
+            # Check for and store artifacts from initial program
+            initial_artifacts = self.evaluator.get_pending_artifacts(initial_program_id)
+            if initial_artifacts:
+                self.database.store_artifacts(initial_program_id, initial_artifacts)
+                logger.info(f"Stored artifacts for initial program")
+
             # Check if combined_score is present in the metrics
             if "combined_score" not in initial_metrics:
                 # Calculate average of numeric metrics