work in progress, docker, nextflow

.dockerignore

@@ -0,0 +1,10 @@
+.git/
+__pycache__/
+*.pyc
+*.pyo
+# Generated outputs / intermediates — recreated at runtime, no need to bake in.
+data/
+results/
+work/
+.nextflow*
+*.log

.gitignore

@@ -18,3 +18,10 @@ data/annotated_report.json
 data/enrichment_cache.json
 data/*.log
 res.txt
+
+# Nextflow run artifacts
+.nextflow/
+.nextflow.log*
+data/nextflow_logs/.nextflow.log*
+work/
+results/

Dockerfile

@@ -0,0 +1,36 @@
+# Ollama runtime + the llmize Python pipeline in one image.
+FROM ollama/ollama:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends python3 python3-pip curl \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/llmize
+
+ARG INSTALL_ENRICH=false
+RUN python3 -m pip install --no-cache-dir --break-system-packages "ollama>=0.5" \
+ && if [ "$INSTALL_ENRICH" = "true" ]; then \
+        python3 -m pip install --no-cache-dir --break-system-packages tooluniverse "PyYAML>=6" ; \
+    fi
+
+# TEST_MODEL is baked in for offline runs; LLMIZE_MODEL is the runtime default.
+# Set BAKE_MODEL=true to also bake LLMIZE_MODEL for air-gapped use.
+ARG TEST_MODEL=smollm2:135m
+ARG LLMIZE_MODEL=gemma4
+ENV LLMIZE_MODEL=${LLMIZE_MODEL}
+ARG BAKE_MODEL=false
+RUN if [ -n "$TEST_MODEL" ] || [ "$BAKE_MODEL" = "true" ]; then \
+        ollama serve & srv=$! ; \
+        until curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; do sleep 1; done ; \
+        if [ -n "$TEST_MODEL" ]; then ollama pull "$TEST_MODEL" ; fi ; \
+        if [ "$BAKE_MODEL" = "true" ]; then ollama pull "$LLMIZE_MODEL" ; fi ; \
+        kill $srv ; \
+    fi
+
+COPY . .
+RUN mkdir -p /opt/llmize/data && chmod -R a+rwX /opt/llmize/data \
+ && chmod +x docker/boot_ollama.sh docker/entrypoint.sh
+
+ENTRYPOINT ["/opt/llmize/docker/entrypoint.sh"]
+CMD ["python3", "pipeline.py", "--check"]

docker/boot_ollama.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Start the in-container Ollama server, wait for it, and pull the model.
+#
+# Reusable so both the image ENTRYPOINT (docker run) and the Nextflow process
+# script can call it. The server is started with nohup + disown so it survives
+# this script exiting and is reachable by later commands in the same task.
+#
+# Honours:
+#   LLMIZE_MODEL  - model tag to pull/run (default: gemma4)
+#   OLLAMA_HOST   - server/client endpoint (default: 127.0.0.1:11434)
+set -euo pipefail
+
+MODEL="${LLMIZE_MODEL:-gemma4}"
+ENDPOINT="http://${OLLAMA_HOST:-127.0.0.1:11434}"
+
+# Start the server only if one isn't already answering.
+if ! curl -sf "${ENDPOINT}/api/tags" >/dev/null 2>&1; then
+    echo "[boot] starting 'ollama serve'..."
+    nohup ollama serve >/tmp/ollama.log 2>&1 &
+    disown || true
+fi
+
+echo "[boot] waiting for Ollama at ${ENDPOINT} ..."
+for i in $(seq 1 60); do
+    if curl -sf "${ENDPOINT}/api/tags" >/dev/null 2>&1; then
+        break
+    fi
+    sleep 1
+    if [ "$i" -eq 60 ]; then
+        echo "[boot] ERROR: Ollama did not become ready in 60s" >&2
+        cat /tmp/ollama.log >&2 || true
+        exit 1
+    fi
+done
+
+# Skip the pull if the model is already present (baked into the image, or cached
+# on a mounted volume). This is what makes an offline / air-gapped image work:
+# 'ollama pull' would otherwise contact the registry even for an existing model.
+if ollama list 2>/dev/null | grep -qF "${MODEL}"; then
+    echo "[boot] model '${MODEL}' already present; skipping pull (offline-safe)."
+else
+    echo "[boot] pulling model '${MODEL}' (runtime pull; needs network)..."
+    ollama pull "${MODEL}"
+fi
+echo "[boot] ready."

docker/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Image ENTRYPOINT for plain `docker run`: boot Ollama, then run the given command.
+# (Nextflow overrides the entrypoint, so its process script calls boot_ollama.sh itself.)
+set -euo pipefail
+
+source /opt/llmize/docker/boot_ollama.sh
+
+# Default to the env preflight if no command was given.
+if [ "$#" -eq 0 ]; then
+    exec python3 /opt/llmize/pipeline.py --check
+fi
+exec "$@"

interpret.py

@@ -6,10 +6,13 @@
 import sys
 import statistics
 import subprocess
+import time
 from datetime import datetime
 
 import ollama
 
+from verify import deterministic_findings
+
 
 # Prompt builder
 
@@ -376,6 +379,59 @@ def combine_responses(responses: list, summary: str = None) -> str:
     return "\n".join(parts)
 
 
+REVIEW_SYSTEM_PROMPT = """You are an expert bioinformatician reviewing a spatial transcriptomics interpretation for factual grounding and internal consistency. Correct statements that are not supported by the underlying report data, remove claims about genes, proteins, or cell types that do not appear in the data, and resolve contradictions between sections. Preserve the document's headings, tables, and structure. Do not add new findings and do not soften the removal of unsupported claims. Output only the corrected document, with no preamble.""" + STYLE_GUIDE + EVIDENCE_RULES
+
+
+def build_review_prompt(text: str, findings: list) -> str:
+    parts = [
+        "Review the spatial transcriptomics interpretation below. Correct any statement "
+        "that is not supported by the report data, and resolve any internal contradiction "
+        "between sections. Do not introduce new claims, and do not invent gene or protein "
+        "functions."
+    ]
+    if findings:
+        max_listed = 50
+        joined = ", ".join(findings[:max_listed])
+        more = len(findings) - max_listed
+        suffix = f" (and {more} more)" if more > 0 else ""
+        parts.append(
+            "These gene or cell-type symbols appear in the text but are absent from the "
+            f"report data. Remove them or correct the surrounding claim: {joined}{suffix}."
+        )
+    parts.append("Return only the corrected interpretation, preserving its markdown structure.")
+    parts.append("\n---\n\n" + text)
+    return "\n\n".join(parts)
+
+
+def review_interpretation(text, model, report, num_ctx=32768, passes=2,
+                          think=True, gen_options=None):
+    for i in range(max(1, passes)):
+        findings = deterministic_findings(text, report)
+        if i > 0 and not findings:
+            break
+        if findings:
+            print(f"[review] pass {i + 1}: {len(findings)} unsupported symbol(s): {', '.join(findings)}")
+        else:
+            print(f"[review] pass {i + 1}: no unsupported symbols detected; evidence and consistency review")
+        from enrich import extract_entities
+        entities = extract_entities(report, max_genes=50)
+        review_prompt = (
+            "Deterministic entities extracted from the report (use these as the allowed symbol set):\n"
+            f"```json\n{json.dumps(entities, indent=2)}\n```\n\n"
+            + build_review_prompt(text, findings)
+        )
+        text, thinking = chat_ollama(
+            review_prompt,
+            model=model,
+            system=REVIEW_SYSTEM_PROMPT,
+            num_ctx=num_ctx,
+            think=think,
+            gen_options=gen_options,
+        )
+        print_thinking(f"Review pass {i + 1}", thinking)
+    return text
+
+
 def print_thinking(label: str, thinking: str) -> None:
     """Print the model's chain-of-thought to the terminal (it is not saved anywhere)."""
     if not thinking:
@@ -407,23 +463,29 @@ def interpret_per_section(
 
     responses = []
     sections = list(iter_analysis_sections(report))
-    print(f"[interpret] Analyzing {len(sections)} sections one-by-one with model '{model}'"
-          f"{' (thinking)' if think else ''}.")
-    for name, section_obj in sections:
-        print(f"[interpret]   -> {name}")
+    total = len(sections)
+    print(f"[interpret] Analyzing {total} sections one-by-one with model '{model}'"
+          f"{' (thinking)' if think else ''}.", flush=True)
+    for idx, (name, section_obj) in enumerate(sections, 1):
+        print(f"[interpret]   [{idx}/{total}] -> {name}", flush=True)
+        started = time.monotonic()
         prompt = build_section_prompt(name, section_obj, groups=groups)
         content, thinking = chat_ollama(
             prompt, model=model, system=system, num_ctx=num_ctx, think=think, gen_options=gen_options
         )
+        elapsed = time.monotonic() - started
+        print(f"[interpret]   [{idx}/{total}] {name} done in {elapsed:.1f}s", flush=True)
         print_thinking(name, thinking)
         responses.append((name, content))
 
     summary = None
     if synthesize_final and responses:
-        print("[interpret] Synthesizing executive summary from per-section analyses...")
+        print("[interpret] Synthesizing executive summary from per-section analyses...", flush=True)
+        started = time.monotonic()
         summary, summary_thinking = synthesize_sections(
             responses, model, num_ctx, samplesheet_context, think=think, gen_options=gen_options
         )
+        print(f"[interpret] Synthesis done in {time.monotonic() - started:.1f}s", flush=True)
         print_thinking("Overview (synthesis)", summary_thinking)
 
     return combine_responses(responses, summary)

main.nf

@@ -0,0 +1,47 @@
+process INTERPRET {
+    tag "${report.baseName}"
+    container 'llmize:latest'
+    publishDir params.outdir, mode: 'copy'
+
+    input:
+    path report
+
+    output:
+    path "*_interpretation_*.md", emit: interpretation
+
+    script:
+    def home = workflow.containerEngine ? '/opt/llmize' : "${projectDir}"
+    def boot = workflow.containerEngine ? "export LLMIZE_MODEL='${params.model}'\n    bash ${home}/docker/boot_ollama.sh" : ''
+    def think_flag   = "${params.think}".toBoolean()        ? '--think' : '--no-think'
+    def enrich_flag  = "${params.enrich}".toBoolean()       ? '--enrich' : ''
+    def review_flag  = "${params.review}".toBoolean()       ? "--review --review-passes ${params.review_passes}" : ''
+    def whole_flag   = "${params.whole_report}".toBoolean() ? '--whole-report' : ''
+    def synth_flag   = "${params.synthesis}".toBoolean()    ? '' : '--no-synthesis'
+    def prompt_escaped = params.prompt ? params.prompt.toString().replace("'", "'\"'\"'") : ''
+    def prompt_flag  = params.prompt ? "--prompt '${prompt_escaped}'" : ''
+    def temp_flag    = params.temperature != null ? "--temperature ${params.temperature}" : ''
+    def seed_flag    = params.seed        != null ? "--seed ${params.seed}" : ''
+    def top_p_flag   = params.top_p       != null ? "--top_p ${params.top_p}" : ''
+    def top_k_flag   = params.top_k       != null ? "--top_k ${params.top_k}" : ''
+    def numpred_flag = params.num_predict != null ? "--num_predict ${params.num_predict}" : ''
+    """
+    ${boot}
+    STAMP=\$(date +%Y%m%d_%H%M%S)
+    python3 ${home}/pipeline.py \\
+        --input '${report}' \\
+        --model '${params.model}' \\
+        --num_ctx ${params.num_ctx} \\
+        ${think_flag} ${enrich_flag} ${review_flag} ${whole_flag} ${synth_flag} \\
+        ${prompt_flag} ${temp_flag} ${seed_flag} ${top_p_flag} ${top_k_flag} ${numpred_flag} \\
+        --output "${report.baseName}_interpretation_\${STAMP}.md"
+    """
+}
+
+workflow {
+    if( !params.input )
+        error "Provide --input <multiqc_data.json>"
+
+    ch_input = channel.fromPath(params.input, checkIfExists: true)
+
+    INTERPRET(ch_input)
+}

nextflow.config

@@ -0,0 +1,48 @@
+params {
+    input        = null
+    outdir       = 'results'
+
+    model        = 'gemma4'
+    think        = true
+    enrich       = false
+    review       = false
+    review_passes = 2
+    whole_report = false
+    synthesis    = true
+    prompt       = null
+    num_ctx      = 32768
+    temperature  = null
+    seed         = null
+    top_p        = null
+    top_k        = null
+    num_predict  = null
+}
+
+docker {
+    enabled = true
+}
+
+profiles {
+    native {
+        docker.enabled = false
+    }
+}
+
+report {
+    enabled   = true
+    file      = "data/nextflow_logs/report.html"
+    overwrite = true
+}
+
+timeline {
+    enabled   = true
+    file      = "data/nextflow_logs/timeline.html"
+    overwrite = true
+}
+
+process {
+    withName: INTERPRET {
+        container = 'llmize:latest'
+        containerOptions = "-v ${System.getProperty('user.home')}/.llmize-ollama:/root/.ollama"
+    }
+}

pipeline.py

@@ -24,6 +24,7 @@
     build_user_instruction,
     build_glossary_context,
     build_run_footer,
+    review_interpretation,
     SYSTEM_PROMPT,
 )
 
@@ -41,8 +42,10 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--model", "-m",
-        default="gemma4",
-        help="Ollama model name to use (default: gemma4).",
+        default=os.environ.get("LLMIZE_MODEL", "gemma4"),
+        help="Ollama model name to use. Defaults to $LLMIZE_MODEL, else gemma4. "
+             "In the Docker image, boot pulls the same $LLMIZE_MODEL, so setting the env "
+             "var alone keeps boot and the pipeline in sync.",
     )
     parser.add_argument(
         "--descriptor",
@@ -85,6 +88,18 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Skip the final executive-summary synthesis pass (section-by-section mode only).",
     )
+    parser.add_argument(
+        "--review",
+        action="store_true",
+        help="Run a capped self-review of the interpretation, driven by a deterministic "
+             "check for gene/cell-type symbols that do not appear in the report data.",
+    )
+    parser.add_argument(
+        "--review-passes",
+        type=int,
+        default=2,
+        help="Maximum number of review passes when --review is set (default: 2).",
+    )
     parser.add_argument(
         "--think",
         action=argparse.BooleanOptionalAction,
@@ -158,6 +173,8 @@ def run_pipeline(
     think: bool = True,
     gen_options: dict = None,
     user_instruction: str = "",
+    review: bool = False,
+    review_passes: int = 2,
 ) -> str:
     input_path = resolve_input_path(input_json)
     print(f"[pipeline] Loading raw JSON: {input_path}")
@@ -210,6 +227,13 @@ def run_pipeline(
             user_instruction=user_instruction,
         )
 
+    if review:
+        print(f"[pipeline] Reviewing interpretation (up to {review_passes} pass(es))...")
+        response = review_interpretation(
+            response, model=model, report=report, num_ctx=num_ctx,
+            passes=review_passes, think=think, gen_options=gen_options,
+        )
+
     if output_path is None:
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         stem = os.path.splitext(os.path.basename(input_path))[0]
@@ -250,6 +274,8 @@ def main() -> None:
         think=args.think,
         gen_options=gen_options,
         user_instruction=args.prompt,
+        review=args.review,
+        review_passes=args.review_passes,
     )
     print(f"[pipeline] Completed. Final report: {final_path}")