diff --git a/.github/workflows/daily_trading_bot.yml b/.github/workflows/daily_trading_bot.yml index d465a74..e772d0c 100644 --- a/.github/workflows/daily_trading_bot.yml +++ b/.github/workflows/daily_trading_bot.yml @@ -6,8 +6,8 @@ concurrency: on: schedule: - # Runs at 00:00 UTC (8:00 AM SGT) Tuesday-Sunday - - cron: '0 0 * * 2-7' + # Runs at 00:00 UTC (8:00 AM SGT) Tuesday-Saturday + - cron: '0 0 * * 2-6' workflow_dispatch: inputs: run_in_lightning_studio: @@ -74,6 +74,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Enforce schedule day guard (Tue-Sat only) + if: ${{ github.event_name == 'schedule' }} + run: | + dow=$(date -u +%u) + # ISO day-of-week: 1=Mon ... 7=Sun. Allow Tue(2) through Sat(6) only. + if [ "${dow}" -lt 2 ] || [ "${dow}" -gt 6 ]; then + echo "Scheduled run blocked: UTC weekday ${dow} is outside Tue-Sat window." >&2 + exit 1 + fi + - name: Set up Python uses: actions/setup-python@v5 with: @@ -196,15 +206,35 @@ jobs: - name: Install Lightning dependencies id: install_lightning_deps - if: ${{ steps.install_base_dependencies.outcome == 'success' && ((github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) || (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && ((github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) || ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading)) }} continue-on-error: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) }} timeout-minutes: 20 run: | python -m pip install "lightning[app]==2.3.2" lightning-cloud==0.5.70 + - name: Validate SMTP configuration for AI reports + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} + env: + SMTP_SERVER: ${{ secrets.SMTP_SERVER }} + SMTP_PORT: ${{ secrets.SMTP_PORT }} + SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} + SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} + RECIPIENT_EMAIL: ${{ secrets.RECIPIENT_EMAIL }} + run: | + missing="" + for name in SMTP_SERVER SMTP_PORT SENDER_EMAIL SENDER_PASSWORD RECIPIENT_EMAIL; do + if [ -z "${!name}" ]; then + missing="${missing} ${name}" + fi + done + if [ -n "${missing}" ]; then + echo "Missing required SMTP env vars for AI report:${missing}" >&2 + exit 1 + fi + - name: Plan AI runtime id: plan_ai_runtime - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} run: | mkdir -p results configured_inference_url="${CEREBRIUM_TRAINED_MODEL_URL}" @@ -239,7 +269,7 @@ jobs: - name: Validate Cerebrium primary configuration id: validate_cerebrium_primary_config - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} run: | runtime_mode="${{ steps.plan_ai_runtime.outputs.runtime_mode }}" selected_backend="${{ steps.plan_ai_runtime.outputs.selected_backend }}" @@ -276,7 +306,7 @@ jobs: - name: Enforce AI routing invariants id: enforce_ai_routing_invariants - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} run: | runtime_mode="${{ steps.plan_ai_runtime.outputs.runtime_mode }}" has_inference_url="${{ steps.validate_cerebrium_primary_config.outputs.has_inference_url }}" @@ -294,7 +324,7 @@ jobs: fi - name: Emit AI runtime decision - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} run: | echo "AI runtime mode: ${{ steps.plan_ai_runtime.outputs.runtime_mode }}" echo "AI backend: ${{ steps.plan_ai_runtime.outputs.selected_backend }}" @@ -303,7 +333,7 @@ jobs: - name: Warm Cerebrium inference app id: warm_cerebrium_inference - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' }} continue-on-error: true timeout-minutes: 45 env: @@ -367,7 +397,7 @@ jobs: - name: Verify Cerebrium predict endpoint id: verify_cerebrium_predict - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' }} continue-on-error: true timeout-minutes: 3 env: @@ -428,7 +458,7 @@ jobs: - name: Run AI Trading Bot on Cerebrium id: run_ai_bot_cerebrium - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && steps.verify_cerebrium_predict.outcome == 'success' }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && steps.verify_cerebrium_predict.outcome == 'success' }} continue-on-error: true timeout-minutes: 90 env: @@ -459,7 +489,7 @@ jobs: - name: Run AI Trading Bot on Cerebrium (Retry) id: run_ai_bot_cerebrium_retry - if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && steps.verify_cerebrium_predict.outcome == 'success' && steps.run_ai_bot_cerebrium.outcome == 'failure' }} + if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && steps.verify_cerebrium_predict.outcome == 'success' && steps.run_ai_bot_cerebrium.outcome == 'failure' }} continue-on-error: true timeout-minutes: 90 env: @@ -490,7 +520,7 @@ jobs: - name: Launch Lightning inference studio id: launch_lightning_inference - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && steps.install_lightning_deps.outcome == 'success' }} + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && steps.install_lightning_deps.outcome == 'success' }} continue-on-error: true timeout-minutes: 60 env: @@ -519,7 +549,7 @@ jobs: - name: Run AI Trading Bot in Lightning Studio id: run_ai_bot_in_lightning_studio - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && steps.install_lightning_deps.outcome == 'success' && steps.launch_lightning_inference.outcome == 'success' }} + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && steps.install_lightning_deps.outcome == 'success' && steps.launch_lightning_inference.outcome == 'success' }} continue-on-error: true timeout-minutes: 90 env: @@ -553,7 +583,7 @@ jobs: - name: Run AI Trading Bot (Distilled Local Fallback) id: run_ai_bot_distilled_local - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && (steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local' || (steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success'))) || (steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure'))) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && (steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local' || (steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success'))) || (steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure'))) }} continue-on-error: true timeout-minutes: 60 env: @@ -575,7 +605,7 @@ jobs: - name: Run AI Trading Bot (Emergency Distilled Retry) id: run_ai_bot_distilled_emergency_retry - if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.run_ai_bot_distilled_local.outcome == 'failure' }} + if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.run_ai_bot_distilled_local.outcome == 'failure' }} continue-on-error: true timeout-minutes: 60 env: @@ -599,7 +629,7 @@ jobs: - name: Send AI Failure Report id: send_ai_failure_report - if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && (((steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true') && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success'))) || ((steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full') && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success') || ((steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success')) }} + if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && (((steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true') && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success'))) || ((steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full') && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success') || ((steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success')) }} continue-on-error: true env: SMTP_SERVER: ${{ secrets.SMTP_SERVER }} @@ -712,7 +742,7 @@ jobs: RECIPIENT_EMAIL: ${{ secrets.RECIPIENT_EMAIL }} RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} CORE_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_core_trading) }} - AI_EXPECTED: "false" + AI_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} BASE_DEPS_OUTCOME: ${{ steps.install_base_dependencies.outcome }} CORE_OUTCOME: ${{ steps.run_core_bot.outcome }} CORE_RETRY_OUTCOME: ${{ steps.run_core_bot_retry.outcome }} @@ -772,7 +802,7 @@ jobs: CORE_OUTCOME: ${{ steps.run_core_bot.outcome }} CORE_RETRY_OUTCOME: ${{ steps.run_core_bot_retry.outcome }} CORE_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_core_trading) }} - AI_EXPECTED: "false" + AI_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} INSTALL_LIGHTNING_OUTCOME: ${{ steps.install_lightning_deps.outcome }} AI_RUNTIME_MODE: ${{ steps.plan_ai_runtime.outputs.runtime_mode }} LAUNCH_LIGHTNING_OUTCOME: ${{ steps.launch_lightning_inference.outcome }} diff --git a/main.py b/main.py index cc2ee52..650674e 100644 --- a/main.py +++ b/main.py @@ -49,6 +49,28 @@ def _resolve_path(base_dir, path_value): return os.path.join(base_dir, path_value) +def _position_cash_metrics(open_positions, current_capital): + """Return gross exposure and non-negative cash using side-aware notional. + + Gross exposure includes longs and shorts for reporting. Available cash only + subtracts long notional; short exposure is risk exposure, not cash spent. + """ + if open_positions is None or not hasattr(open_positions, "empty") or open_positions.empty: + return 0.0, max(0.0, float(current_capital or 0.0)) + + df = open_positions.copy() + price_col = "current_price" if "current_price" in df.columns else "entry_price" + prices = pd.to_numeric(df.get(price_col), errors="coerce").fillna(0.0).clip(lower=0.0) + quantities = pd.to_numeric(df.get("quantity"), errors="coerce").fillna(0.0).abs() + notional = prices * quantities + sides = df.get("side", pd.Series(["LONG"] * len(df), index=df.index)).fillna("LONG").astype(str).str.upper() + + gross_exposure = float(notional.sum() or 0.0) + long_notional = float(notional[sides != "SHORT"].sum() or 0.0) + available_cash = max(0.0, float(current_capital or 0.0) - long_notional) + return gross_exposure, available_cash + + def _get_open_position_symbols(config_path, table_names=("positions", "positions_ai")): """Return distinct OPEN symbols across the requested position tables.""" config = _load_config(config_path) @@ -824,12 +846,10 @@ def run_daily_test(self, test_date=None, pipeline_stats=None, backtest_signals=N total_account_return = ((total_realized_dollars + total_unrealized_dollars) / initial_capital) if initial_capital else 0.0 current_capital = initial_capital + total_realized_dollars - # Available cash (notional-based; shorts consume capital too) + # Report gross exposure while keeping cash side-aware. Shorts add exposure + # but do not spend cash like long purchases, so they must not drive cash negative. open_positions_now = self.core_tracker.get_open_positions() - invested_notional = 0.0 - if not open_positions_now.empty: - invested_notional = float((open_positions_now["entry_price"] * open_positions_now["quantity"]).sum() or 0.0) - available_cash = float(current_capital) - invested_notional + invested_notional, available_cash = _position_cash_metrics(open_positions_now, current_capital) period_start_core = self.core_tracker.get_performance_period_start() report = { @@ -1420,12 +1440,7 @@ def add_symbol(value): ai_realized_today = (ai_realized_today_dollars / ai_capital_pre_actions) if ai_capital_pre_actions else 0.0 ai_current_capital = ai_initial_capital + ai_realized_total_dollars + ai_unreal_total_dollars - ai_invested_notional = 0.0 - if ai_unrealized is not None and hasattr(ai_unrealized, "empty") and not ai_unrealized.empty: - price_series = pd.to_numeric(ai_unrealized.get("current_price"), errors="coerce").fillna(0.0) - quantity_series = pd.to_numeric(ai_unrealized.get("quantity"), errors="coerce").fillna(0.0) - ai_invested_notional = float((price_series * quantity_series).sum() or 0.0) - ai_available_cash = float(ai_current_capital) - ai_invested_notional + ai_invested_notional, ai_available_cash = _position_cash_metrics(ai_unrealized, ai_current_capital) if isinstance(ai_llm_status, dict): ai_llm_status["target_positions"] = len(ai_trades) diff --git a/run_ai_daily_cerebrium.py b/run_ai_daily_cerebrium.py index 4aef305..d66ebda 100644 --- a/run_ai_daily_cerebrium.py +++ b/run_ai_daily_cerebrium.py @@ -4,6 +4,7 @@ import os import subprocess import sys +import time from pathlib import Path import requests @@ -41,13 +42,26 @@ def _preflight_predict(url: str, api_key: str) -> tuple[bool, str]: if api_key: headers["Authorization"] = f"Bearer {api_key}" payload = {"candidates": [{"symbol": "AAPL", "return_5d": 0.01, "news_sentiment_7d": 0.0}]} - try: - resp = requests.post(url, headers=headers, json=payload, timeout=45) - if 200 <= resp.status_code < 300: - return True, f"status={resp.status_code}" - return False, f"status={resp.status_code}" - except Exception as exc: - return False, str(exc) + + attempts = 3 + transient_statuses = {429, 500, 502, 503, 504} + details: list[str] = [] + + for attempt in range(1, attempts + 1): + try: + resp = requests.post(url, headers=headers, json=payload, timeout=45) + details.append(f"attempt={attempt}:status={resp.status_code}") + if 200 <= resp.status_code < 300: + return True, "; ".join(details) + if resp.status_code not in transient_statuses: + return False, "; ".join(details) + except requests.RequestException as exc: + details.append(f"attempt={attempt}:exception={exc}") + + if attempt < attempts: + time.sleep(min(8, 2 * attempt)) + + return False, "; ".join(details) def main() -> None: @@ -56,7 +70,6 @@ def main() -> None: base_env["AI_RUNTIME_MODE"] = "full" base_env["AI_PRIMARY_BACKEND"] = "cerebrium" base_env["AI_ROUTER_REASON"] = "cerebrium_primary_direct" - base_env["ALLOW_MISSING_EMAIL"] = "1" resolved_url = _first_non_empty( base_env.get("CEREBRIUM_TRAINED_MODEL_URL"),