diff --git a/.github/workflows/daily_trading_bot.yml b/.github/workflows/daily_trading_bot.yml index d465a74..37549be 100644 --- a/.github/workflows/daily_trading_bot.yml +++ b/.github/workflows/daily_trading_bot.yml @@ -6,8 +6,8 @@ concurrency: on: schedule: - # Runs at 00:00 UTC (8:00 AM SGT) Tuesday-Sunday - - cron: '0 0 * * 2-7' + # Runs at 00:00 UTC (8:00 AM SGT) Tuesday-Saturday + - cron: '0 0 * * 2-6' workflow_dispatch: inputs: run_in_lightning_studio: @@ -74,6 +74,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Enforce schedule day guard (Tue-Sat only) + if: ${{ github.event_name == 'schedule' }} + run: | + dow=$(date -u +%u) + # ISO day-of-week: 1=Mon ... 7=Sun. Allow Tue(2) through Sat(6) only. + if [ "${dow}" -lt 2 ] || [ "${dow}" -gt 6 ]; then + echo "Scheduled run blocked: UTC weekday ${dow} is outside Tue-Sat window." >&2 + exit 1 + fi + - name: Set up Python uses: actions/setup-python@v5 with: @@ -159,22 +169,6 @@ jobs: run: | python main.py daily_job - - name: Send Core Failure Report - if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && steps.run_core_bot.outcome == 'failure' && steps.run_core_bot_retry.outcome != 'success' }} - continue-on-error: true - env: - SMTP_SERVER: ${{ secrets.SMTP_SERVER }} - SMTP_PORT: ${{ secrets.SMTP_PORT }} - SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} - SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} - RECIPIENT_EMAIL: ${{ secrets.RECIPIENT_EMAIL }} - RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} - run: | - python send_failure_report.py \ - --strategy-tag Core \ - --source "Run Trading Bot" \ - --message "Core daily job failed before sending its report email. Run: ${RUN_URL}" - - name: Run Trading Bot (Core Retry) id: run_core_bot_retry if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && !inputs.disable_core_trading && steps.run_core_bot.outcome == 'failure' }} @@ -194,17 +188,54 @@ jobs: run: | python main.py daily_job + - name: Send Core Failure Report + if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && steps.run_core_bot.outcome == 'failure' && steps.run_core_bot_retry.outcome != 'success' }} + continue-on-error: true + env: + SMTP_SERVER: ${{ secrets.SMTP_SERVER }} + SMTP_PORT: ${{ secrets.SMTP_PORT }} + SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} + SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} + RECIPIENT_EMAIL: ${{ secrets.RECIPIENT_EMAIL }} + RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} + run: | + python send_failure_report.py \ + --strategy-tag Core \ + --source "Run Trading Bot" \ + --message "Core daily job failed after retry and did not confirm its report email. Run: ${RUN_URL}" + + - name: Install Lightning dependencies id: install_lightning_deps - if: ${{ steps.install_base_dependencies.outcome == 'success' && ((github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) || (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading)) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && ((github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) || ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading)) }} continue-on-error: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) }} timeout-minutes: 20 run: | python -m pip install "lightning[app]==2.3.2" lightning-cloud==0.5.70 + - name: Validate SMTP configuration for AI reports + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} + env: + SMTP_SERVER: ${{ secrets.SMTP_SERVER }} + SMTP_PORT: ${{ secrets.SMTP_PORT }} + SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} + SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} + RECIPIENT_EMAIL: ${{ secrets.RECIPIENT_EMAIL }} + run: | + missing="" + for name in SMTP_SERVER SMTP_PORT SENDER_EMAIL SENDER_PASSWORD RECIPIENT_EMAIL; do + if [ -z "${!name}" ]; then + missing="${missing} ${name}" + fi + done + if [ -n "${missing}" ]; then + echo "Missing required SMTP env vars for AI report:${missing}" >&2 + exit 1 + fi + - name: Plan AI runtime id: plan_ai_runtime - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} run: | mkdir -p results configured_inference_url="${CEREBRIUM_TRAINED_MODEL_URL}" @@ -239,7 +270,7 @@ jobs: - name: Validate Cerebrium primary configuration id: validate_cerebrium_primary_config - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} run: | runtime_mode="${{ steps.plan_ai_runtime.outputs.runtime_mode }}" selected_backend="${{ steps.plan_ai_runtime.outputs.selected_backend }}" @@ -276,7 +307,7 @@ jobs: - name: Enforce AI routing invariants id: enforce_ai_routing_invariants - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} run: | runtime_mode="${{ steps.plan_ai_runtime.outputs.runtime_mode }}" has_inference_url="${{ steps.validate_cerebrium_primary_config.outputs.has_inference_url }}" @@ -294,7 +325,7 @@ jobs: fi - name: Emit AI runtime decision - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} run: | echo "AI runtime mode: ${{ steps.plan_ai_runtime.outputs.runtime_mode }}" echo "AI backend: ${{ steps.plan_ai_runtime.outputs.selected_backend }}" @@ -303,7 +334,7 @@ jobs: - name: Warm Cerebrium inference app id: warm_cerebrium_inference - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' }} continue-on-error: true timeout-minutes: 45 env: @@ -367,7 +398,7 @@ jobs: - name: Verify Cerebrium predict endpoint id: verify_cerebrium_predict - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' }} continue-on-error: true timeout-minutes: 3 env: @@ -428,7 +459,7 @@ jobs: - name: Run AI Trading Bot on Cerebrium id: run_ai_bot_cerebrium - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && steps.verify_cerebrium_predict.outcome == 'success' }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && steps.verify_cerebrium_predict.outcome == 'success' }} continue-on-error: true timeout-minutes: 90 env: @@ -459,7 +490,7 @@ jobs: - name: Run AI Trading Bot on Cerebrium (Retry) id: run_ai_bot_cerebrium_retry - if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && steps.verify_cerebrium_predict.outcome == 'success' && steps.run_ai_bot_cerebrium.outcome == 'failure' }} + if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.enforce_ai_routing_invariants.outcome == 'success' && steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && steps.verify_cerebrium_predict.outcome == 'success' && steps.run_ai_bot_cerebrium.outcome == 'failure' }} continue-on-error: true timeout-minutes: 90 env: @@ -490,7 +521,7 @@ jobs: - name: Launch Lightning inference studio id: launch_lightning_inference - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && steps.install_lightning_deps.outcome == 'success' }} + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && steps.install_lightning_deps.outcome == 'success' }} continue-on-error: true timeout-minutes: 60 env: @@ -519,7 +550,7 @@ jobs: - name: Run AI Trading Bot in Lightning Studio id: run_ai_bot_in_lightning_studio - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && steps.install_lightning_deps.outcome == 'success' && steps.launch_lightning_inference.outcome == 'success' }} + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && steps.install_lightning_deps.outcome == 'success' && steps.launch_lightning_inference.outcome == 'success' }} continue-on-error: true timeout-minutes: 90 env: @@ -553,7 +584,7 @@ jobs: - name: Run AI Trading Bot (Distilled Local Fallback) id: run_ai_bot_distilled_local - if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && (steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local' || (steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success'))) || (steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure'))) }} + if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && (steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local' || (steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true' && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success'))) || (steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full' && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure'))) }} continue-on-error: true timeout-minutes: 60 env: @@ -575,7 +606,7 @@ jobs: - name: Run AI Trading Bot (Emergency Distilled Retry) id: run_ai_bot_distilled_emergency_retry - if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && steps.run_ai_bot_distilled_local.outcome == 'failure' }} + if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.run_ai_bot_distilled_local.outcome == 'failure' }} continue-on-error: true timeout-minutes: 60 env: @@ -599,7 +630,7 @@ jobs: - name: Send AI Failure Report id: send_ai_failure_report - if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) && (((steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true') && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success'))) || ((steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full') && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success') || ((steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success')) }} + if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && (((steps.plan_ai_runtime.outputs.runtime_mode == 'cerebrium_full' && steps.validate_cerebrium_primary_config.outputs.has_inference_url == 'true') && (steps.warm_cerebrium_inference.outcome == 'failure' || steps.verify_cerebrium_predict.outcome == 'failure' || (steps.run_ai_bot_cerebrium.outcome == 'failure' && steps.run_ai_bot_cerebrium_retry.outcome != 'success'))) || ((steps.plan_ai_runtime.outputs.runtime_mode == 'lightning_full') && (steps.launch_lightning_inference.outcome == 'failure' || steps.run_ai_bot_in_lightning_studio.outcome == 'failure') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success') || ((steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local') && steps.run_ai_bot_distilled_local.outcome == 'failure' && steps.run_ai_bot_distilled_emergency_retry.outcome != 'success')) }} continue-on-error: true env: SMTP_SERVER: ${{ secrets.SMTP_SERVER }} @@ -712,7 +743,7 @@ jobs: RECIPIENT_EMAIL: ${{ secrets.RECIPIENT_EMAIL }} RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} CORE_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_core_trading) }} - AI_EXPECTED: "false" + AI_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} BASE_DEPS_OUTCOME: ${{ steps.install_base_dependencies.outcome }} CORE_OUTCOME: ${{ steps.run_core_bot.outcome }} CORE_RETRY_OUTCOME: ${{ steps.run_core_bot_retry.outcome }} @@ -772,7 +803,7 @@ jobs: CORE_OUTCOME: ${{ steps.run_core_bot.outcome }} CORE_RETRY_OUTCOME: ${{ steps.run_core_bot_retry.outcome }} CORE_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_core_trading) }} - AI_EXPECTED: "false" + AI_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} INSTALL_LIGHTNING_OUTCOME: ${{ steps.install_lightning_deps.outcome }} AI_RUNTIME_MODE: ${{ steps.plan_ai_runtime.outputs.runtime_mode }} LAUNCH_LIGHTNING_OUTCOME: ${{ steps.launch_lightning_inference.outcome }} diff --git a/main.py b/main.py index cc2ee52..a277d30 100644 --- a/main.py +++ b/main.py @@ -49,6 +49,7 @@ def _resolve_path(base_dir, path_value): return os.path.join(base_dir, path_value) + def _get_open_position_symbols(config_path, table_names=("positions", "positions_ai")): """Return distinct OPEN symbols across the requested position tables.""" config = _load_config(config_path) diff --git a/run_ai_daily_cerebrium.py b/run_ai_daily_cerebrium.py index 4aef305..d66ebda 100644 --- a/run_ai_daily_cerebrium.py +++ b/run_ai_daily_cerebrium.py @@ -4,6 +4,7 @@ import os import subprocess import sys +import time from pathlib import Path import requests @@ -41,13 +42,26 @@ def _preflight_predict(url: str, api_key: str) -> tuple[bool, str]: if api_key: headers["Authorization"] = f"Bearer {api_key}" payload = {"candidates": [{"symbol": "AAPL", "return_5d": 0.01, "news_sentiment_7d": 0.0}]} - try: - resp = requests.post(url, headers=headers, json=payload, timeout=45) - if 200 <= resp.status_code < 300: - return True, f"status={resp.status_code}" - return False, f"status={resp.status_code}" - except Exception as exc: - return False, str(exc) + + attempts = 3 + transient_statuses = {429, 500, 502, 503, 504} + details: list[str] = [] + + for attempt in range(1, attempts + 1): + try: + resp = requests.post(url, headers=headers, json=payload, timeout=45) + details.append(f"attempt={attempt}:status={resp.status_code}") + if 200 <= resp.status_code < 300: + return True, "; ".join(details) + if resp.status_code not in transient_statuses: + return False, "; ".join(details) + except requests.RequestException as exc: + details.append(f"attempt={attempt}:exception={exc}") + + if attempt < attempts: + time.sleep(min(8, 2 * attempt)) + + return False, "; ".join(details) def main() -> None: @@ -56,7 +70,6 @@ def main() -> None: base_env["AI_RUNTIME_MODE"] = "full" base_env["AI_PRIMARY_BACKEND"] = "cerebrium" base_env["AI_ROUTER_REASON"] = "cerebrium_primary_direct" - base_env["ALLOW_MISSING_EMAIL"] = "1" resolved_url = _first_non_empty( base_env.get("CEREBRIUM_TRAINED_MODEL_URL"),