diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 8342225..0ee55bb 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -2,7 +2,9 @@ name: Deploy GitHub Pages on: push: - branches: [main] + branches: + - main + - 'claude/great-bardeen-FRLnY' paths: - 'docs/**' - 'concepts.md' @@ -28,6 +30,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: { python-version: '3.11' } + + - name: Merge extended graph (if generated axes present) + run: | + if [ -d docs/data/generated ]; then + python3 tools/merge_graph.py || true + fi + - name: Stage site run: | mkdir -p _site diff --git a/docs/atlas2d.html b/docs/atlas2d.html new file mode 100644 index 0000000..8de3be7 --- /dev/null +++ b/docs/atlas2d.html @@ -0,0 +1,139 @@ + + + + + +Autonomous-Driving Learning Atlas + + + + + + + + + + + + + +
+
+ + AD Learning Atlas + 从数学直觉到自动驾驶前沿 · ML · RL · VLA +
+ +
+ + + +
+ +
+ +

第一次来?

+

把它当成一张可以点击的学习路线图。

+
    +
  1. 先选背景:左侧 Path A–D 会把你最可能需要的节点点亮。
  2. +
  3. 再点一个节点:右侧卡片会先讲直觉,再讲公式和工程细节。
  4. +
  5. 卡住时对比:按住 Shift 点两个节点,看看两条路线真正分歧在哪里。
  6. +
  7. 最后动手:lab 节点把论文里的想法变成可运行的小实验。
  8. +
+
+ + + + + + + + diff --git a/docs/atlas3d.css b/docs/atlas3d.css new file mode 100644 index 0000000..2554d8f --- /dev/null +++ b/docs/atlas3d.css @@ -0,0 +1,361 @@ +/* Autonomous-driving research-insight 3D atlas — visual chrome. + The WebGL canvas owns the page; the HTML chrome is a thin overlay. */ + +:root { + --bg-deep: #03040b; + --bg-mid: #070a1c; + --ink: #e6edf3; + --ink-dim: #95a3b8; + --ink-mute:#586378; + --line: #1f2638; + --line-bright: #2d3a55; + --accent: #6cb1ff; + --accent-warm:#ffaa55; + --accent-strong:#ff6b6b; + --accent-soft:#a78bfa; + --pane-bg: rgba(8, 12, 24, 0.72); + --pane-bg-strong: rgba(8, 12, 24, 0.92); + --pane-border: rgba(91, 117, 161, 0.28); + --chip-bg: rgba(22, 30, 50, 0.78); + --chip-bg-active: rgba(108, 177, 255, 0.22); + --shadow-soft: 0 10px 35px rgba(0, 0, 0, 0.45); + --radius: 12px; +} + +* { box-sizing: border-box; } + +html, body { + margin: 0; + padding: 0; + height: 100%; + overflow: hidden; + font-family: -apple-system, BlinkMacSystemFont, "PingFang SC", "Hiragino Sans GB", + "Microsoft YaHei", "Segoe UI", system-ui, sans-serif; + color: var(--ink); + background: radial-gradient(ellipse at 50% 40%, #0b1129 0%, #03040b 60%, #000000 100%); +} + +body { -webkit-font-smoothing: antialiased; } + +canvas#atlasCanvas { + position: fixed; + inset: 0; + width: 100vw; + height: 100vh; + display: block; + z-index: 0; +} + +/* ---------- top bar ---------- */ +.topbar { + position: fixed; + top: 0; + left: 0; + right: 0; + z-index: 30; + display: flex; + align-items: center; + justify-content: space-between; + padding: 10px 16px; + background: linear-gradient(180deg, rgba(3,5,15,0.86) 0%, rgba(3,5,15,0.35) 75%, transparent 100%); + backdrop-filter: blur(8px); + pointer-events: auto; +} +.brand { display: flex; align-items: center; gap: 12px; } +.brand .logo { + font-size: 22px; + color: var(--accent); + text-shadow: 0 0 12px rgba(108,177,255,0.65); +} +.brand .title-block { display: flex; flex-direction: column; line-height: 1.15; } +.brand .title { font-weight: 600; font-size: 15px; } +.brand .subtitle { font-size: 11px; color: var(--ink-dim); } +.topbar-actions { display: flex; gap: 8px; align-items: center; } +.topbar-actions input[type="search"] { + background: var(--chip-bg); + border: 1px solid var(--pane-border); + color: var(--ink); + padding: 7px 11px; + border-radius: 8px; + font-size: 13px; + width: 280px; + outline: none; +} +.topbar-actions input[type="search"]:focus { border-color: var(--accent); box-shadow: 0 0 0 2px rgba(108,177,255,0.25); } + +.iconbtn { + background: var(--chip-bg); + border: 1px solid var(--pane-border); + color: var(--ink); + border-radius: 8px; + padding: 6px 11px; + font-size: 12px; + cursor: pointer; + font-family: inherit; + text-decoration: none; + display: inline-flex; + align-items: center; + justify-content: center; + transition: background 0.15s ease, border-color 0.15s ease, color 0.15s ease; + white-space: nowrap; +} +.iconbtn:hover { background: rgba(108,177,255,0.22); border-color: rgba(108,177,255,0.55); } +.iconbtn.active { background: rgba(255,170,85,0.22); border-color: rgba(255,170,85,0.55); color: var(--accent-warm); } + +/* ---------- side panels ---------- */ +.side-panel { + position: fixed; + top: 60px; + bottom: 12px; + width: 320px; + background: var(--pane-bg); + backdrop-filter: blur(12px); + border: 1px solid var(--pane-border); + border-radius: var(--radius); + padding: 14px 14px 18px; + overflow-y: auto; + z-index: 25; + box-shadow: var(--shadow-soft); +} +.side-panel.left { left: 12px; } +.side-panel.right { right: 12px; width: 460px; } +.side-panel.collapsed { transform: translateX(-110%); } +.side-panel[aria-hidden="true"].right { transform: translateX(110%); pointer-events: none; } +.side-panel.right { transition: transform 0.28s ease; } +.side-panel.left { transition: transform 0.28s ease; } + +.pane-section { margin-bottom: 18px; } +.pane-section h3 { + margin: 0 0 8px; + font-size: 11.5px; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--ink-dim); + font-weight: 600; +} +.layer-buttons { display: flex; flex-direction: column; gap: 6px; } +.layer-btn { + text-align: left; + background: var(--chip-bg); + border: 1px solid var(--pane-border); + color: var(--ink); + border-radius: 8px; + padding: 8px 12px; + cursor: pointer; + font-size: 13px; + font-family: inherit; + transition: background 0.15s ease, border-color 0.15s ease; +} +.layer-btn:hover { background: rgba(108,177,255,0.22); } +.layer-btn.active { background: rgba(108,177,255,0.32); border-color: var(--accent); color: var(--accent); } +.hint { font-size: 11.5px; color: var(--ink-mute); margin-top: 8px; line-height: 1.5; } + +.chips { display: flex; flex-wrap: wrap; gap: 6px; } +.chip { + background: var(--chip-bg); + border: 1px solid var(--pane-border); + color: var(--ink); + border-radius: 999px; + padding: 4px 10px; + cursor: pointer; + font-size: 11.5px; + font-family: inherit; + transition: background 0.15s ease, border-color 0.15s ease, color 0.15s ease; +} +.chip:hover { background: rgba(108,177,255,0.18); } +.chip.active { background: var(--chip-bg-active); border-color: rgba(108,177,255,0.55); color: var(--accent); } +.chip .swatch { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; vertical-align: middle; } + +.legend { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 5px; } +.legend li { font-size: 11.5px; color: var(--ink-dim); display: flex; align-items: center; gap: 8px; } +.legend .swatch { width: 16px; height: 3px; border-radius: 2px; flex-shrink: 0; } +.legend .swatch.dashed { background-image: linear-gradient(90deg, currentColor 50%, transparent 50%); background-size: 6px 100%; } + +.time-row { display: flex; align-items: center; justify-content: space-between; margin-top: 6px; font-size: 12px; color: var(--ink-dim); } +input#yearSlider { width: 100%; accent-color: var(--accent); } + +.graph-stats p { font-size: 11.5px; color: var(--ink-mute); line-height: 1.6; margin: 0; } + +/* ---------- right side card ---------- */ +.rp-header { display: flex; align-items: flex-start; gap: 10px; margin-bottom: 8px; } +.rp-header .iconbtn.close, .rp-header #closeRight { + flex-shrink: 0; + font-size: 16px; + padding: 2px 9px; +} +.rp-title-block { display: flex; flex-direction: column; flex: 1; } +.rp-kind { font-size: 10.5px; color: var(--accent-warm); letter-spacing: 0.06em; text-transform: uppercase; } +.rp-title { font-size: 16px; font-weight: 600; color: var(--ink); line-height: 1.3; margin-top: 2px; } +.tabs { display: flex; gap: 4px; margin-bottom: 10px; border-bottom: 1px solid var(--line); padding-bottom: 4px; } +.tab { + background: transparent; + border: none; + color: var(--ink-dim); + padding: 6px 10px; + font-size: 12px; + cursor: pointer; + border-radius: 6px 6px 0 0; + font-family: inherit; +} +.tab.active { color: var(--accent); background: rgba(108,177,255,0.10); } +.card-body { font-size: 13.5px; line-height: 1.65; color: var(--ink); } +.card-body h1, .card-body h2, .card-body h3 { color: var(--accent-warm); margin-top: 18px; margin-bottom: 6px; } +.card-body h1 { font-size: 18px; } +.card-body h2 { font-size: 15px; } +.card-body h3 { font-size: 13.5px; color: var(--accent); } +.card-body code { background: rgba(36, 50, 80, 0.6); padding: 0 4px; border-radius: 3px; font-size: 0.9em; } +.card-body pre { background: rgba(10, 16, 30, 0.85); padding: 10px 12px; border-radius: 8px; overflow-x: auto; font-size: 12px; } +.card-body a { color: var(--accent); text-decoration: none; border-bottom: 1px dashed rgba(108,177,255,0.4); } +.card-body a:hover { color: var(--accent-warm); border-bottom-color: var(--accent-warm); } +.card-body blockquote { border-left: 3px solid var(--accent); padding-left: 10px; color: var(--ink-dim); margin: 10px 0; } +.card-body ul, .card-body ol { padding-left: 22px; } +.card-body table { width: 100%; border-collapse: collapse; margin: 10px 0; font-size: 12.5px; } +.card-body th, .card-body td { border: 1px solid var(--line); padding: 6px 8px; text-align: left; } +.card-body .katex { font-size: 1em; } + +.trace-block { background: rgba(108,177,255,0.08); border-left: 3px solid var(--accent); padding: 10px 12px; margin: 10px 0; border-radius: 4px; } +.trace-block h4 { margin: 0 0 6px; font-size: 12.5px; color: var(--accent); letter-spacing: 0.04em; text-transform: uppercase; } +.trace-block ul { padding-left: 18px; margin: 4px 0; } +.trace-block li { font-size: 12.5px; line-height: 1.55; cursor: pointer; } +.trace-block li:hover { color: var(--accent-warm); } +.trace-block .blockkind { font-size: 10.5px; color: var(--ink-mute); margin-right: 6px; } + +/* ---------- HUD overlay (3D labels) ---------- */ +.hud-overlay { + position: fixed; + inset: 0; + pointer-events: none; + z-index: 10; +} +.hud-label { + position: absolute; + transform: translate(-50%, -100%); + background: rgba(8, 12, 24, 0.72); + border: 1px solid rgba(255,255,255,0.10); + padding: 2px 7px; + border-radius: 5px; + font-size: 10.5px; + color: var(--ink); + white-space: nowrap; + pointer-events: none; + text-shadow: 0 0 4px rgba(0,0,0,0.9); + transition: opacity 0.15s ease; + max-width: 220px; + overflow: hidden; + text-overflow: ellipsis; +} +.hud-label.dim { opacity: 0.18; } +.hud-label.highlight { + background: rgba(255, 170, 85, 0.92); + color: #0f172a; + border-color: rgba(255,170,85,0.85); + font-weight: 600; + text-shadow: none; + z-index: 5; +} +.hud-label.paradigm { + font-size: 14px; + letter-spacing: 0.15em; + font-weight: 700; + background: transparent; + border: none; + text-shadow: 0 0 18px rgba(255,170,85,0.6), 0 0 6px rgba(0,0,0,0.95); + color: #ffe9c8; +} + +/* ---------- context HUD (hover detail) ---------- */ +.context-hud { + position: fixed; + top: 70px; + left: 50%; + transform: translateX(-50%); + z-index: 15; + pointer-events: none; +} +.context-hover { + background: var(--pane-bg-strong); + border: 1px solid var(--pane-border); + border-radius: 10px; + padding: 10px 14px; + font-size: 12.5px; + color: var(--ink); + max-width: 520px; + display: none; + box-shadow: var(--shadow-soft); +} +.context-hover.visible { display: block; } +.context-hover .ch-kind { font-size: 10.5px; color: var(--accent-warm); letter-spacing: 0.06em; text-transform: uppercase; } +.context-hover .ch-title { font-size: 14px; font-weight: 600; margin: 2px 0 4px; } +.context-hover .ch-summary { font-size: 12px; color: var(--ink-dim); line-height: 1.5; } +.context-hover .ch-meta { font-size: 10.5px; color: var(--ink-mute); margin-top: 4px; } + +/* ---------- loading overlay ---------- */ +.loading-overlay { + position: fixed; + inset: 0; + z-index: 100; + display: flex; + align-items: center; + justify-content: center; + background: radial-gradient(ellipse at 50% 50%, #0b1129 0%, #030712 70%); + transition: opacity 0.5s ease; +} +.loading-overlay.fade { opacity: 0; pointer-events: none; } +.loading-core { text-align: center; } +.loading-ring { + width: 64px; + height: 64px; + border-radius: 50%; + border: 2px solid rgba(108,177,255,0.18); + border-top-color: var(--accent); + border-right-color: var(--accent-warm); + margin: 0 auto 18px; + animation: spin 1.6s linear infinite; +} +@keyframes spin { to { transform: rotate(360deg); } } +.loading-title { font-size: 16px; color: var(--ink); margin: 4px 0; } +.loading-status { font-size: 12px; color: var(--ink-dim); margin: 0; } + +/* ---------- help ---------- */ +.help-overlay { + position: fixed; + inset: 0; + background: rgba(0,0,0,0.65); + z-index: 60; + display: none; + align-items: center; + justify-content: center; +} +.help-overlay.visible { display: flex; } +.help-card { + background: var(--pane-bg-strong); + border: 1px solid var(--pane-border); + border-radius: 12px; + padding: 24px 28px; + max-width: 580px; + position: relative; + box-shadow: var(--shadow-soft); +} +.help-card h2 { margin-top: 0; font-size: 18px; color: var(--accent-warm); } +.help-card ul { padding-left: 18px; } +.help-card li { margin-bottom: 6px; font-size: 13px; line-height: 1.55; color: var(--ink); } +.help-card .help-note { font-size: 12px; color: var(--ink-dim); margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--line); } +.help-card kbd { background: var(--chip-bg); border: 1px solid var(--pane-border); padding: 1px 6px; border-radius: 4px; font-size: 11px; } +.iconbtn.close { position: absolute; top: 12px; right: 12px; font-size: 18px; padding: 1px 9px; } + +/* mobile */ +@media (max-width: 920px) { + .topbar-actions input[type="search"] { width: 160px; } + .side-panel.left { width: 280px; } + .side-panel.right { width: 100vw; right: 0; top: auto; bottom: 0; max-height: 60vh; border-radius: 14px 14px 0 0; } + .brand .subtitle { display: none; } +} +@media (max-width: 640px) { + .side-panel.left { display: none; } + .topbar-actions input[type="search"] { width: 120px; } +} + +::-webkit-scrollbar { width: 8px; height: 8px; } +::-webkit-scrollbar-track { background: transparent; } +::-webkit-scrollbar-thumb { background: rgba(108,177,255,0.22); border-radius: 4px; } +::-webkit-scrollbar-thumb:hover { background: rgba(108,177,255,0.45); } diff --git a/docs/data/cards/extended/insight_attention_is_typed_entity_communication.md b/docs/data/cards/extended/insight_attention_is_typed_entity_communication.md new file mode 100644 index 0000000..31ecec3 --- /dev/null +++ b/docs/data/cards/extended/insight_attention_is_typed_entity_communication.md @@ -0,0 +1,43 @@ +# 跨学科洞察 · 注意力是带类型的实体间通信 + +> 看似一个数学动作(query × key → value 加权和),但它真正的物理含义是"让一群带类型的实体直接对话"。理解这一点就能看出为什么注意力在跨学科一次又一次出现。 + +## 抽象内核 + +把每一个 token / patch / query 都看作一个 *实体*,其类型由它在网络里的角色决定(对象 query、车道 query、行人 token、词 token、…)。Self-attention 就是这群实体里"我可以问任何人任何东西"的通信协议;cross-attention 是"我作为外来者,可以从另一群实体里取我需要的东西"。 + +权重 $\text{softmax}(QK^T)$ 不是个机制,而是个 *协议*:让每个询问者根据"我想要的类型"自动选择回答者。 + +## 在不同领域的具现 + +| 领域 | 实体类型 | 协议作用 | +|---|---|---| +| 翻译 | 源语言词、目标语言词 | 让目标词知道源句里哪些词与它对应 | +| 视觉分类 | 图像 patch | 让全局信息进入每个 patch 的表征 | +| 物体检测 (DETR) | 对象 query、图像 patch | 让 query "找到自己关心的对象" | +| 多任务驾驶 (UniAD) | 跟踪 query、运动 query、规划 query | 让跨任务模块直接交换信息 | +| 多模态 (LLaVA) | 文本 token、图像 patch token | 让语言模型理解视觉证据 | +| 神经形态 (Spike-driven Transformer) | 脉冲事件 | 让稀疏事件按需对齐 | +| 机器人 (RT-2) | 视觉 token、动作 token | 让动作生成条件化于场景理解 | +| 蛋白结构 (AlphaFold) | 残基对、氨基酸 | 让分子内远程接触可微计算 | + +## 这条洞察对自动驾驶研究的具体意义 + +1. **凡是想引入新的"研究对象",都可以先把它定义为一类 query**:可行驶区域、对手意图、ego 状态、未来若干秒、社会规范。query 是把"我想关心 X"工程化的最快路径。 +2. **跨模态融合不必是早期 / 晚期 concat**:给每个模态一组 query,让它们与所有模态 cross-attend,得到一个统一的对话广场。 +3. **规划即是 query 写日程**:把每条候选轨迹看成一个 query,让它去 cross-attend 占据、地图、agent 来"问出"风险评估。 +4. **解释性出口**:attention 权重 *作为元数据* 本身就是可解释性,无须额外解释头。 + +## 这条洞察什么时候被滥用 + +- 把任何东西改成 attention 并不必然提升性能;当"实体"本质上没有相互关系(比如音频中的相邻样本),attention 只是浪费计算。 +- 当 query 数量极大时,O(N^2) 的注意力本身成为瓶颈。这才是 FlashAttention / Linear Attention / Performer 的真正动机。 + +## 它跟其它洞察的关系 + +- 它解释为什么 [`insight:end_to_end_differentiable_beats_handcraft_when_signal_strong`](insight_end_to_end_differentiable_beats_handcraft_when_signal_strong.md) 成立:当模块边界由 query 接口对话时,"打通"是免费的。 +- 它跟 [`insight:tokenization_collapses_modality_gap`](insight_tokenization_collapses_modality_gap.md) 是孪生:tokenize 是承诺"我也是实体",attention 是承诺"我们都说同种语言"。 + +## 推演链路 + +[`paper:bahdanau2014_attention`](paper_bahdanau2014_attention.md) → [`paper:vaswani2017`](paper_vaswani2017.md) → [`paper:vit`](paper_vit.md) → [`paper:carion2020`](paper_carion2020.md) → [`paper:li2022bevformer`](paper_li2022bevformer.md) → [`paper:2212.10156`](paper_2212.10156_uniad.md) → [`paper:2402.12289`](paper_2402.12289_drivevlm.md)。 diff --git a/docs/data/cards/extended/insight_dual_system_fast_slow_loop_marries_reactive_and_deliberative_control.md b/docs/data/cards/extended/insight_dual_system_fast_slow_loop_marries_reactive_and_deliberative_control.md new file mode 100644 index 0000000..e2ff8de --- /dev/null +++ b/docs/data/cards/extended/insight_dual_system_fast_slow_loop_marries_reactive_and_deliberative_control.md @@ -0,0 +1,64 @@ +# 跨学科洞察 · 快慢双系统融合反应式与审议式控制 + +> 把"廉价快速的反应"和"昂贵深思的决策"拼成一个闭环,是一条横跨认知科学、强化学习、自动驾驶的反复出现的元设计。它的具现一次比一次更具操作性。 + +## 抽象内核 / Abstract core + +构造两条控制回路: +- **快速回路 $\pi_\text{fast}$**:响应延迟 < 50 ms,覆盖 99% 的常规驾驶;通常是一个轻量神经网络或规则系统。 +- **慢速回路 $\pi_\text{slow}$**:响应延迟 200 ms ~ 几秒,覆盖剩余 1% 的复杂或长时序场景;通常是一个大模型、视觉语言模型、或 MCTS 类搜索。 + +两条回路通过一种"门控 / 调度"机制串联: +- $\pi_\text{slow}$ 给 $\pi_\text{fast}$ 提供子目标或元动作 +- $\pi_\text{fast}$ 在 $\pi_\text{slow}$ 还没输出时维持安全 +- 调度器决定是否切换,常常基于不确定性、场景复杂度或注意力 + +## 在其它学科里的具现 + +| 学科 | 快速回路 | 慢速回路 | 调度机制 | +|---|---|---|---| +| 认知科学 (Kahneman) | 系统 1 直觉 | 系统 2 推理 | 任务难度感知 | +| 游戏 AI (AlphaGo) | 策略网络快速候选 | MCTS 搜索深度评估 | 时间预算 | +| 机器人控制 | PID / MPC | 模型预测 / 优化 | 异常检测 | +| 大语言模型 | 直接采样 | Chain-of-Thought / Tree-of-Thoughts | 答案不确定性 | +| 视觉语言驾驶 | 端到端 BEV 规划 | 多模态大模型审议 | 场景复杂度 / 慢车 / 行人意图 | + +## 这条洞察在自动驾驶里的具体投影 + +### DriveVLM-Dual 路线 +- 快速回路:UniAD-like 端到端规划 +- 慢速回路:视觉语言模型对当前帧做语义注释 + meta-action 建议 +- 调度:场景复杂度(车道变化、行人意图、施工区)触发慢回路接管 + +参见 [`paper:2402.12289`](paper_2402.12289_drivevlm.md) 与 [`paper:2512.24426`](paper_2512.24426_cfvla.md)。 + +### SwiftSage / 通用 agent 路线 +- 快速回路:模仿 LLM +- 慢速回路:reflection + tool use +- 调度:自我评估不确定度 + +### 工程含义 + +1. **延迟预算可控** — 把硬延迟约束分摊给两个回路,单点不再要求"既快又准"。 +2. **安全可证性提升** — 快速回路用更小、更易验证的模型;大模型只在它能贡献时介入。 +3. **数据效率更高** — 慢速回路的输出可作为快速回路的额外监督信号(蒸馏)。 +4. **可解释性出口** — 慢速回路通常用自然语言中介,自动产生 trace。 + +## 这条洞察什么时候失效 + +- **调度器本身错** — 把"该慢"的场景误判成"该快",错失关键审议机会;这是 [`problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop`](problem_hallucinated_action_from_vision_language_model_in_safety_critical_loop.md) 的根因之一。 +- **两条回路目标不一致** — 当 $\pi_\text{slow}$ 优化"解释合理"而 $\pi_\text{fast}$ 优化"轨迹平滑",调度时会产生 chattering。 +- **慢回路过慢** — 大模型推理 > 500 ms 时,整体延迟与不安全。 + +## 这条洞察可以孵化的下一步研究 + +1. **从联合分布学调度器** —— 把调度本身作为 RL 问题,奖励是"何时切换最划算"。 +2. **慢回路的反事实蒸馏** —— 用慢回路在反事实场景里的决定,反过来训练快回路([CF-VLA](paper_2512.24426_cfvla.md) 的初步形态)。 +3. **三系统拓展** —— 引入超慢 (offline) 回路做离线学习与策略更新。 +4. **时间预算自适应** —— 让慢回路给出可解释的"我还需要 X ms"。 + +## 在图谱里的邻居 + +- 提供基础:[`paper:silver2017_alphazero`](paper_silver2017_alphazero.md)(快策略 + MCTS);[`paper:gpt3`](paper_gpt3.md)(CoT 的可行性)。 +- 落地范式:[`paradigm:foundation_model_zero_shot_driving_agent`](paradigm_foundation_model_zero_shot_driving_agent.md)。 +- 相邻洞察:[`insight:test_time_compute_substitutes_train_time_via_search`](insight_test_time_compute_substitutes_train_time_via_search.md)。 diff --git a/docs/data/cards/extended/insight_end_to_end_differentiable_beats_handcraft_when_signal_strong.md b/docs/data/cards/extended/insight_end_to_end_differentiable_beats_handcraft_when_signal_strong.md new file mode 100644 index 0000000..d09d60a --- /dev/null +++ b/docs/data/cards/extended/insight_end_to_end_differentiable_beats_handcraft_when_signal_strong.md @@ -0,0 +1,37 @@ +# 跨学科洞察 · 信号足够强时,端到端可微胜过手工中间表示 + +> 这条洞察反复在不同学科被验证:当下游任务可以提供足够强的可微监督,曾经被工程师精心切分的"模块边界"会被梯度悄悄抹平。 + +## 抽象内核 / Abstract core + +把流水线 $\text{output} = f_n(\,f_{n-1}(\,\dots f_1(\text{input})\,))$ 中的每一个 $f_i$ 都换成一个可微神经函数,并允许梯度从最终损失沿着复合函数链回传到所有 $f_i$。在足够强的信号下,这种端到端学习会自动放弃中间任务的"语义合理性",转而采用让最终损失最小的中间表示。这往往跟工程师事先切分的"模块责任"不一致——这正是这条洞察的反直觉之处。 + +## 在其它学科里的具现 / Manifestations across fields + +| 学科 | 旧式手工切分 | 端到端取代 | 关键论文 | +|---|---|---|---| +| 语音识别 | 声学模型 + 发音词典 + 语言模型,三段独立训练 | "Listen Attend Spell" 把声学帧直接映射到字符序列 | Chan et al. 2015 | +| 机器翻译 | 短语对齐 + 翻译模板 + 语言模型重排 | 序列到序列加注意力一气呵成 | Bahdanau et al. 2014 | +| 自动驾驶 | 检测—跟踪—预测—规划四段流水线 | 共享 BEV 查询的统一架构 | [UniAD](paper_2212.10156_uniad.md) | +| 机器人操作 | 视觉感知 → 状态估计 → 运动规划 → 控制 | RT-2 / OpenVLA 把视觉直接映射到动作 token | RT-2, 2023 | + +## 为什么"信号强"是前提 + +只有当 (a) 最终损失能稳定地把"做错"的方向编码出来、(b) 每一步中间表示都被允许由梯度重新分配语义、(c) 数据规模够大、(d) 计算预算允许整张图同时优化,这条洞察才成立。在这些条件中任何一个不满足时,把工程师的领域知识打进模块边界依然有价值——这是 [PlanT](paper_2210.14222_plant.md)、[InterFuser](paper_interfuser.md) 等"半端到端"方案在小数据上仍然有效的原因。 + +## 在自动驾驶研究里的可操作含义 / What it suggests for AD + +1. **不要去捍卫旧分界**:当你能给上游一个稳定可微的下游目标,旧的检测—跟踪—预测分界会先变模糊再消失。 +2. **把损失放在最后**:UniAD 的规划损失是骨架,其它损失只是辅助权重;要承认这一点。 +3. **预留可解释性出口**:端到端不等于黑盒。`move:add_explanation_head_to_promote_interpretability` 让中间表示在保持端到端的同时仍然可读。 +4. **它会跟苦涩教训争锋**:参见 [`essay:bitter_lesson`](essay_bitter_lesson.md)。Sutton 会说"连 BEV 这个人工坐标系都该让位"。这条洞察是过渡期的实用主义:在 *scaling laws* 真正接管以前,给端到端一个工程化的注入口。 + +## 当这条洞察"不成立"时 + +* 信号稀疏(罕见安全事件)→ 转向 [反事实数据增广](paradigm_counterfactual_data_centric_safety.md)。 +* 数据私有且贵 → 转向自监督预训练 + 微调([DINOv3 路线](paper_2508.10104_dinov3.md))。 +* 模型部署预算极紧 → 转向 [类脑稀疏推理](paradigm_brain_inspired_neuromorphic_co_design.md)。 + +## 谱系 / Lineage in this atlas + +[ResNet 残差](paper_he2015_resnet.md) → [Transformer](paper_vaswani2017.md) → [DETR](paper_carion2020.md) → [BEVFormer](paper_li2022bevformer.md) → [UniAD](paper_2212.10156_uniad.md) → [DriveVLM](paper_2402.12289_drivevlm.md) → [CF-VLA](paper_2512.24426_cfvla.md) diff --git a/docs/data/cards/extended/insight_masked_prediction_yields_self_supervised_signal.md b/docs/data/cards/extended/insight_masked_prediction_yields_self_supervised_signal.md new file mode 100644 index 0000000..f36e4b4 --- /dev/null +++ b/docs/data/cards/extended/insight_masked_prediction_yields_self_supervised_signal.md @@ -0,0 +1,44 @@ +# 跨学科洞察 · 掩码预测榨取出自监督信号 + +> 这条洞察是过去十年自监督学习的核心引擎。它在不同领域以同一个机械配方反复出现:随机遮住一部分输入,让模型预测被遮住的部分;得到的表征质量远超有监督任务对应的特征。 + +## 抽象内核 / Abstract core + +给定一段结构化输入 $x = (x_1, x_2, ..., x_T)$,定义掩码集 $\mathcal{M} \subset \{1..T\}$。模型学习 $\mathbb{E}\big[ x_{\mathcal{M}} \mid x_{\setminus \mathcal{M}} \big]$。重要的是:损失只在被遮住位置评估;模型不会"作弊"地直接复制输入。 + +这一配方在不同模态中表现为: +- **语言**:随机遮住 token,预测原始 token(BERT)。 +- **视觉**:随机遮住 patch,预测 pixel 或潜在 patch token(MAE / BEiT)。 +- **视频**:遮住时空 cube,预测原 frames(VideoMAE)。 +- **驾驶占据**:遮住 BEV 格点,预测占据值(OccMAE)。 +- **轨迹**:遮住未来时间步,预测下一帧动作(Trajeglish)。 + +## 这条洞察的物理含义 + +它在隐含地告诉模型:"**世界有冗余结构**:当我蒙住一块,剩下的部分足够你推出它是什么。" 让模型把这一推断能力内化,等于让它把世界的"约束几何"学进去——这种几何远比标签里编码的语义更通用。 + +## 在自动驾驶研究里的具体投影 + +1. **感知预训练**:用百万小时无标注驾驶视频做 masked patch prediction,得到的视觉骨干迁移到 nuScenes 检测器优于纯监督。[`paper:2508.10104` (DINOv3)](paper_2508.10104_dinov3.md) 路线的延伸。 +2. **BEV 占据预训练**:对 BEV 占据图做 masked prediction,得到的特征显著加快下游规划微调。 +3. **运动学预训练**:对 ego trajectory 做 masked next-step,借语言模型经验把轨迹也变成"句子"。 +4. **多模态对齐**:联合掩码两个模态(图像 + LiDAR),强制模型学跨模态结构。 + +## 它什么时候不奏效 + +- 当输入太"密集"(每个位置几乎独立),mask 不强制结构归纳。 +- 当 mask ratio 太低,模型只需局部插值,学不到全局表征。 +- 当解码器太弱,损失主要被解码器吸收,编码器无激励学好表征。这是 MAE 论文里被反复强调的"不对称设计"原则。 + +## 派生的研究问题 / Open derivative questions + +1. **mask 设计的最优空间**:随机、规整、对象级、运动相关——哪一种 mask 在驾驶里给最快的下游收敛? +2. **跨模态 mask 协议**:图像 mask + LiDAR 不 mask vs 反之 vs 双 mask,哪个学到更对齐的表征? +3. **闭环 finetune 的稳定性**:自监督预训练的特征在闭环 RL 下是否会被破坏? +4. **mask 损失与 contrastive 损失的混合**:DINOv2 / DINOv3 已经显示两者互补,最优配比尚未确定。 + +## 推演链路 / Lineage in this atlas + +[BERT / GPT-3](paper_gpt3.md) → [MAE](paper_mae.md) / [BEiT](paper_beit.md) → [DINOv2](paper_dinov2.md) → [DINOv3](paper_2508.10104_dinov3.md) → 投影到驾驶 → BEV 占据 mask、video mask 等专门化变体。 + +[`move:add_auxiliary_self_supervision_signal_from_geometry`](move_add_auxiliary_self_supervision_signal_from_geometry.md) 是这条洞察的姐妹:它用几何一致性而非 mask,但内核相同——免费的监督信号。 diff --git a/docs/data/cards/extended/insight_scaling_data_unlocks_capabilities_not_present_in_smaller_models.md b/docs/data/cards/extended/insight_scaling_data_unlocks_capabilities_not_present_in_smaller_models.md new file mode 100644 index 0000000..579cbff --- /dev/null +++ b/docs/data/cards/extended/insight_scaling_data_unlocks_capabilities_not_present_in_smaller_models.md @@ -0,0 +1,44 @@ +# 跨学科洞察 · 数据规模解锁的能力,小模型不会预先显示 + +> 这条洞察是过去十年人工智能发展的核心引擎。它的反直觉之处在于:"想看到 X 能力,必须先把模型与数据都放大到某个阈值"——而在阈值之下,X 从未出现过。 + +## 抽象内核 / Abstract core + +设某项能力 $\mathcal{C}$ 在数据规模 $D$、参数规模 $P$ 下的出现概率为 $\Pr[\mathcal{C} \mid D, P]$。经验上 $\Pr[\mathcal{C} \mid D, P]$ 在某个阈值 $(\tilde D, \tilde P)$ 处呈现 *阶跃*:阈值以下接近 0,阈值以上稳定接近 1。这一阶跃既无法被理论从第一性原理预先算出,也无法从小模型实验中外推。 + +## 已观察到的能力涌现 + +| 能力 | 大致涌现阈值 | 在自动驾驶里的对应 | +|---|---|---| +| In-context learning | GPT-3 (175B) | DiLu / Agent-Driver 的提示式决策 | +| Chain-of-Thought | Flan-PaLM (62B) | DriveVLM 的语言推理链 | +| 多模态 grounding | LLaVA / Flamingo 级 | DriveLM 的视觉 + 自然语言驾驶解释 | +| 零样本指令跟随 | InstructGPT | LMDrive 的 high-level meta-action | +| Tool use | Toolformer / GPT-4 | Agent-Driver 的 perception / memory 工具循环 | +| Reflection / self-correction | GPT-4 / Claude 3 | DiLu 的反思循环 | + +## 这条洞察跟苦涩教训的关系 + +[`essay:bitter_lesson`](essay_bitter_lesson.md) 是这条洞察的姐妹:它在解释 *为什么* 应该投入算力而非工程化先验。这条洞察告诉你 *何时* 这一投资会兑现——只在跨过阈值之后。 + +## 在自动驾驶里的预测含义 + +1. **VLM 直接做规划**:当前 7B 级模型在轨迹规划上还在挣扎;推测 70B+ 级在 corner case 推理上将出现质变。这条洞察让人有理由把基础设施提前布置好。 +2. **闭环 self-play**:当模型规模足够大、世界模型足够好时,闭环 RL 可能从"难以工程化"突变为"几乎免费"。 +3. **多车队 / 多城市数据规模回报**:在某一阈值后,跨城市泛化可能突然变得稳定。 + +## 这条洞察什么时候不奏效 + +- 当评估指标本身是非线性平滑的(如 BLEU),能力涌现可能被指标平滑化掉,看起来像连续提升。 +- 当数据质量是瓶颈(重复、噪声、单一来源),单纯放大不解决问题。这是为什么 [`paradigm:closed_loop_data_engine_centric_development`](paradigm_closed_loop_data_engine_centric_development.md) 在 scaling 之外仍是必修。 +- 当任务的核心难点在物理常识或感知精度(自动驾驶最常见),scaling 单独不够,需要配合多模态。 + +## 给研究决策的启发 + +- **不要用小模型实验否定大模型可能性**:负结果在小规模上常见,但在大规模可能消失。 +- **观察 scaling 曲线本身**:如果损失曲线在加大模型时呈现 *变曲* 而非平稳,可能是涌现的前兆。 +- **算力 vs 数据 vs 标注质量**:在不同阈值附近,三者的杠杆率不同。Chinchilla 提供初步标定。 + +## 推演链路 + +[`paper:vaswani2017`](paper_vaswani2017.md) → [`paper:gpt3`](paper_gpt3.md)(第一份显式 emergent capability 的工作)→ [Chinchilla scaling laws](paper_chinchilla.md) → [`paper:llava`](paper_llava.md) → [`paper:2402.12289`](paper_2402.12289_drivevlm.md) → [`paper:rt2`](paper_rt2.md) → 推演给驾驶 [`paradigm:foundation_model_zero_shot_driving_agent`](paradigm_foundation_model_zero_shot_driving_agent.md)。 diff --git a/docs/data/cards/extended/insight_test_time_compute_substitutes_train_time_via_search.md b/docs/data/cards/extended/insight_test_time_compute_substitutes_train_time_via_search.md new file mode 100644 index 0000000..c6f87b1 --- /dev/null +++ b/docs/data/cards/extended/insight_test_time_compute_substitutes_train_time_via_search.md @@ -0,0 +1,44 @@ +# 跨学科洞察 · 测试时间的计算可以通过搜索替代训练时间的计算 + +> 这条洞察反复出现:当模型已经学到一些基础能力,提供更多的 *推理时间* 算力,可以在不重新训练的前提下显著提高决策质量。 + +## 抽象内核 / Abstract core + +设决策质量 $Q$ 是 (训练算力 $C_\text{train}$, 测试算力 $C_\text{test}$) 的函数。经验上 $Q$ 关于这两个变量是 *次可加* 的:增大其中任意一个都能提高 $Q$,但二者之间存在替代关系。当训练算力达到边际递减区间,加大测试算力(通过搜索、采样、辩论、自我批评)反而更高效。 + +## 在不同领域的具现 + +| 领域 | 训练侧 | 测试侧的搜索机制 | +|---|---|---| +| 围棋 / 国际象棋 | 神经策略 | MCTS 树搜索 (AlphaGo, AlphaZero) | +| 大语言模型推理 | Pretrain + RLHF | Chain-of-Thought, Self-Consistency, Tree-of-Thoughts | +| 程序生成 | Codex 等编码大模型 | 候选采样 + 单元测试验证 | +| 蛋白结构 | AlphaFold 网络 | MSA 与模板的扩展查询 | +| 自动驾驶规划 | 模仿 + 微调 | 多候选轨迹生成 + 显式 cost 评估 | +| Agent 决策 | 基础 LM | ReAct / Reflexion 多轮反思 | + +## 这条洞察在自动驾驶研究里的具体投影 + +1. **轨迹候选 + 显式打分**:模型生成 K 条候选轨迹,由独立模块(碰撞预测、平稳性、合规性)打分。等价于在 K 维离散空间做搜索。 +2. **慢回路的反思**:[DriveVLM-Dual](paper_2402.12289_drivevlm.md) 的慢回路本质是测试时搜索。 +3. **反事实重规划**:[CF-VLA](paper_2512.24426_cfvla.md) 让模型在反事实分支里探索,再选最稳的。 +4. **MCTS-style 驾驶**:让 ego 在世界模型里 rollout 多种动作组合,挑回报最高的——本质是 [`paradigm:model_based_world_imagination_planning`](paradigm_model_based_world_imagination_planning.md) 的测试时版本。 + +## 怎样诊断你的问题适合这条洞察 + +* 训练数据足够大但模型仍在边际递减区间 → 加测试算力可能更值。 +* 决策需要长时序考虑、却受 context window 限制 → 搜索分摊。 +* 决策可被验证 / 评分但难以直接训练 → 测试时验证胜过训练时蒸馏。 +* 延迟预算允许 → 100–500 ms 的额外搜索是可接受的。 + +## 当这条洞察失效 + +* 任务实时性极强(< 50 ms):搜索做不动。 +* 评分模块本身不准:测试时搜索只会放大错误。 +* 模型基础能力差到搜索覆盖不到正确解:先训好再搜。 + +## 推演链路 + +[`paper:silver2017_alphazero`](paper_silver2017_alphazero.md)(这条洞察的极致证明)→ CoT / Self-Consistency / ToT 在 LLM 里复现 → [DriveVLM 的 dual-system](paper_2402.12289_drivevlm.md) → [CF-VLA 的反事实搜索](paper_2512.24426_cfvla.md)。 + +它的兄弟洞察:[`insight:dual_system_fast_slow_loop_marries_reactive_and_deliberative_control`](insight_dual_system_fast_slow_loop_marries_reactive_and_deliberative_control.md) 提供调度框架;[`insight:world_model_as_inner_simulator_unlocks_long_horizon_planning`](insight_world_model_as_inner_simulator_unlocks_long_horizon_planning.md) 提供搜索的环境。 diff --git a/docs/data/cards/extended/move_lift_2d_to_3d.md b/docs/data/cards/extended/move_lift_2d_to_3d.md new file mode 100644 index 0000000..36a2ffb --- /dev/null +++ b/docs/data/cards/extended/move_lift_2d_to_3d.md @@ -0,0 +1,51 @@ +# 方法学原语 · 把 2D 图像特征抬升到 3D + +> 这一动作在自动驾驶感知里出现的频率极高。它的本质是:当你只有几路 2D 相机、却想让所有下游模块在 3D(或鸟瞰图)坐标系里工作时,需要一个可微的"升维"算子。 + +## 这个动作做什么 + +给定 $N$ 路相机的 2D 特征图 $\{F_i \in \mathbb{R}^{H \times W \times C}\}_{i=1..N}$ 以及它们的内外参,构造一个 BEV 特征图 $G \in \mathbb{R}^{B \times B \times C}$,使得 $G[u, v]$ 反映"鸟瞰图上 $(u, v)$ 位置周围的语义"。 + +主要难点:**深度未知**。同一条像素射线对应的世界点位姿是一条无限远的射线。研究者发明了几种把深度处理掉的策略,每一种都给出了一个独立的"升维原语"。 + +## 主要分支 / Sub-moves + +| 子分支 | 怎样处理深度 | 代表工作 | +|---|---|---| +| **Lift-Splat-Shoot** | 预测一个离散深度概率分布 $p(d \mid u, v)$,然后把每个像素的特征"散布"到一组沿射线的 3D 体素 | LSS (Philion 2020) | +| **可学习的几何投影** | 学一组对象 query 直接 cross-attend 到所有相机的 2D 特征,由 attention 隐式决定来自哪个 (i, u, v) | DETR3D (Wang 2021) | +| **3D 位置编码** | 把 3D 坐标投影回 2D,把这一编码作为额外的 token 拼到图像 token 上 | PETR / PETRv2 | +| **时空 BEV query** | 在 BEV 平面预先布一组 query,让它们在每帧 cross-attend 到当前相机特征,并在时间上做 recurrent 累积 | BEVFormer | +| **显式深度监督** | 用 LiDAR 深度图监督深度分支,加快收敛 | BEVDet, BEVDepth | + +## 为什么是一个"原语"而不是"一个方法" + +因为它能被反复重新组合成新方法: + +- **+ 自监督预训练** → 在百万小时无标注视频上学几何先验,再把 lift-splat 接到下游 (近似 DINOv3 + BEVFormer 路线)。 +- **+ 占据预测** → 用 3D voxel grid 直接预测每个体素的占据概率,得到统一的 [可行驶区域 + 障碍物 + 语义] 表征 (Tesla Occupancy Network 路线)。 +- **+ 时间 recurrent** → 在 BEVFormer 上叠 streaming 模式,得到流式 BEV (StreamPETR)。 +- **+ 端到端规划** → 把升维结果直接喂给共享 query 头,得到 [UniAD](paper_2212.10156_uniad.md)、[VADv2](paper_vadv2.md)。 +- **+ 神经辐射场** → 把升维替换为可渲染的 3D 场,得到 [EmerNeRF / DrivingGaussian](paper_emernerf.md) 路线。 + +## 物理直觉 / Physical intuition + +把这一动作看成"在一组只有侧面投影的 2D 影子里复原 3D 体的概率分布"。它跟传统多视几何里的三角化等价,区别在于: +- 三角化要求点对应;升维允许"概率性多重对应" +- 三角化把深度估计当独立步骤;升维把深度作为可学习的内部变量、由下游任务隐式监督 + +## 这个动作什么时候不该用 + +- 当下游任务只关心 2D 像素级语义(比如车道线检测)时,升维只会引入额外噪声。 +- 当你有稠密 LiDAR 时,传统点云方法(PointPillars、CenterPoint)仍然更鲁棒,升维的好处在弱传感配置下才明显。 +- 当目标场景几何先验极强(如固定道路结构)时,可以省去升维,直接用相机到地面的同质性映射。 + +## 推演溯源 / Components needed to invent this move from scratch + +1. **可微数学** —— 把分布的"抬升"操作写成可反向传播的算子。 +2. **集合预测接口** —— 由 [DETR](paper_carion2020.md) 提供:让模型可以在 BEV 上输出一组互不重复的对象。 +3. **多视几何先验** —— 经典视觉里的相机模型 + 三角化的几何直觉。 +4. **跨视角共享 BEV 的需求** —— 由 [模块化感知到规划流水线](paradigm_modular_perception_to_planning_pipeline.md) 的痛点 (跨相机融合) 直接推出。 +5. **可微深度分布的可行性证据** —— LSS 2020 给出了第一份。 + +只要前 4 项在图谱里齐全,第 5 项就是必然出现的工程结果。 diff --git a/docs/data/cards/extended/paradigm_brain_inspired_neuromorphic_co_design.md b/docs/data/cards/extended/paradigm_brain_inspired_neuromorphic_co_design.md new file mode 100644 index 0000000..0e12719 --- /dev/null +++ b/docs/data/cards/extended/paradigm_brain_inspired_neuromorphic_co_design.md @@ -0,0 +1,52 @@ +# 范式 · 类脑神经形态硬件—算法协同设计 + +> 当稠密 transformer 每瓦特只能跑出有限 TOPS 时,自动驾驶的能耗预算迟早遇到边界。这条范式不修改任务,而修改"我们计算它的方式":用事件驱动的稀疏脉冲计算把算法与硬件重新捆绑。 + +## 押注 / Bet + +1. **驾驶场景本身具有强稀疏性**:大多数像素与时间步是冗余的。事件相机 (DVS) 把这一稀疏性直接暴露为输入分布。 +2. **生物启发的脉冲神经网络 (SNN)** 在硬件上每瓦特可以做出比稠密 GPU 推理高 1–2 个数量级的能效,前提是算法配合稀疏调度。 +3. **硬件—算法协同设计** 比单独优化任何一方都更值钱。神经形态芯片 (Loihi 2、Tianjic、TrueNorth、GrAI) 已经能跑这样的工作负载。 + +## 家族成员 + +| 工作 / 平台 | 角色 | +|---|---| +| [Spike-driven Transformer](paper_2307.01694_spike_driven_transformer.md) | 提供"用脉冲信号实现 attention"的第一份证据 | +| Loihi 2 | 真实的事件驱动神经形态芯片 | +| Tianjic | 国产混合范式芯片 | +| DVS event cameras | 微秒级延迟视觉传感 | +| 早期工作:Maass 等的 SNN 理论基础 | 提供数学骨架 | + +## 必备组件 + +* **脉冲神经元的可微近似**:surrogate gradient,使 SNN 可用 backprop 训练。 +* **事件驱动注意力**:稀疏 attention,仅在 spike 到达时计算。`move:replace_dense_attention_with_sparse_event_driven_attention`。 +* **硬件—算法仿真器**:Loihi / Tianjic 仿真器,让算法研究可以在没有真实芯片时迭代。 +* **能效—精度权衡曲线**:把每瓦特 mAP 作为一等指标。 +* 推荐:[`insight:event_driven_computation_matches_natural_sparsity_of_driving_scene`](insight_event_driven_computation_matches_natural_sparsity_of_driving_scene.md) 与 [`insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency`](insight_hardware_software_co_design_unlocks_orders_of_magnitude_efficiency.md)。 + +## 这条范式回应什么痛点 + +- [`problem:energy_budget_too_small_for_full_transformer_at_30fps`](problem_energy_budget_too_small_for_full_transformer_at_30fps.md) +- [`problem:simulator_visual_gap_breaks_perception_models`](problem_simulator_visual_gap_breaks_perception_models.md)(事件相机几乎不受光照变化影响) +- 长尾延迟敏感场景:紧急制动需要 < 50 ms 反应。 + +## 打开的研究突破口 + +1. **事件驱动 BEV 感知**:直接从 DVS 流构建占据网格。 +2. **SNN 蒸馏**:把稠密 transformer 蒸馏成等价 SNN,保留精度同时获得能效。 +3. **混合系统**:常规帧用稠密 CNN/Transformer,关键事件触发 SNN 反射弧。 +4. **量化—稀疏—事件三位一体**:让神经形态硬件同时利用三种稀疏来源。 + +## 它跟谁争锋 + +- 跟 [`essay:bitter_lesson`](essay_bitter_lesson.md) 对头:苦涩教训说所有"工程化的稀疏先验"最终都会被 scaling 取代。这条范式的辩护是:边缘部署能耗预算不会按 Moore 定律放大,所以硬件—算法协同的实用主义短期内胜出。 +- 跟 [`paradigm:scaling_data_with_self_supervision`](paradigm_scaling_data_with_self_supervision.md) 的方向相反,但两者可以叠加:用稠密自监督预训练特征作为 SNN 学生模型的教师。 + +## 推荐起步 + +1. [Spike-driven Transformer 卡片](paper_2307.01694_spike_driven_transformer.md)。 +2. [`concept:spiking_nn`](../../concepts.md#脉冲神经网络)。 +3. 跑 [`labs/lab06_spike_driven_attention_mnist`](../../../labs/lab06_spike_driven_attention_mnist.ipynb)。 +4. 读 [`paper:flashattention`](paper_flashattention.md) 卡片对照"稠密世界里的能效优化",理解为什么神经形态值得独立路径。 diff --git a/docs/data/cards/extended/paradigm_closed_loop_data_engine_centric_development.md b/docs/data/cards/extended/paradigm_closed_loop_data_engine_centric_development.md new file mode 100644 index 0000000..9b86a2b --- /dev/null +++ b/docs/data/cards/extended/paradigm_closed_loop_data_engine_centric_development.md @@ -0,0 +1,51 @@ +# 范式 · 以闭环数据引擎为中心的开发 + +> 在工业自动驾驶里,最值钱的资产不是任何单一模型,而是一个完整的"采集—自动标注—难例挖掘—合成扩增—闭环回放—模型选择—部署反馈"循环。这条范式把这一循环本身作为研究对象。 + +## 押注 / Bet + +1. **静态数据集是死的,数据引擎是活的**。一个标注协议不再升级、不再产出难例、不再被模型反馈,就停止增值。 +2. **闭环验证是评估的真值**。模型选择必须基于在自己分布上的行为,而非专家分布上的拟合。 +3. **车队的规模回报是流程化的**:当上百辆车持续上传日志,怎样自动定位、合成、标注、回放新场景比模型架构本身重要。 +4. **失败案例可以被工程化转化为训练信号**:从"事故复盘"到"自动反事实场景生成"的工程链路。 + +## 家族成员(多为工业实践 / industrial paradigms) + +| 平台 / 工作 | 角色 | +|---|---| +| Tesla 数据引擎 | 第一份大规模工程化范例:auto-label + shadow mode + corner case mining | +| [Waymo Scenario Mining](paper_waymo_scenario_mining.md) | 多模态搜索任意行为模式 | +| [nuPlan](paper_nuplan.md) | 离线 + 闭环联合评分基础设施 | +| [Bench2Drive / NAVSIM](paper_bench2drive.md) | 闭环驱动评估的开源版本 | +| [CARLA Leaderboard 2.0](paper_carla_leaderboard.md) | 闭环挑战赛 | +| [CF-VLA](paper_2512.24426_cfvla.md) | 把反事实生成绑定进数据引擎 | + +## 必备组件 / Building blocks + +* **自动标注**:[`move:auto_label_with_offline_model_then_human_in_loop_validate`](move_auto_label_with_offline_model_then_human_in_loop_validate.md)。 +* **难例挖掘**:[`move:run_active_learning_loop_to_query_hardest_unlabeled_frames`](move_run_active_learning_loop_to_query_hardest_unlabeled_frames.md)。 +* **场景检索**:基于自然语言或运动模式的查询引擎。 +* **闭环回放**:[`move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness`](move_run_replay_simulation_with_perturbed_initial_conditions_for_robustness.md)。 +* **离线—闭环联合评分**:[`move:design_closed_loop_metric_correlated_with_real_world_safety`](move_design_closed_loop_metric_correlated_with_real_world_safety.md)。 +* **影子模式**:在已部署模型旁运行候选模型,记录分歧但不下发其控制。 +* **持续学习**:[`move:run_continual_learning_with_rehearsal_buffer_against_forgetting`](move_run_continual_learning_with_rehearsal_buffer_against_forgetting.md)。 + +## 这条范式正面回应 + +- [`problem:offline_metric_does_not_predict_closed_loop_performance`](problem_offline_metric_does_not_predict_closed_loop_performance.md) +- [`problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented`](problem_rare_safety_critical_events_dominate_real_risk_but_are_under_represented.md) +- [`problem:catastrophic_forgetting_under_continual_learning`](problem_catastrophic_forgetting_under_continual_learning.md) +- [`problem:label_noise_for_3d_object_categories`](problem_label_noise_for_3d_object_categories.md) + +## 它打开的研究突破口 + +1. **数据引擎的可学习调度器**:让 RL agent 决定下一步采集什么场景、合成什么反事实、标注哪一帧。 +2. **跨车队 federated 数据引擎**:把不同车厂的难例匿名化共享。 +3. **影子模式作为安全对齐信号**:候选模型的分歧统计直接喂入模型选择。 +4. **闭环—离线 oracle**:用一组少量精心选定的闭环测试预测大规模离线模型性能。 + +## 推荐起步 + +1. [Tesla AI Day 卡片](paper_tesla_ai_day.md):第一份工业级数据引擎复盘。 +2. [nuPlan 卡片](paper_nuplan.md):开源世界里离线—闭环联合评估的最佳起点。 +3. 读 [`insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners`](insight_closed_loop_evaluation_is_the_only_ground_truth_for_planners.md) 与 [`insight:data_engine_loop_is_more_valuable_than_static_dataset`](insight_data_engine_loop_is_more_valuable_than_static_dataset.md)。 diff --git a/docs/data/cards/extended/paradigm_counterfactual_data_centric_safety.md b/docs/data/cards/extended/paradigm_counterfactual_data_centric_safety.md new file mode 100644 index 0000000..db72931 --- /dev/null +++ b/docs/data/cards/extended/paradigm_counterfactual_data_centric_safety.md @@ -0,0 +1,54 @@ +# 范式 · 以反事实数据为中心的安全 + +> 这条范式直面自动驾驶里最尴尬的事实:罕见安全事件支配真实风险,但又几乎不在数据集中出现。它拒绝"再收集 10× 真实数据"的简单回应,转而主张"用对抗合成 + 反事实生成补齐分布"。 + +## 押注 / Bet + +1. **真实数据收集的边际收益正在递减**:当车队规模线性增长,新罕见事件的发现率呈对数衰减。 +2. **生成式模型已经够好**:视频扩散、3D 重建、神经辐射场,可以以足够保真度伪造危险场景。 +3. **反事实推理是工程化的安全声明**:"在这一帧把行人挪到马路中央,模型会怎么决策?" — 这个问题可问、可验证、可监督。 +4. **数据中心论 (data-centric AI)**:算法稳定时,数据分布是安全的真正决定因素。 + +## 家族成员 / Members + +| 工作 | 角色 | +|---|---| +| [CF-VLA](paper_2512.24426_cfvla.md) | 第一份把反事实生成显式接入 VLA loop 的工作 | +| [SHIFT](paper_shift.md) | 早期的 sim-to-real 中跨域偏移数据集 | +| [V2X-Sim](paper_v2x_sim.md) | 多代理协同的合成数据 | +| Tesla 数据引擎 | 工业届的工程化先例 (auto-labelling + scenario mining) | +| [DriveDreamer](paper_drivedreamer.md) | 提供条件化生成的视频底座 | + +## 必备组件 / Required building blocks + +* 一个高保真世界/视频生成模型:[GAIA-1](paper_gaia1.md) / [DriveDreamer](paper_drivedreamer.md) / [Cosmos](paper_cosmos.md)。 +* 反事实编辑算子:能在场景里"插入、删除、移位"对象。`move:augment_via_counterfactual_object_insertion`。 +* 闭环评估器:在合成场景里能跑完整规划→控制 loop,看模型怎样反应。 +* 故障案例本体:建立一个"危险情形"的形式化字典,用于条件化生成。 +* 长尾敏感损失:训练阶段加权罕见场景。 + +## 这条范式正面回应什么痛点 + +- [`problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented`](problem_rare_safety_critical_events_dominate_real_risk_but_are_under_represented.md) +- [`problem:offline_metric_does_not_predict_closed_loop_performance`](problem_offline_metric_does_not_predict_closed_loop_performance.md) +- [`problem:zero_shot_generalization_to_unseen_driving_scenes`](problem_zero_shot_generalization_to_unseen_driving_scenes.md) +- [`problem:counterfactual_reasoning_about_other_agents_intent`](problem_counterfactual_reasoning_about_other_agents_intent.md) + +## 它打开的研究突破口 + +1. **自动场景挖掘 + 反事实扩增 + 闭环回放** 三件套作为新一代数据中心 pipeline。 +2. **生成模型的安全证书**:怎样形式化保证 X 张反事实场景覆盖了"碰撞前 2 秒可能态"的某一覆盖率下界。 +3. **VLA 的反事实蒸馏**:让大模型在 corner case 上做长 CoT,再把决策蒸馏给快回路。 +4. **跨车队反事实交换**:A 车队遇到的罕见事件可以条件化生成给 B 车队,加速跨地区迭代。 + +## 谁是它的对头 + +* [`essay:bitter_lesson`](essay_bitter_lesson.md) 会指出"靠合成数据并非长期解",scaling 真实日志 + 通用算法应该胜出。这条范式承认这一点,但赌 *在 scaling laws 真正接管之前的几年*,合成是收益最高的工程路径。 +* [`paradigm:safety_by_constraint_layered_architecture`](paradigm_safety_by_constraint_layered_architecture.md) 提供另一种思路:不靠数据,而靠 hard 约束。在多数实际系统里两条线并行使用。 + +## 推荐起步路径 + +1. [CF-VLA 卡片](paper_2512.24426_cfvla.md):把这条范式的当代代表读熟。 +2. [GAIA-1 / DriveDreamer](paper_drivedreamer.md):理解世界模型作为合成器。 +3. 读 [`insight:long_tail_solved_by_synthesis_not_by_data_alone`](insight_long_tail_solved_by_synthesis_not_by_data_alone.md)。 +4. 跑 [`labs/lab10_cfvla_counterfactual_replanner`](../../../labs/lab10_cfvla_counterfactual_replanner.ipynb)。 diff --git a/docs/data/cards/extended/paradigm_differentiable_end_to_end_imitation.md b/docs/data/cards/extended/paradigm_differentiable_end_to_end_imitation.md new file mode 100644 index 0000000..f6ef45e --- /dev/null +++ b/docs/data/cards/extended/paradigm_differentiable_end_to_end_imitation.md @@ -0,0 +1,51 @@ +# 范式 · 可微端到端模仿 + +> 端到端模仿不是"把所有模块塞进一张网络",而是一组共享的押注:信号强、可微、由模仿数据驱动、由规划损失指挥。这一条范式覆盖了 2022 年以来自动驾驶研究里最显眼的一批工作。 + +## 这条范式押注什么 + +1. **数据驱动而非知识驱动**:相信大规模真实驾驶日志蕴含的隐式经验比工程师手写的规则更细腻。 +2. **端到端可微**:相信梯度比模块边界更值钱;任何一个被工程师切分出来的中间任务,只要可以可微,就应该被纳入主干。 +3. **以规划为顶层目标**:相信感知、预测、占据等子任务的存在意义是"让规划更好",而非"完美地完成自己"。 +4. **模仿是入口**:相信收集人类驾驶日志比设计奖励函数容易;强化学习与偏好对齐留给后期阶段。 + +## 范式内的家族 / Members + +| 工作 | 子范式 | 关键差异 | +|---|---|---| +| [UniAD](paper_2212.10156_uniad.md) | dense BEV + 共享 query | 显式 5 个任务头共用 BEV | +| [PlanT](paper_2210.14222_plant.md) | 对象级稀疏 token | 赌"司机本来只看少数对象" | +| [VADv2](paper_vadv2.md) | 向量化 + 概率规划 | 用向量化替代 BEV,规划改成概率分布 | +| [TransFuser](paper_transfuser.md) | 多模态融合 BC | 早期奠基;融合 LiDAR-camera | +| [InterFuser](paper_interfuser.md) | 显式中间监督 | 在 CARLA 上加强可解释性 | +| [DriveVLM](paper_2402.12289_drivevlm.md) | 在 UniAD 之上接 VLM | 把语言推理叠在端到端骨架之上 | + +## 这条范式靠什么活下来 / Conditions that make it work + +* 大规模真实驾驶日志(nuScenes、Waymo Open Motion、Argoverse 2、内部车队) +* 可微 BEV 表征([BEVFormer](paper_li2022bevformer.md) 提供事实标准) +* 可微集合预测 query([DETR](paper_carion2020.md) 起源) +* 充足算力(≥ 8×A100 训一次) +* 离线指标 + 闭环仿真协同评估([nuPlan](paper_nuplan.md)、[NAVSIM](paper_navsim.md)、[CARLA Leaderboard](paper_carla_leaderboard.md)) + +## 当下未解决的痛点 / Open pain points + +* **闭环—离线差距**:高 L2/Collision 离线分数不能保证闭环安全。参见 [`problem:offline_metric_does_not_predict_closed_loop_performance`](problem_offline_metric_does_not_predict_closed_loop_performance.md)。 +* **罕见安全事件代表性不足**:[Counterfactual data-centric](paradigm_counterfactual_data_centric_safety.md) 范式正面回应。 +* **可解释性 vs 端到端**:解释头与 trace 链路是过渡方案,长期解法尚不清楚。 +* **算力—延迟权衡**:完整 UniAD 推理 ~1.8 FPS,工程化压缩仍在路上。 + +## 与其它范式的关系 + +* **互补**:[基础模型零样本驾驶 agent](paradigm_foundation_model_zero_shot_driving_agent.md) 用 VLM 接到这条范式的接口之上,得到 DriveVLM 一类的 dual-system 架构。 +* **对位**:[`essay:bitter_lesson`](essay_bitter_lesson.md) 会说所有的 hand-crafted prior(BEV、规划损失权重、任务划分)最终都会被纯数据驱动取代——这条范式承认这一前提,但停在"过渡阶段的实用主义"。 +* **对头**:[模块化感知到规划流水线](paradigm_modular_perception_to_planning_pipeline.md) 是这条范式想要替代的对象。 +* **替代**:[模型基于的世界模型想象规划](paradigm_model_based_world_imagination_planning.md) 是它的潜在继任者:当世界模型够强,闭环训练直接在想象里完成。 + +## 一条研究路径建议 / A starter trail + +1. 把 [BEVFormer](paper_li2022bevformer.md) 与 [DETR](paper_carion2020.md) 的卡片读熟,理解共享 query 与集合预测。 +2. 跑 [`lab03_uniad_query_intuition`](../../../labs/lab03_uniad_query_intuition.ipynb),亲手观察"是否共用 query"对联合性能的影响。 +3. 阅读 [PlanT 卡片](paper_2210.14222_plant.md),理解"稀疏对象级"作为对照实验。 +4. 把视线投向 [`validation:trace_unified_planning_oriented_e2e_driving`](validation_trace_unified_planning_oriented_e2e_driving.md),看看从更小的零件如何把这一切再推演一次。 +5. 最后跳到 [CF-VLA](paper_2512.24426_cfvla.md),看这条范式如何与反事实数据增广结合,回应"罕见事件"痛点。 diff --git a/docs/data/cards/extended/paradigm_foundation_model_zero_shot_driving_agent.md b/docs/data/cards/extended/paradigm_foundation_model_zero_shot_driving_agent.md new file mode 100644 index 0000000..5c57ad6 --- /dev/null +++ b/docs/data/cards/extended/paradigm_foundation_model_zero_shot_driving_agent.md @@ -0,0 +1,60 @@ +# 范式 · 基础模型作为零样本驾驶 agent + +> 这条范式押注:当视觉语言模型 (VLM) 与视觉语言动作模型 (VLA) 规模足够大、训练数据足够多样、对齐足够干净时,它们对"驾驶"这一任务可以做到几乎零样本的高语义理解,再由轻量动作头落地到控制信号。 + +## 押注 / Bet + +1. **语言是被高度压缩的世界模型**:人类用语言描述驾驶情形时已经隐含了大量物理常识、社会规则、风险评估。 +2. **预训练—对齐—蒸馏** 三段式:先在互联网规模上预训练,再用驾驶指令微调,最后蒸馏给实时小模型。 +3. **agent loop 是被工具调用与反思扩展的语言模型**:能感知、能规划、能批判自己的行为。 +4. **dual system 拼接** 让大模型只在该慢的时候介入,平衡延迟与质量。 + +## 家族成员 + +| 工作 | 角色 | +|---|---| +| [GPT-3](paper_gpt3.md) | scaling laws 与 in-context learning 的奠基 | +| [LLaVA / Qwen-VL](paper_llava.md) | 把视觉接到语言模型的开源样板 | +| [Agent-Driver](paper_2311.10813_agent_driver.md) | LLM 作为驾驶决策核心的第一份完整工作 | +| [DiLu](paper_2309.16292_dilu.md) | 知识驱动 + 反思循环 | +| [DriveVLM / DriveVLM-Dual](paper_2402.12289_drivevlm.md) | dual-system 范式的代表 | +| [LMDrive](paper_lmdrive.md) | 把语言作为感知与规划的中介 | +| [RT-1, RT-2, RT-X, OpenVLA, Octo](paper_rt2.md) | 机器人侧的 VLA 蒸馏对比 | +| [CF-VLA](paper_2512.24426_cfvla.md) | 把反事实数据闭环接进 VLA | + +## 必备组件 / Building blocks + +* **scaling laws 实证**:[`insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models`](insight_scaling_data_unlocks_capabilities_not_present_in_smaller_models.md)。 +* **多模态对齐**:`move:plug_in_modality_encoder_to_frozen_language_model_via_projection`。 +* **指令微调 + RLHF**:[`paper:rlhf_dpo`](paper_rlhf_dpo.md)。 +* **工具调用循环**:`move:wrap_language_model_with_tool_calling_loop`。 +* **反思**:`move:add_reflection_step_so_agent_critiques_its_own_output`。 +* **动作 tokenization**:`move:cast_continuous_action_as_discretized_token_sequence`。 +* **快慢双回路**:[`insight:dual_system_fast_slow_loop_marries_reactive_and_deliberative_control`](insight_dual_system_fast_slow_loop_marries_reactive_and_deliberative_control.md)。 +* **大模型蒸馏与延迟优化**:[`move:distill_large_VLM_into_small_realtime_specialist`](move_distill_large_VLM_into_small_realtime_specialist.md)。 + +## 它正面回应什么痛点 + +- [`problem:long_horizon_reasoning_with_finite_context_window`](problem_long_horizon_reasoning_with_finite_context_window.md) +- [`problem:counterfactual_reasoning_about_other_agents_intent`](problem_counterfactual_reasoning_about_other_agents_intent.md) +- [`problem:zero_shot_generalization_to_unseen_driving_scenes`](problem_zero_shot_generalization_to_unseen_driving_scenes.md) +- [`problem:auditability_of_decisions_for_regulatory_compliance`](problem_auditability_of_decisions_for_regulatory_compliance.md)(语言中介自带 trace) + +## 主要悬而未决 + +1. **幻觉风险**:[`problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop`](problem_hallucinated_action_from_vision_language_model_in_safety_critical_loop.md)。 +2. **细粒度空间精度**:[`problem:fine_grained_spatial_understanding_in_vision_language_model`](problem_fine_grained_spatial_understanding_in_vision_language_model.md)。 +3. **延迟**:1B+ 大模型推理速度 < 30 Hz 时难做主回路。 +4. **接地问题**:[`problem:grounding_language_token_to_continuous_physical_world`](problem_grounding_language_token_to_continuous_physical_world.md)。 + +## 跟其它范式的关系 + +- **互补**:可以叠加在 [可微端到端模仿](paradigm_differentiable_end_to_end_imitation.md) 之上作为慢回路。 +- **对位**:与 [模型基于世界模型想象规划](paradigm_model_based_world_imagination_planning.md) 在"显式物理 vs 语言压缩"上有分歧。 +- **争锋**:[`essay:bitter_lesson`](essay_bitter_lesson.md) 是这条范式最大的同盟,因为它鼓励 scaling 与通用方法。但 *把语言作为接口* 仍然是一种人为先验,Sutton 可能会觉得未来会被纯像素 token 取代。 + +## 推荐起步 + +1. [GPT-3 卡片](paper_gpt3.md) → [LLaVA 卡片](paper_llava.md) → [DriveVLM 卡片](paper_2402.12289_drivevlm.md)。 +2. 跑 [`labs/lab07_dilu_llm_decision_loop`](../../../labs/lab07_dilu_llm_decision_loop.ipynb) 与 [`labs/lab08_agent_driver_tool_calling`](../../../labs/lab08_agent_driver_tool_calling.ipynb)。 +3. 读 [`validation:trace_vision_language_action_dual_loop`](validation_trace_vision_language_action_dual_loop.md) 与 [`validation:trace_counterfactual_vla_replanner`](validation_trace_counterfactual_vla_replanner.md)。 diff --git a/docs/data/cards/extended/paradigm_knowledge_driven_reflective_agent.md b/docs/data/cards/extended/paradigm_knowledge_driven_reflective_agent.md new file mode 100644 index 0000000..3031180 --- /dev/null +++ b/docs/data/cards/extended/paradigm_knowledge_driven_reflective_agent.md @@ -0,0 +1,62 @@ +# 范式 · 知识驱动反思智能体 + +> 它跟"基础模型零样本驾驶 agent"是孪生但视角不同:基础模型路线赌"数据规模 + 涌现",这条范式赌"显式知识 + 反思循环 + 工具"——即使在小模型上,也能凭知识结构 + 自我批评得到稳健决策。 + +## 押注 / Bet + +1. **驾驶常识可以被显式表达**:交通规则、礼让模式、典型场景库可写成结构化知识。 +2. **反思循环让小模型变强**:让模型批判自己上一次决策,往往比直接采样更准。 +3. **工具使用是接地点**:把决策的物理后果交给精确的工具(地图、运动学、规则库),让 LM 只负责高层意图。 +4. **可解释 trace 是产物**:每一步决策的思考过程都可记录,作为合规与教育资产。 + +## 家族成员 + +| 工作 | 角色 | +|---|---| +| [DiLu](paper_2309.16292_dilu.md) | 知识驱动 + 反思的开山之作 | +| [Agent-Driver](paper_2311.10813_agent_driver.md) | 把 perception/memory/planning 作为可调用工具 | +| [ReAct](paper_react.md) | reason-act-observe 循环的奠基 | +| [Reflexion](paper_reflexion.md) | 错误—反思—修正的形式化 | +| [Tree-of-Thoughts](paper_tot.md) | 在思考过程中显式搜索 | +| [Toolformer](paper_toolformer.md) | 让 LM 学会调用工具 | +| [SwiftSage](paper_swiftsage.md) | 快慢双系统在 agent 上的实例 | + +## 必备组件 + +* **知识结构**:交规、地图、典型案例库、车辆动力学约束。 +* **反思机制**:让模型在每一步决策后被强制"批评自己",输出一组改进点。 +* **工具调用接口**:明确定义 inputs / outputs,能让 LM 调用并解析结果。 +* **CoT 与搜索的混合**:浅层用 CoT,深层用 ToT。 +* **记忆系统**:短期场景记忆 + 长期经验记忆 + 检索机制。 +* **可观察的 trace**:每一步思考、调用、决策都被记录。 + +## 推荐前置阅读 + +- [`concept:cot`](../../concepts.md#chain-of-thought) +- [`concept:tool_use`](../../concepts.md#tool-use) +- [`insight:agent_loop_is_just_iterated_conditional_generation`](insight_agent_loop_is_just_iterated_conditional_generation.md) + +## 它正面回应 + +- [`problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop`](problem_hallucinated_action_from_vision_language_model_in_safety_critical_loop.md):反思 + 工具调用是已知最好的 hallucination 屏障之一。 +- [`problem:auditability_of_decisions_for_regulatory_compliance`](problem_auditability_of_decisions_for_regulatory_compliance.md):trace 自带审计价值。 +- [`problem:counterfactual_reasoning_about_other_agents_intent`](problem_counterfactual_reasoning_about_other_agents_intent.md):CoT + ToT 让"假如对方那样开"成为可表达的查询。 + +## 它跟其它范式的关系 + +- **互补**:可以作为 [基础模型零样本驾驶 agent](paradigm_foundation_model_zero_shot_driving_agent.md) 的 *运行时增强*——同一个底层 VLM,加上反思与工具,得到更稳的 agent。 +- **对位**:与 [`essay:bitter_lesson`](essay_bitter_lesson.md) 短期内争锋——后者会预测"知识结构最终被通用模型吸收"。这条范式的辩护:在 *安全敏感* 应用中,显式知识仍然有不可替代的可解释收益。 +- **协作**:与 [闭环数据引擎](paradigm_closed_loop_data_engine_centric_development.md) 互补——反思 trace 可被自动挖掘成新的训练样本。 + +## 主要悬而未决 + +1. **反思的边际收益曲线**:何时反思才值得算力? +2. **知识结构的自动获取**:能否让模型从日志中自学交规? +3. **工具失败的鲁棒性**:当一个工具返回错误结果,agent 怎样发现? +4. **延迟预算**:每次决策若都跑 3 轮反思,hard real-time 难。 + +## 推荐起步 + +1. [DiLu 卡片](paper_2309.16292_dilu.md) 与 [Agent-Driver 卡片](paper_2311.10813_agent_driver.md)。 +2. 跑 [`labs/lab07_dilu_llm_decision_loop`](../../../labs/lab07_dilu_llm_decision_loop.ipynb) 与 [`labs/lab08_agent_driver_tool_calling`](../../../labs/lab08_agent_driver_tool_calling.ipynb)。 +3. 读 ReAct 与 Reflexion 卡片,对比通用 agent 文献。 diff --git a/docs/data/cards/extended/paradigm_model_based_world_imagination_planning.md b/docs/data/cards/extended/paradigm_model_based_world_imagination_planning.md new file mode 100644 index 0000000..aa0d6a9 --- /dev/null +++ b/docs/data/cards/extended/paradigm_model_based_world_imagination_planning.md @@ -0,0 +1,59 @@ +# 范式 · 把规划放到世界模型的想象里 + +> 这条范式的核心押注是:与其在真实环境里冒着代价试错,不如先学一个可以以假乱真的内部模拟器(世界模型),在它的想象里搜索最优策略。它在自动驾驶里正在从远景变成可工程化的方案。 + +## 押注什么 / The bet + +1. **视频/驾驶日志蕴含的物理规律可以被压缩到一个潜在的状态—动作—未来模型里**。 +2. **当世界模型保真度过门槛后**,在它内部跑大量 rollout 比在真实环境里跑更便宜、更安全、更可控。 +3. **罕见安全事件可以被生成**:通过条件化世界模型,可以人为提高 corner case 的密度。 +4. **规划可以被改成"在模型梦境里搜索"**:MPC / MCTS / RL all 都可以在 imagination 上进行。 + +## 家族成员 / Members + +| 工作 | 提供的零件 | +|---|---| +| [World Models (Ha & Schmidhuber 2018)](paper_world_models.md) | 把这一思路在像素级别上证伪 | +| [Dreamer V1–V3](paper_dreamer_v3.md) | latent imagination 训练,证明从图像观测可以恢复出可用世界模型 | +| [MuZero](paper_muzero.md) | 在没有显式环境模型时也能蒸馏出一个隐式模型 + 搜索 | +| [GAIA-1](paper_gaia1.md) | 视频生成世界模型应用到驾驶 | +| [DriveDreamer](paper_drivedreamer.md) | 控制条件化的驾驶视频扩散世界模型 | +| [Cosmos](paper_cosmos.md) | NVIDIA 的物理世界基础模型 | +| [CF-VLA](paper_2512.24426_cfvla.md) | 把世界模型作为反事实重规划的批评者 | + +## 想要进这条范式必须先齐的零件 / Prerequisites + +* **可微的潜在动力学**:由 [`move:learn_world_model_then_plan_in_latent_imagination`](move_learn_world_model_then_plan_in_latent_imagination.md) 提供。 +* **视频生成模型的稳定训练秘诀**:扩散模型、tokenizer、长上下文。 +* **行动到画面的条件化**:[`move:condition_video_generative_model_on_control_action_for_world_model`](move_condition_video_generative_model_on_control_action_for_world_model.md)。 +* **离线 + 闭环联合度量**:[`problem:offline_metric_does_not_predict_closed_loop_performance`](problem_offline_metric_does_not_predict_closed_loop_performance.md) 让这条范式天然有吸引力——因为它直接消解这一痛点。 +* **算力**:训一个 1B+ 视频世界模型需要可观资源。 + +## 这条范式打开了什么 / What it unlocks + +- **罕见事件的可定向合成**:直接在世界模型里生成"前车突然紧急刹车 + 你身后有摩托"这种长尾。 +- **闭环训练而不撞车**:RL 在想象里跑,policy 在真实日志上微调。 +- **跨车队、跨地区的零样本试驾**:用相同世界模型在不同城市数据下做 cookbook 化的策略对比。 +- **可解释的 what-if 反事实**:world model 自身就是 counterfactual 引擎。 + +## 它和谁争锋 / Where it pushes back + +- vs [可微端到端模仿](paradigm_differentiable_end_to_end_imitation.md):后者依赖真实数据的稠密信号,前者依赖 imagination 的密度。两条线在 [CF-VLA](paper_2512.24426_cfvla.md) 里短暂合流。 +- vs [基础模型零样本驾驶 agent](paradigm_foundation_model_zero_shot_driving_agent.md):是否需要"显式的物理世界模型"还是"大语言模型隐式压缩世界知识"。 +- vs [模块化感知到规划](paradigm_modular_perception_to_planning_pipeline.md):模块化范式相信"工程切分稳定",世界模型范式相信"梦境里的端到端搜索"。 + +## 主要悬而未决 / Open challenges + +1. **长时一致性**:当前视频世界模型在 8 秒之后开始漂移。 +2. **物理一致性**:行人意图、轮胎抓地力等需要嵌入硬约束。 +3. **可控性 vs 多样性**:condition 太强失多样、太弱失精度。 +4. **延迟**:当前 1B+ 世界模型推理速度无法满足 30 Hz 控制。 +5. **评估指标**:怎样判断一个世界模型 *作为规划环境* 是合格的? + +## 推荐一条起步路径 / Starter trail + +1. [World Models 1.0 卡片](paper_world_models.md):奠基直觉。 +2. [GAIA-1 卡片](paper_gaia1.md):第一份大规模驾驶视频世界模型证据。 +3. 跑一个最小 Dreamer 复现,验证 imagination rollout 可用。 +4. 读 [CF-VLA](paper_2512.24426_cfvla.md) 看世界模型作为评估器的现实应用。 +5. 把它跟 [`paradigm:differentiable_end_to_end_imitation`](paradigm_differentiable_end_to_end_imitation.md) 拼成 hybrid。 diff --git a/docs/data/cards/extended/paradigm_modular_perception_to_planning_pipeline.md b/docs/data/cards/extended/paradigm_modular_perception_to_planning_pipeline.md new file mode 100644 index 0000000..66dc3d3 --- /dev/null +++ b/docs/data/cards/extended/paradigm_modular_perception_to_planning_pipeline.md @@ -0,0 +1,61 @@ +# 范式 · 模块化感知到规划的流水线 + +> 这是自动驾驶最早成立、也是工业界部署最广的范式。它的核心立场是:把驾驶任务切成感知、定位、预测、规划、控制五个独立模块,每个都有明确的输入输出与可测试的接口。 + +## 押注 / Bet + +1. **工程可分解性比理论最优更值钱**:能让 200 人团队同时迭代的架构胜过黑盒。 +2. **可测性与可审计性是合规前提**:每个模块的输入输出都有独立指标和回归测试。 +3. **故障可定位**:当系统出错,错的是某一个模块,可以独立分析与修复。 +4. **领域知识可显式注入**:地图、交规、车辆动力学都可在对应模块中作为先验编码。 + +## 家族成员 / Members + +| 工作 / 平台 | 角色 | +|---|---| +| [Apollo / Autoware](paper_apollo_autoware.md) | 开源模块化栈的事实标准 | +| [Tesla AI Day 早期](paper_tesla_ai_day.md) | "向量空间"分层 | +| Waymo 早期堆栈 | 高精地图 + 模块化栈的工业先例 | +| nuPlan 基线方案 (IDM, PDM-Closed) | 模块化规划基线 | +| [PointPillars / VoxelNet](paper_pointpillars.md) | 模块化感知典型组件 | +| 卡尔曼 / 粒子滤波 跟踪 | 经典跟踪模块 | + +## 必备组件 / Building blocks + +* **感知模块**:3D 检测、语义分割、深度估计 +* **定位模块**:HD 地图匹配、SLAM、GNSS 融合 +* **跟踪模块**:多目标关联,可用 Hungarian、JPDA、Kalman +* **预测模块**:物体未来轨迹分布,可用社交 LSTM、CoverNet、MTR +* **规划模块**:路径规划(A*、RRT*)+ 行为规划(FSM、POMDP)+ 轨迹优化(QP、iLQR) +* **控制模块**:PID / MPC / LQR +* **接口设计**:每个模块输出的"中间表示"必须可视化、可校验、可独立测试 + +## 它正面回应的痛点 + +- **可审计性**:[`problem:auditability_of_decisions_for_regulatory_compliance`](problem_auditability_of_decisions_for_regulatory_compliance.md) 与 [`paradigm:safety_by_constraint_layered_architecture`](paradigm_safety_by_constraint_layered_architecture.md) 互补。 +- **工程化部署**:当算力或验证预算紧时,模块化允许独立加速、独立验证。 + +## 它跟其它范式的关系 + +- **被替代候选**:[可微端到端模仿](paradigm_differentiable_end_to_end_imitation.md) 想溶解模块边界;当下游信号够强、算力够大时,模块化的边际收益递减。 +- **共生**:实际部署里两条范式常常并存——端到端 backbone 输出最终 trajectory,但保留模块化的可解释中间表征用于调试与合规。 +- **被批评**:[`essay:bitter_lesson`](essay_bitter_lesson.md) 与 [`insight:end_to_end_differentiable_beats_handcraft_when_signal_strong`](insight_end_to_end_differentiable_beats_handcraft_when_signal_strong.md) 都暗示模块边界终将让位。 + +## 主要悬而未决 + +1. **误差累积**:上游模块的小错经过 5 层放大成最终决策的大错。 +2. **接口设计的固化**:当一个新研究方向(如占据预测)需要打破已有接口,整条流水线被迫共同迭代。 +3. **多模块联合优化**:跨模块梯度不可微,所有指标只能分模块独立。 + +## 当代演化方向 + +- **半端到端**:[InterFuser](paper_interfuser.md)、[ThinkTwice](paper_thinktwice.md) 保留模块化结构但联合训练。 +- **可微接口**:在传统接口外加上 differentiable proxy,让上游模块知道下游的偏好。 +- **数据引擎注入**:[`paradigm:closed_loop_data_engine_centric_development`](paradigm_closed_loop_data_engine_centric_development.md) 在模块化范式上叠加,重点提升单模块质量。 + +## 推荐起步 + +1. 看 [Apollo / Autoware 文档](paper_apollo_autoware.md),理解工业级模块拆分。 +2. 跑一个最小 SLAM + 检测 + Kalman 跟踪的 demo(如 NuScenes Devkit 自带)。 +3. 读 [Tesla AI Day 2022](paper_tesla_ai_day.md) 对照"工业实践到端到端"的过渡。 +4. 跳到 [UniAD](paper_2212.10156_uniad.md) 看模块化怎样被部分溶解。 diff --git a/docs/data/cards/extended/paradigm_scaling_data_with_self_supervision.md b/docs/data/cards/extended/paradigm_scaling_data_with_self_supervision.md new file mode 100644 index 0000000..5959f74 --- /dev/null +++ b/docs/data/cards/extended/paradigm_scaling_data_with_self_supervision.md @@ -0,0 +1,61 @@ +# 范式 · 用自监督把数据规模化 + +> 当下游标签昂贵且品类长尾,研究者把希望寄托在"无标注数据是天然丰富的,只要找到一种把它榨出监督信号的方式"。这条范式过去五年是视觉与语言两条线共同的引擎。 + +## 押注 / Bet + +1. **绝大多数数据可以免费**:百万小时驾驶视频、互联网图像、未标注 LiDAR。 +2. **结构本身就是监督**:掩码预测、对比学习、几何一致性、时间预测,都不需要人工标注。 +3. **预训练 → finetune** 的两段架构稳定可工程化。 +4. **基础特征会迁移**:当骨干够好,下游任务只需少量标签就能达到顶峰。 + +## 家族成员 / Members + +| 工作 | 提供的零件 | +|---|---| +| [SimCLR / MoCo v3](paper_simclr_moco.md) | 对比学习的稳定配方 | +| [BYOL / SimSiam](paper_byol.md) | 非对比的 siamese 自监督 | +| [DINO / DINOv2 / DINOv3](paper_2508.10104_dinov3.md) | 自蒸馏 + 多视图,得到强大零样本特征 | +| [MAE / BEiT](paper_mae.md) | 掩码图像建模 | +| [BERT / GPT-3](paper_gpt3.md) | 把同一配方做到语言上 | +| [CLIP](paper_clip.md) | 跨模态对比 | +| [VICReg / Barlow Twins](paper_vicreg.md) | 信息论视角的非对比 | + +## 必备组件 / Building blocks + +* **对比 / 掩码 / 自蒸馏的任一稳定配方**:参见 [`insight:masked_prediction_yields_self_supervised_signal`](insight_masked_prediction_yields_self_supervised_signal.md) 与 `insight:contrastive_alignment_creates_zero_shot_transfer`。 +* **数据增广策略**:随机裁剪、颜色抖动、时间抽样。 +* **大批量训练稳定的工程**:LARS / LAMB 优化器、混合精度、ZeRO 切片。 +* **特征评估协议**:linear probing、KNN、few-shot finetune。 +* **可迁移的下游任务**:检测、分割、占据、规划。 + +## 在自动驾驶里的具体投影 + +- 视觉骨干:百万小时驾驶视频 → DINOv3 自监督预训练 → 接 BEVFormer head → finetune at 1/10 标签量。 +- LiDAR 自监督:用点云遮挡预测、场景流自监督。 +- 多模态对齐:相机—LiDAR—轨迹三者的对比学习。 +- 闭环 self-play 预训练:用世界模型生成大规模仿真数据做自监督。 + +## 它正面回应的痛点 + +- [`problem:label_efficiency_for_3d_annotation`](problem_label_efficiency_for_3d_annotation.md) +- [`problem:long_tail_object_categories_in_open_world`](problem_long_tail_object_categories_in_open_world.md) +- [`problem:label_noise_for_3d_object_categories`](problem_label_noise_for_3d_object_categories.md) + +## 主要悬而未决 + +1. **驾驶视频与互联网视频差距**:通用 SSL 骨干迁移到驾驶仍有 gap,是该重训练还是该联合训练? +2. **多模态 SSL 的协议设计**:相机 / LiDAR / 雷达 / GPS / IMU 怎样联合自监督? +3. **闭环安全 finetune 的稳定性**:SSL 特征在 RL finetune 下是否会被破坏? + +## 跟其它范式的关系 + +- **互补**:是 [可微端到端模仿](paradigm_differentiable_end_to_end_imitation.md) 的天然预训练阶段。 +- **基础**:是 [基础模型零样本驾驶 agent](paradigm_foundation_model_zero_shot_driving_agent.md) 的隐性前提(视觉编码器都来自 SSL)。 +- **对位**:与 [类脑神经形态协同设计](paradigm_brain_inspired_neuromorphic_co_design.md) 短期争算力,长期可以叠加。 + +## 推荐起步 + +1. [DINOv3 卡片](paper_2508.10104_dinov3.md) → [DINOv2 卡片](paper_dinov2.md)。 +2. 跑 [`labs/lab05_dinov3_features_minidata`](../../../labs/lab05_dinov3_features_minidata.ipynb)。 +3. 读 [`insight:masked_prediction_yields_self_supervised_signal`](insight_masked_prediction_yields_self_supervised_signal.md)。 diff --git a/docs/data/cards/extended/problem_offline_metric_does_not_predict_closed_loop_performance.md b/docs/data/cards/extended/problem_offline_metric_does_not_predict_closed_loop_performance.md new file mode 100644 index 0000000..88eeddd --- /dev/null +++ b/docs/data/cards/extended/problem_offline_metric_does_not_predict_closed_loop_performance.md @@ -0,0 +1,55 @@ +# 悬而未决 · 离线指标不能预测闭环性能 + +> 这是自动驾驶规划研究里被反复观察到、却长期没有干净解法的开放问题。它影响每一项依赖离线评估的研究决策。 + +## 现象 / Observation + +在 nuScenes、Waymo Open Motion 等离线数据集上: +- L2 轨迹误差越小不代表碰撞率越低 +- 碰撞率越低不代表闭环驾驶通过率越高 +- 多任务损失曲线下降不代表实车安全 +- 自监督预训练 + finetune 显著改善离线指标,却可能在闭环里劣化 + +学界已有多份独立观察:[InterFuser](paper_interfuser.md)、[ThinkTwice](paper_thinktwice.md)、[Tesla AI Day](paper_tesla_ai_day.md) 的复盘,以及 [nuPlan 挑战赛](paper_nuplan.md) 的官方裁定。 + +## 它为什么难解 + +闭环性能由"模型在它自己制造的状态序列中的累积行为"决定。离线指标只在"专家产生的状态分布"上度量。这正是 [`concept:covariate_shift`](../../concepts.md#协变量偏移) 的另一面: + +$$\text{离线指标}(\pi)\ \approx\ \mathbb{E}_{s \sim p_\text{expert}}\!\big[\,\ell(\pi(s), a_\text{expert}(s))\,\big]$$ +$$\text{闭环表现}(\pi)\ =\ \mathbb{E}_{s \sim p_\pi}\!\big[\,\text{event}(s)\,\big]$$ + +只要 $p_\pi \neq p_\text{expert}$,两个量就没有定量关系。罕见安全事件让差距更夸张:闭环风险来自数据集里几乎不出现的状态。 + +## 现有半解法 / Partial mitigations in the atlas + +| 半解法 | 怎样接近 | 它没解决什么 | +|---|---|---| +| 闭环仿真器 ([CARLA](paper_carla_leaderboard.md)、[nuPlan](paper_nuplan.md)、[MetaDrive](paper_metadrive.md)) | 在仿真里跑闭环,跟离线分数对照 | 仿真器与现实仍有差距,他车行为质量是上界 | +| [DAgger](paper_ross2011_dagger.md) 数据聚合 | 让模型自身生成状态再请专家纠正 | 实车上拿专家修正成本极高 | +| [反事实数据增广](paradigm_counterfactual_data_centric_safety.md) | 用世界模型生成自身分布下的危险场景 | 世界模型保真度还在迭代 | +| [世界模型想象规划](paradigm_model_based_world_imagination_planning.md) | 把规划放到模型自己的 imagination 里 | 需要高保真世界模型才能成立 | +| 离线—闭环联合评分 (nuPlan-style) | 把多种指标线性组合 | 权重选择仍是经验艺术 | +| [`move:track_metric_correlation_offline_vs_closed_loop_to_select_models`](move_track_metric_correlation_offline_vs_closed_loop_to_select_models.md) | 每个发布周期度量相关性,淘汰高度不一致的离线代理 | 只是发现问题、不解决问题 | + +## 这个痛点会破坏哪些工作 / Affected research lines + +- 任何只报离线 L2/Collision 的端到端规划论文 +- 任何用 nuScenes 训练但用 CARLA 评估的工作(其实是两个不同的离线指标比较) +- 任何不在闭环里跑大量 seed 的强化学习类驾驶研究 + +## 它打开了哪些研究突破口 + +下面这几条 *研究方向* 都把这一问题当作起点: + +1. **可证可信的闭环代理**:找到一类闭环安全的可证性度量,使其与离线代理之间有形式化的 lower-bound 关系。 +2. **世界模型作为代理评估器**:用 [GAIA-1](paper_gaia1.md)/[DriveDreamer](paper_drivedreamer.md) 一类的视频世界模型生成闭环 rollout,对比真车跑分。 +3. **反事实压力测试**:在 [CF-VLA](paper_2512.24426_cfvla.md) 思路下,用合成对抗场景定向探查模型边界。 +4. **离线分数的多目标剖分**:把 L2/Collision 拆成"专家分布上的拟合"与"自我分布上的安全"两个独立目标,并显式约束它们的差距。 +5. **闭环回放正则化**:在训练时直接惩罚模型在自我状态分布上的失败。 + +## 它跟其它节点的连接 + +- 反向:被多份开创性工作 [UniAD](paper_2212.10156_uniad.md)、[VADv2](paper_vadv2.md)、[CF-VLA](paper_2512.24426_cfvla.md) 选作主要痛点。 +- 正向:动机驱动 `move:design_closed_loop_metric_correlated_with_real_world_safety`,进而推出 [闭环数据引擎为中心的开发范式](paradigm_closed_loop_data_engine_centric_development.md)。 +- 平行:跟 [`problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented`](problem_rare_safety_critical_events_dominate_real_risk_but_are_under_represented.md) 互为孪生:罕见性是离线指标失真的最主要源。 diff --git a/docs/data/cards/extended/validation_trace_alpha_zero_self_play_with_mcts_guided_policy.md b/docs/data/cards/extended/validation_trace_alpha_zero_self_play_with_mcts_guided_policy.md new file mode 100644 index 0000000..efd6cdd --- /dev/null +++ b/docs/data/cards/extended/validation_trace_alpha_zero_self_play_with_mcts_guided_policy.md @@ -0,0 +1,50 @@ +# 再发现 · AlphaZero(自我对弈 + MCTS 引导的策略) + +> 如果 DeepMind 的 AlphaZero 系列论文从未发表,但图谱里其它素材齐全,下面这张组件清单足以让研究者必然推出 AlphaZero 这套框架。 + +## 必备构件 + +| 类别 | 节点 | 角色 | +|---|---|---| +| 概念 | [`concept:mdp`](../../concepts.md#mdp) | 把游戏写成 (S, A, T, R) | +| 概念 | `concept:minimax` | 两人零和游戏的解法骨架 | +| 移动 | `move:learn_policy_value_jointly_with_shared_trunk` | 双头网络(policy + value)共享 backbone | +| 移动 | `move:use_self_play_to_generate_unlimited_training_signal` | 自己跟自己下,得到无限数据 | +| 移动 | `move:bootstrap_target_network_to_stabilize_off_policy_learning` | 让 policy 更新与目标 policy 解耦 | +| 移动 | `move:mcts_to_guide_policy_via_test_time_search` | 蒙特卡洛树搜索作为测试时的强化 | +| 洞察 | [`insight:test_time_compute_substitutes_train_time_via_search`](insight_test_time_compute_substitutes_train_time_via_search.md) | 决定性想法:"搜索就是测试时的训练" | +| 洞察 | `insight:bootstrap_self_improvement_loop` | "今天的我训练明天的我"循环可以收敛 | +| 论文 | [`paper:mnih2015_dqn`](paper_mnih2015_dqn.md) | 提供深度 RL 的可行性证据 | +| 问题 | `problem:reward_specification_for_safe_polite_driving`(驾驶投影下) | 在游戏里:胜负是稀疏奖励,长 horizon 难 | + +## 推演逻辑 + +1. 起点是"有简明胜负规则的对抗游戏"——这一情境天然适合 MDP + Minimax。 +2. 由 [`paper:mnih2015_dqn`](paper_mnih2015_dqn.md) 知道:深度网络可以作为 Q-function。 +3. 由 `move:learn_policy_value_jointly_with_shared_trunk` 推出:policy 与 value 共用一个 backbone,互相提升。 +4. 由 `move:use_self_play_to_generate_unlimited_training_signal` 推出:让网络自己对弈生成数据。 +5. 关键跳跃:由 [`insight:test_time_compute_substitutes_train_time_via_search`](insight_test_time_compute_substitutes_train_time_via_search.md) 推出:把 MCTS 嵌入测试时,用其访问统计修正 policy。 +6. 把整套放进 `insight:bootstrap_self_improvement_loop`:训练 policy → 搜索改进 → 训练新 policy → …… + +→ 必然得到 AlphaZero。 + +## 自动驾驶里的同型再发明 + +把这条 trace 投到驾驶: +- 自我对弈 → 用世界模型与多车交互生成 closed-loop rollout +- MCTS → 在世界模型里搜索 ego 动作序列 +- policy + value → ego 策略 + 风险评分网络 +- 自我改进循环 → [`paradigm:model_based_world_imagination_planning`](paradigm_model_based_world_imagination_planning.md) + +结果即是 *把 AlphaZero 哲学投到驾驶* 的雏形:让模型在 imagination 里反复迭代,得到比模仿学习更强的策略。 + +## 它没能预测到什么 + +AlphaZero 的训练算力对一般实验室难以负担;从 self-play 跳到 *从人类数据 bootstrap* 的工作流(如 AlphaGo 第一版)依赖了真实人类对弈数据。这两者是后续 [`paradigm:offline_rl`](paradigm_offline_rl.md) 与 [`paradigm:imitation_learning`](paradigm_imitation_learning.md) 等独立线条。 + +## 推荐起步 + +1. [AlphaZero 卡片](paper_silver2017_alphazero.md)。 +2. 写一个最小 MCTS:6×6 五子棋 + 浅层 policy/value 网络。 +3. 读 [`paper:muzero`](paper_muzero.md) 看怎样省去显式环境模型。 +4. 把这条思路投到驾驶 [`paradigm:model_based_world_imagination_planning`](paradigm_model_based_world_imagination_planning.md)。 diff --git a/docs/data/cards/extended/validation_trace_self_attention_replaces_recurrence.md b/docs/data/cards/extended/validation_trace_self_attention_replaces_recurrence.md new file mode 100644 index 0000000..e05d13e --- /dev/null +++ b/docs/data/cards/extended/validation_trace_self_attention_replaces_recurrence.md @@ -0,0 +1,51 @@ +# 再发现 · 自注意力取代循环(Transformer 的推演) + +> 假设 Vaswani 等 2017 那篇短小精悍的论文没发表,图谱里其它节点齐全的情况下,怎样一步步必然得到 Transformer。 + +## 必须先齐的构件 / Components + +| 类别 | 节点 | 在重新发明里的角色 | +|---|---|---| +| 概念 | [`concept:self_attention`](../../concepts.md#self-attention) | 让序列内任意两点之间可以直接通信 | +| 概念 | `concept:positional_encoding` | 把无序的 attention 输入恢复成有序序列 | +| 概念 | `concept:layer_normalization` | 让深层堆叠稳定可训 | +| 移动 | `move:residual_connection` | 让深度堆叠的梯度通畅([ResNet](paper_he2015_resnet.md) 提供) | +| 移动 | `move:tokenize_continuous_signal_to_use_transformer` (待催生) | 让任意模态都能塞进序列范式 | +| 移动 | `move:scale_pretraining_then_fine_tune` | 解释为什么自注意力会值得放大 | +| 洞察 | [`insight:attention_is_typed_entity_communication`](insight_attention_is_typed_entity_communication.md) | 揭示自注意力的物理含义 | +| 洞察 | `insight:bottleneck_information_is_recoverable_when_global_communication_is_cheap` | 解释为什么 RNN 的"瓶颈隐状态"是不必要的 | +| 问题 | `problem:long_range_dependency_is_hard_with_recurrence` | 直接动机 | +| 问题 | `problem:training_recurrence_is_serial_and_slow` | 工程动机 | +| 论文 | `paper:bahdanau2014_attention` | 早期"注意力作为对齐"的奠基 | +| 论文 | [`paper:he2015_resnet`](paper_he2015_resnet.md) | 残差结构作为前置 | + +## 推演逻辑 + +1. **认定循环架构有结构缺陷**:由 `problem:long_range_dependency_is_hard_with_recurrence` 与 `problem:training_recurrence_is_serial_and_slow` 双痛点驱动。 +2. **接受"注意力作为对齐"的工具**:由 `paper:bahdanau2014_attention` 提供。 +3. **泛化到 self-attention**:让 query/key/value 来自同一序列。这一步只需要把"对齐"从跨语言转向"序列内"——一旦认识到 `insight:attention_is_typed_entity_communication`,这一推广是显然的。 +4. **去掉循环**:由 `insight:bottleneck_information_is_recoverable_when_global_communication_is_cheap` 给出勇气。 +5. **加上 positional encoding 与 layer norm**:让无序到有序、深层到稳定,由 `concept:positional_encoding` 与 `concept:layer_normalization` 提供。 +6. **堆叠并加 residual**:[`paper:he2015_resnet`](paper_he2015_resnet.md) 提供。 +7. **scaling 与 pretraining**:由 `move:scale_pretraining_then_fine_tune` 给出。 + +每一步都在图谱里已有独立节点。把它们串起来,必然得到 multi-head self-attention + residual + LN + positional encoding + 堆叠的架构——也就是 Transformer。 + +## 自动驾驶里的同型再发明 + +把上述推演投射到 BEV 感知: +- 序列 = BEV 上的格点 +- 注意力 = 跨格点的查询 +- 残差 = 跨任务头的残差注入 + +→ 自然得到 [BEVFormer](paper_li2022bevformer.md) / [DETR3D](paper_detr3d.md) / [UniAD](paper_2212.10156_uniad.md) 这条线。 + +## 它没能预测到什么 + +Transformer 的 in-context learning 能力、scaling laws、emergent capabilities,是 *在做出 Transformer 之后* 才被发现的。这些后续洞察构成下一代 validation——参见 [`validation:trace_few_shot_in_context_learning_at_scale`](validation_trace_few_shot_in_context_learning_at_scale.md)。 + +## 推荐起步 + +1. [Transformer 卡片](paper_vaswani2017.md) → [ViT 卡片](paper_vit.md) → [DETR 卡片](paper_carion2020.md)。 +2. 跑一个最小 attention 实现,对比一个最小 LSTM 实现,体会两条路线的内在差异。 +3. 把这条推演投到 BEV 感知上,看你怎样独立再发明 [BEVFormer](paper_li2022bevformer.md)。 diff --git a/docs/data/cards/extended/validation_trace_set_prediction_with_object_queries.md b/docs/data/cards/extended/validation_trace_set_prediction_with_object_queries.md new file mode 100644 index 0000000..07efed2 --- /dev/null +++ b/docs/data/cards/extended/validation_trace_set_prediction_with_object_queries.md @@ -0,0 +1,47 @@ +# 再发现 · 用对象 query 做集合预测(DETR 的推演) + +> 在 DETR 之前,物体检测被绑死在 anchor + NMS 上。这个再发明节点告诉你:图谱里只要齐了 5 件素材,把检测重新定义为"集合预测"是必然的。 + +## 必须先齐的构件 + +| 类别 | 节点 | 角色 | +|---|---|---| +| 论文 | [`paper:vaswani2017`](paper_vaswani2017.md) | 提供 self-attention 与 cross-attention | +| 概念 | [`concept:transformer`](../../concepts.md#transformer) | 序列范式 | +| 移动 | `move:treat_detection_as_set_prediction_with_learnable_queries` | 核心推演动作 | +| 移动 | `move:set_prediction_with_hungarian` | 二分图匹配损失,去掉 NMS | +| 移动 | `move:cross_attention_query` | query 怎样从特征里"问出"答案 | +| 洞察 | [`insight:attention_is_typed_entity_communication`](insight_attention_is_typed_entity_communication.md) | 让 query = "我关心的实体" 这一拟人化变得自然 | +| 问题 | `problem:nms_is_a_handcrafted_heuristic_and_breaks_dense_scenes` | 直接动机 | +| 问题 | `problem:anchor_design_couples_data_distribution_with_model` | 工程动机 | + +## 推演逻辑 + +1. **承认 NMS 是手工启发**:由 `problem:nms_is_a_handcrafted_heuristic_and_breaks_dense_scenes` 起手。 +2. **承认 anchor 是数据耦合**:由 `problem:anchor_design_couples_data_distribution_with_model` 推动。 +3. **重新定义任务**:把检测改写为 *从图像产生一个固定大小的对象集合*。 +4. **采用 Hungarian 匹配作为损失**:由 `move:set_prediction_with_hungarian` 提供。 +5. **用 query 作为"我关心的实体"**:由 `move:treat_detection_as_set_prediction_with_learnable_queries` 给出。 +6. **用 cross-attention 把 query 与图像特征连接**:由 [`paper:vaswani2017`](paper_vaswani2017.md) 提供基础。 + +→ 必然得到 DETR:一个固定数量的 query 通过 cross-attention 与 CNN 特征对话,输出对象集合,匹配损失代替 NMS。 + +## 这条 trace 在自动驾驶里的延伸 + +DETR 提供了"对象 query 作为可微输出接口"这一基本积木: +- → [BEVFormer](paper_li2022bevformer.md) 的 BEV query +- → [UniAD](paper_2212.10156_uniad.md) 的多任务共享 query +- → [VADv2](paper_vadv2.md) 的向量化 query +- → [PlanT](paper_2210.14222_plant.md) 的对象 token + +如果没有 DETR 这一支路,端到端自动驾驶很可能还停在"anchor + NMS + 多任务串联"的老结构里。 + +## 一条值得注意的反例 + +DETR 的训练效率长期不如 anchor-based 方法,这也是为什么后来出现了 Deformable DETR、DINO(注意:与 DINO 自监督是同名不同物)等改进。这说明:**推演到正确范式只是第一步,工程化优化是另一项独立工作**。 + +## 推荐起步 + +1. [DETR 卡片](paper_carion2020.md) 与 [`concept:detr_query`](../../concepts.md#detr-query)。 +2. 写一个 50 行的简化 DETR:一组 16 个 query + 一个小 transformer + Hungarian loss。 +3. 跑 [`labs/lab03_uniad_query_intuition`](../../../labs/lab03_uniad_query_intuition.ipynb) 看 query 如何在 BEV 上工作。 diff --git a/docs/data/cards/extended/validation_trace_unified_planning_oriented_e2e_driving.md b/docs/data/cards/extended/validation_trace_unified_planning_oriented_e2e_driving.md new file mode 100644 index 0000000..da75420 --- /dev/null +++ b/docs/data/cards/extended/validation_trace_unified_planning_oriented_e2e_driving.md @@ -0,0 +1,60 @@ +# 再发现 · UniAD:统一规划导向端到端驾驶 + +> 一份"假设这项工作从未发表过,我们沿着图谱里现成的素材如何把它推演出来"的工程清单。每一项构件在这片图谱中独立存在、被其它路径反复借用;把它们叠在一起,UniAD 的形状会自然显现。 + +## 构件的物理意义 + +UniAD 把"分模块流水线"折叠成"一组共享 BEV 查询沿任务图共同优化"。这一形变并不是凭空发生的:研究者必须先具备四样东西。 + +| 构件 | 它在 UniAD 里的角色 | +|---|---| +| 鸟瞰图(BEV)特征 | 一个跨摄像头、跨时间的统一坐标系,让所有任务模块在同一张图上写字 | +| 对象 query 与集合预测 | 把"我关心的实体"显式地编码成一组可学习向量,使任务模块的输出接口可微 | +| 任务到任务的 cross-attention | 让跟踪 query 把身份信息送给运动 query,运动 query 把未来轨迹送给规划 query | +| 模仿学习作为大规模监督 | 在已有的 nuScenes/真实驾驶日志上,把"未来人类轨迹"作为规划损失的来源 | + +只要这四样独立存在,再叠上一个普适层面的洞见——"当下游信号足够强、传统人工分界就该让位"——研究者会被迫得到 UniAD 的形状:一个以 *规划损失驱动*、*BEV 共享中介*、*query 串联多任务* 的端到端架构。 + +## 推演链路 / Derivation chain + +把图谱里相关的边按方向铺出来: + +``` +BEVFormer ──prereq──┐ +DETR ──prereq──┤ +ResNet 残差 ──prereq──┤ +BC + 协变量偏移──prereq──┤ + ├──composes──> UniAD(再发现节点) +"信号强时端到端胜过 │ + 手工中介" 的跨学科洞察 ──┤ +"cross-attention 是带类 │ + 型实体之间的通信" ──┘ +``` + +如果把上面这张子图喂给一个研究者,要求"我想在 nuScenes 上同时完成 *感知—预测—规划*,并让 *规划质量* 反过来约束上游",那么他/她**必须**得到 UniAD。这就是这一节点存在的意义:不是再讲一遍 UniAD,而是声明"以下七件事在图谱里齐了,UniAD 就被自动召唤"。 + +## 七件事的可观察判据 + +下面这张清单是用图谱里其它节点来表达的判据,研究者可以用它做自查: + +1. **BEV 几何投影** — 由 [`paper:li2022bevformer`](paper_li2022bevformer.md) 与 [`concept:bev`](../../concepts.md#bev感知) 描述,知道如何把六路图像 lift 到 BEV。 +2. **可微的输出接口** — 由 [`paper:carion2020`](paper_carion2020.md)、[`concept:detr_query`](../../concepts.md#detr-query) 与 `move:set_prediction_with_hungarian` 提供。 +3. **query 之间的横向通信** — 由 `move:cross_attention_query` 提供,配套 `insight:attention_is_typed_entity_communication`。 +4. **多任务共享中介** — 由 `move:make_pipeline_differentiable_via_shared_latent` 提供。 +5. **规划损失作为顶层指挥** — 由 `move:set_planning_loss_as_top_objective` 提供。 +6. **模仿学习与协变量偏移的修补** — 由 [`paper:ross2011_dagger`](paper_ross2011_dagger.md)、[`concept:imitation_learning`](../../concepts.md#模仿学习--bc)、[`concept:covariate_shift`](../../concepts.md#协变量偏移) 提供。 +7. **"信号足够强时溶解模块边界"的研究取向** — 由 `insight:end_to_end_differentiable_beats_handcraft_when_signal_strong` 与 [`essay:bitter_lesson`](essay_bitter_lesson.md) 之间的争锋共同界定。 + +## 这份清单怎样帮助下一步 + +* **找空白**:当一个新方向的"推演溯源"链中缺少某个 move 或 insight,往往说明那是一个公认未被破解的开放问题。 +* **测信号**:研究者把自家方法投到这份清单上,可以直接读出"我用上了哪些通用 move、躲开了哪些 insight"。 +* **找邻居**:UniAD 的同型替换链 → [PlanT](paper_2210.14222_plant.md)(保留 query 但去掉 BEV)、[VADv2](paper_vadv2.md)(保留端到端但用向量化)、[DriveVLM](paper_2402.12289_drivevlm.md)(把语言模型接到 UniAD 的接口之上)。 + +## 没出现在清单里、但很容易以为该出现的东西 + +刻意排除掉的: + +- **占据栅格 occupancy** — 是 UniAD 五个任务头之一,但不构成"必须才能想出 UniAD"的前置;它的引入是工程取舍,可被运动预测或可行驶区域分割等价替代。 +- **3D 检测的 anchor 设计** — DETR 已经把它替换为 query,不应在清单里再出现。 +- **特定的 8×A100 训练规模** — 工程预算不是构件。 diff --git a/docs/data/cards/extended/validation_trace_world_model_in_latent_imagination.md b/docs/data/cards/extended/validation_trace_world_model_in_latent_imagination.md new file mode 100644 index 0000000..6ec43b6 --- /dev/null +++ b/docs/data/cards/extended/validation_trace_world_model_in_latent_imagination.md @@ -0,0 +1,48 @@ +# 再发现 · 在潜在想象里训练策略(Dreamer 一类工作) + +> 假设没有 Dreamer / World Models / IRIS / MILE 这些标志性工作,但图谱里其它素材都齐了。要把这一支路重新发明出来,必须凑齐下面这张组件清单。 + +## 组件清单 / Components + +| 类别 | 节点 | 在重新发明里的角色 | +|---|---|---| +| 概念 | [`concept:mdp`](../../concepts.md#mdp) | 把任务表达成状态—动作—奖励 | +| 概念 | [`concept:bellman_eq`](../../concepts.md#bellman方程) | 提供"价值是未来回报的折现"的数学锚 | +| 概念 | [`concept:replay_buffer`](../../concepts.md#replay-buffer) | 支持从历史数据学动力学 | +| 概念 | `concept:autoencoder` | 提供压缩到 latent 的工具 | +| 移动 | `move:learn_world_model_then_plan_in_latent_imagination` | 决定性动作:学一个潜在动力学,再在它上面跑 RL | +| 移动 | `move:co_train_predictive_model_with_policy_to_share_representations` | 联合训练而非串行训练 | +| 移动 | `move:bootstrap_target_network_to_stabilize_off_policy_learning` | 让价值学习不发散 | +| 洞察 | [`insight:world_model_as_inner_simulator_unlocks_long_horizon_planning`](insight_world_model_as_inner_simulator_unlocks_long_horizon_planning.md) | "梦境训练比真实环境便宜" | +| 洞察 | `insight:attention_is_typed_entity_communication` | 提供建模长时序结构的视角 | +| 论文 | [`paper:mnih2015_dqn`](paper_mnih2015_dqn.md) | 提供 off-policy + replay 的祖先 | +| 论文 | [`paper:silver2017_alphazero`](paper_silver2017_alphazero.md) | 提供"在模型里搜索 > 在环境里试错"的早期证据 | +| 论文 | [`paper:world_models`](paper_world_models.md) | 第一份证明潜在想象 RL 可行的工作 | +| 问题 | `problem:exploration_in_safety_critical_systems` | 给出动机:"不能在真车上探索" | + +## 推演逻辑 / Derivation flow + +1. 把 MDP + Bellman 当起点:明确目标是最大化折现回报。 +2. 由 `problem:exploration_in_safety_critical_systems` 推出 *"我们不能在真实环境里探索"*。 +3. 由 `move:learn_world_model_then_plan_in_latent_imagination` 给出唯一可行的回答:先学环境模型。 +4. 由 [`insight:world_model_as_inner_simulator_unlocks_long_horizon_planning`](insight_world_model_as_inner_simulator_unlocks_long_horizon_planning.md) 与 [`paper:silver2017_alphazero`](paper_silver2017_alphazero.md) 验证:"在模型里 rollout 比在真实里试错有效"。 +5. 由 `move:co_train_predictive_model_with_policy_to_share_representations` 与 `concept:autoencoder` 决定具体实现:把状态压成 latent,policy 在 latent 上跑。 +6. 由 `move:bootstrap_target_network_to_stabilize_off_policy_learning` 让训练不发散。 +7. 结果:你必须得到 Dreamer / IRIS / MuZero 一类的算法。 + +## 自动驾驶里的对应版本 + +把上述组件投射到驾驶: +- 状态 = (camera, LiDAR, ego state) 的 latent +- 模型 = 视频扩散或 latent dynamics transformer +- policy = 在 latent 上跑的 RL / planning + +结果即 [GAIA-1](paper_gaia1.md) → [DriveDreamer](paper_drivedreamer.md) → [CF-VLA](paper_2512.24426_cfvla.md) 这条线。这一节点 *显式地告诉你*:把图谱里的 RL 经典工具加上 generative video model 的当代成熟工具,你就站到了驾驶世界模型路线的起点。 + +## 该工作在哪里"还不够" + +- 长时序漂移:潜在动力学预测 30+ 步后开始失真。 +- 跨任务迁移:world model 通常被绑定到训练分布。 +- 闭环安全保证:在 imagination 里的策略未必能在真实物理中复现。 + +这些痛点会推出下一代研究 [`paradigm:model_based_world_imagination_planning`](paradigm_model_based_world_imagination_planning.md) 与 [`paradigm:counterfactual_data_centric_safety`](paradigm_counterfactual_data_centric_safety.md)。 diff --git a/docs/data/generated/decision_axis.json b/docs/data/generated/decision_axis.json new file mode 100644 index 0000000..52ed975 --- /dev/null +++ b/docs/data/generated/decision_axis.json @@ -0,0 +1,336 @@ +{ + "$comment": "Decision-making, planning, control, and reinforcement learning axis. Decomposes how seminal works were invented as paper/move/problem/insight/paradigm nodes.", + "nodes": [ + {"id": "paper:muzero", "label_en": "MuZero", "label_zh": "MuZero(学得隐式动力学模型 + 蒙特卡洛树搜索)", "kind": "paper", "tier": "S", "topic": "world_models", "phase": "core", "year": 2019, "summary_zh": "MuZero 把基于模型的强化学习推到了不需要事先知道环境规则的程度。它在抽象隐空间里同时学习一个表示网络、一个动力学转移网络和一个预测网络,然后在这个隐空间里跑蒙特卡洛树搜索做规划,从而在围棋、国际象棋、将棋以及雅达利游戏上同时取得当时最强的成绩。"}, + {"id": "paper:dreamer_v2", "label_en": "DreamerV2", "label_zh": "DreamerV2(离散隐变量世界模型)", "kind": "paper", "tier": "A", "topic": "world_models", "phase": "core", "year": 2020, "summary_zh": "DreamerV2 在 PlaNet 和 DreamerV1 的基础上把世界模型的隐状态改成离散随机变量,并配合 KL 平衡和直通梯度估计,使得在雅达利套件上仅凭想象中的回放就能训练出与无模型强者相当的策略。它第一次证明纯粹在世界模型内部的想象中训练就可以超过同等数据预算的无模型方法。"}, + {"id": "paper:dreamer_v3", "label_en": "DreamerV3", "label_zh": "DreamerV3(统一超参的通用世界模型)", "kind": "paper", "tier": "S", "topic": "world_models", "phase": "frontier", "year": 2023, "summary_zh": "DreamerV3 通过对回报、价值和奖励做对称对数变换以及一系列规范化技巧,让同一套超参数无需调参就能跨越雅达利、ProcGen、DMLab、Minecraft 等数十个不同动力学的任务取得领先成绩。它把世界模型方法从精细调参的研究原型变成了一个真正可以照搬使用的通用基线。"}, + {"id": "paper:iris_world_model", "label_en": "IRIS", "label_zh": "IRIS(用离散自编码 + transformer 当世界模型)", "kind": "paper", "tier": "A", "topic": "world_models", "phase": "frontier", "year": 2022, "summary_zh": "IRIS 把世界模型重构成两个模块:一个把图像帧压成离散视觉 token 的 VQ-VAE,以及一个像语言模型一样在 token 序列上自回归预测下一帧和奖励的 transformer。这样想象出来的轨迹细节远好于循环型世界模型,使得仅用 100k 步真实数据就能在雅达利上和人类水平相当。"}, + {"id": "paper:sac", "label_en": "Soft Actor-Critic", "label_zh": "SAC(最大熵柔性 Actor-Critic)", "kind": "paper", "tier": "S", "topic": "deep_rl", "phase": "core", "year": 2018, "summary_zh": "SAC 把最大熵强化学习推广到连续动作空间,让策略在最大化期望回报的同时也最大化策略的熵。它自动调节温度系数来控制探索强度,配合双 Q 评论员减小过估计偏差,成为连续控制领域最稳定也最常用的基线算法之一。"}, + {"id": "paper:td3", "label_en": "TD3", "label_zh": "TD3(双延迟深度确定性策略梯度)", "kind": "paper", "tier": "A", "topic": "deep_rl", "phase": "core", "year": 2018, "summary_zh": "TD3 针对 DDPG 在连续控制上不稳定的问题,引入了取两个目标 Q 网络较小值以抑制过估计、延迟更新策略网络以让评论员先收敛、以及在目标动作上加平滑噪声以避免被尖峰估值欺骗这三项关键修正。它把确定性策略梯度方法的样本效率和稳定性同时提升到了实用水平。"}, + {"id": "paper:a3c_a2c", "label_en": "A3C / A2C", "label_zh": "A3C / A2C(异步与同步优势 Actor-Critic)", "kind": "paper", "tier": "A", "topic": "deep_rl", "phase": "prereq", "year": 2016, "summary_zh": "A3C 让多个并行的工人各自和环境交互,把梯度异步推送到一个共享参数服务器,从而在不依赖回放缓冲的情况下打破样本相关性。后来研究者发现把异步改成同步、批量平均梯度就能得到同样甚至更好的效果,于是出现了更简单的 A2C 变体,并成为很多策略梯度方法的入门骨架。"}, + {"id": "paper:impala", "label_en": "IMPALA", "label_zh": "IMPALA(V-trace 重要性修正分布式 RL)", "kind": "paper", "tier": "A", "topic": "deep_rl", "phase": "core", "year": 2018, "summary_zh": "IMPALA 把 actor 和 learner 解耦:actor 不断把轨迹送给 learner,learner 用稍旧的策略数据更新参数。为了纠正这种异步带来的离策略偏差,作者提出了 V-trace 截断重要性采样,使分布式深度强化学习能在几千个 CPU 上稳定扩展并训练单一智能体同时玩几十个任务。"}, + {"id": "paper:mpo", "label_en": "MPO", "label_zh": "MPO(最大后验策略优化)", "kind": "paper", "tier": "A", "topic": "deep_rl", "phase": "core", "year": 2018, "summary_zh": "MPO 把策略改进步骤看成一次后验推断:先在 Q 值加权下找到一个非参数化的最优动作分布,再让参数化策略以 KL 投影去拟合这个分布。这种 E 步加 M 步的结构既保留了策略梯度的灵活性,又获得了类似自然梯度的稳定步长,成为 DeepMind 控制类工作的常用主力算法。"}, + {"id": "paper:decision_transformer", "label_en": "Decision Transformer", "label_zh": "Decision Transformer(把强化学习当作序列建模)", "kind": "paper", "tier": "S", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "Decision Transformer 用一个简单的 GPT 风格 transformer,把过去的回报到 go、状态和动作组成的序列直接当成语言来建模,预测下一个动作。它完全跳过了价值函数和策略梯度,只靠监督式的下一个 token 预测就在 D4RL 等离线数据集上达到与离线 RL 专门算法相当的水平。"}, + {"id": "paper:trajectory_transformer", "label_en": "Trajectory Transformer", "label_zh": "Trajectory Transformer(轨迹 token 化 + 束搜索)", "kind": "paper", "tier": "A", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "Trajectory Transformer 把状态、动作和奖励都离散化成 token,让一个 transformer 学会预测整条轨迹的联合分布。在部署时它对未来轨迹做束搜索,从中挑出预期回报最高的一条,从而把规划重新表述为高似然且高回报的序列搜索问题。"}, + {"id": "paper:diffusion_policy_chi2023", "label_en": "Diffusion Policy", "label_zh": "Diffusion Policy(用扩散模型生成动作序列)", "kind": "paper", "tier": "S", "topic": "deep_rl", "phase": "frontier", "year": 2023, "summary_zh": "Diffusion Policy 把模仿学习中的策略改写成一个条件扩散模型,输入是最近若干帧观察,输出是未来若干步的动作序列。扩散过程的多模态表达能力让它能优雅处理人类示教中的动作多解性,从而在多种机器人操控任务上把成功率显著推高。"}, + {"id": "paper:redq", "label_en": "REDQ", "label_zh": "REDQ(高更新比的集成 Q)", "kind": "paper", "tier": "B", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "REDQ 在 SAC 之上做了两件事:维护一个由十个 Q 网络组成的集成、并且每次环境交互后做二十次梯度更新。集成中的方差用来抑制过估计偏差,使得激进的更新比也能稳定,从而把无模型连续控制的样本效率推到接近基于模型方法的水平。"}, + {"id": "paper:cql", "label_en": "Conservative Q-Learning", "label_zh": "CQL(保守 Q 学习)", "kind": "paper", "tier": "A", "topic": "deep_rl", "phase": "core", "year": 2020, "summary_zh": "CQL 在离线强化学习的 Q 损失里额外加了一个项,使得对数据外动作的 Q 估计被显式压低,从而得到真实 Q 的下界。这种保守化让从离线数据学到的策略在部署时不再倾向于挑那些没见过却看起来高分的动作,极大地缓解了离线 RL 的分布偏移问题。"}, + {"id": "paper:iql", "label_en": "Implicit Q-Learning", "label_zh": "IQL(隐式 Q 学习)", "kind": "paper", "tier": "S", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "IQL 用一个 expectile 回归来估计动作分布在状态下的高分位 Q,从而完全避免在 Bellman 备份里采样数据外动作。配合一个用优势加权的策略提取步骤,它在不显式接触分布外动作的情况下做出隐式的最大化,是当下最稳健的离线 RL 算法之一。"}, + {"id": "paper:calql", "label_en": "Cal-QL", "label_zh": "Cal-QL(校准 CQL 以支持离线到在线微调)", "kind": "paper", "tier": "B", "topic": "deep_rl", "phase": "frontier", "year": 2023, "summary_zh": "Cal-QL 发现 CQL 把分布外动作的 Q 值压得过低,会让随后的在线微调阶段一直在错误的方向上拉扯。它在 CQL 的下界项里加入一个校准项,把分布外 Q 钉在参考策略的真实回报附近,从而让同一份模型能从离线预训练顺畅过渡到在线强化学习。"}, + {"id": "paper:alphastar", "label_en": "AlphaStar", "label_zh": "AlphaStar(星际争霸 II 的大规模联盟训练)", "kind": "paper", "tier": "A", "topic": "deep_rl", "phase": "core", "year": 2019, "summary_zh": "AlphaStar 用监督学习从人类录像预热策略,然后通过一个由主智能体、利用者和联盟探索者构成的联盟博弈系统持续自我对弈,避免在策略空间里陷入循环。它最终在星际争霸 II 上达到大师级水准,展示了如何在巨大动作空间和长时博弈中把分布式强化学习推到极致。"}, + {"id": "paper:openai_five", "label_en": "OpenAI Five", "label_zh": "OpenAI Five(Dota 2 大规模 PPO)", "kind": "paper", "tier": "A", "topic": "deep_rl", "phase": "core", "year": 2019, "summary_zh": "OpenAI Five 用一套相对简单的 PPO 算法,但把训练规模拉到数万 CPU 和成百上千 GPU、每天累积成千年的游戏经验。它通过随机化对手分布和团队奖励塑形把五人合作策略训练到能战胜世界冠军队伍,证明在合适规模下纯粹的策略梯度方法依然非常强大。"}, + {"id": "paper:interfuser", "label_en": "InterFuser", "label_zh": "InterFuser(CARLA 多模态融合 transformer)", "kind": "paper", "tier": "A", "topic": "planning", "phase": "core", "year": 2022, "summary_zh": "InterFuser 用一个 transformer 把多视角摄像头、激光雷达和高精地图特征融合成一个共享表示,并在解码器上同时输出可解释的中间结果,例如他车未来轨迹和交通灯状态。这种结构既提高了 CARLA 闭环驾驶分数,也让规划过程的依据可被人类审查。"}, + {"id": "paper:roach", "label_en": "Roach", "label_zh": "Roach(用特权 RL 教师蒸馏出 BC 学生)", "kind": "paper", "tier": "A", "topic": "planning", "phase": "core", "year": 2021, "summary_zh": "Roach 先在 CARLA 模拟器中用 PPO 训练一个能访问全部真值信息的强化学习教师智能体,再用这个教师生成大规模数据集,让一个只看摄像头的学生策略用监督学习去模仿。它把仿真特权信息变成了能下放给真实感知模型的免费监督信号,在 CARLA 排行榜上长期占据前列。"}, + {"id": "paper:thinktwice", "label_en": "ThinkTwice", "label_zh": "ThinkTwice(粗规划 + 细化的两阶段端到端)", "kind": "paper", "tier": "B", "topic": "planning", "phase": "frontier", "year": 2023, "summary_zh": "ThinkTwice 把端到端驾驶分成两个 transformer 阶段:第一阶段从感知特征里生成一个粗略的未来轨迹,第二阶段用这条粗轨迹回到特征上做条件式的细化预测,把可能的碰撞和遮挡反馈到下一次迭代。这种两次思考的结构让它在 CARLA 复杂场景上取得当年最好的闭环得分。"}, + {"id": "paper:mile_driving", "label_en": "MILE", "label_zh": "MILE(驾驶世界模型 + 模仿学习)", "kind": "paper", "tier": "A", "topic": "world_models", "phase": "frontier", "year": 2022, "summary_zh": "MILE 把 Wayve 内部的 BEV 世界模型与模仿学习联合训练,让策略不仅模仿专家动作,还要在隐世界模型中预测未来的语义 BEV 占用图。该联合目标显著提升了在城市路况下的泛化能力,也是较早把驾驶世界模型用于离线策略学习的代表性工作。"}, + {"id": "paper:nuplan_baselines", "label_en": "nuPlan baselines", "label_zh": "nuPlan 规划基线(IDM, PDM, GameFormer)", "kind": "paper", "tier": "B", "topic": "planning", "phase": "core", "year": 2023, "summary_zh": "nuPlan 团队和社区围绕这一首个大规模真实驾驶规划基准发布了一系列基线,包括基于规则的 IDM 跟车、PDM 预测驱动的轨迹打分,以及 GameFormer 这样的博弈式 transformer 规划器。这些基线共同表明,规则方法在闭环度量上依然非常顽强,纯模仿学习离不开仔细的轨迹后处理。"}, + {"id": "paper:cilqr", "label_en": "CILQR", "label_zh": "CILQR(带约束的迭代 LQR)", "kind": "paper", "tier": "A", "topic": "control", "phase": "core", "year": 2017, "summary_zh": "CILQR 在 iLQR 的二阶迭代解算器上加入了对状态和控制的不等式约束,用对数障碍函数把约束变成可微项。它使得轨迹优化器可以同时考虑车辆动力学、加速度极限和侧向稳定性,成为很多工业级运动规划模块的核心数值求解器。"}, + {"id": "paper:ilqr_classic", "label_en": "iLQR", "label_zh": "iLQR(迭代 LQR)", "kind": "paper", "tier": "A", "topic": "control", "phase": "prereq", "year": 2004, "summary_zh": "iLQR 把非线性最优控制问题在当前名义轨迹附近线性化,套用线性二次调节器解出一次反馈律,然后沿反馈律前向滚动得到新的名义轨迹,反复迭代直到收敛。它兼具 LQR 的解析效率和直接配点法的非线性表达力,是机器人和自动驾驶轨迹优化的标准工具。"}, + {"id": "paper:mpc_book", "label_en": "Model Predictive Control", "label_zh": "MPC(模型预测控制,经典教科书)", "kind": "paper", "tier": "S", "topic": "control", "phase": "prereq", "year": 2002, "summary_zh": "模型预测控制在每一步都用当前模型预测未来若干步的状态轨迹,求解一个带约束的最优控制问题,只执行最优序列的第一个动作然后滚动重新求解。它天然支持显式约束和多输入多输出耦合,是化工、机械和自动驾驶纵向控制最广泛部署的最优控制方法。"}, + {"id": "paper:lqr_classic", "label_en": "Linear Quadratic Regulator", "label_zh": "LQR(线性二次调节器)", "kind": "paper", "tier": "A", "topic": "control", "phase": "prereq", "year": 1960, "summary_zh": "LQR 假设系统动力学是线性的、代价函数是状态和控制的二次型,并通过求解一个代数 Riccati 方程得到最优线性反馈律。它是最优控制最早也是最有教学意义的解析结果,几乎所有更先进的非线性最优控制方法都以它作为局部子问题或起步基线。"}, + {"id": "paper:cpo_safe_rl", "label_en": "Constrained Policy Optimization", "label_zh": "CPO(带约束的策略优化)", "kind": "paper", "tier": "A", "topic": "safety", "phase": "core", "year": 2017, "summary_zh": "CPO 把 TRPO 的信赖域思想推广到带约束的马尔可夫决策过程,在每一步策略更新里同时保证新的策略在期望回报上有改进,并且不会让某些预期约束代价超过预算。它是第一个在大规模深度强化学习里直接对安全约束做硬性保证的算法,奠定了后续安全 RL 的范式。"}, + {"id": "paper:lagrangian_safe_rl", "label_en": "Lagrangian Safe RL", "label_zh": "Lagrangian 安全 RL(约束 RL 的对偶方法)", "kind": "paper", "tier": "B", "topic": "safety", "phase": "core", "year": 2019, "summary_zh": "Lagrangian 风格的安全强化学习把每个约束写成一个不等式,引入对偶变量并和策略参数一起做交替优化。这种方法实现简单,可以套在 PPO、SAC 等任意策略梯度算法上,是安全 RL 的工业基线,但对超参数和奖励 / 约束尺度比较敏感。"}, + {"id": "paper:shielded_rl", "label_en": "Shielded RL", "label_zh": "屏蔽式 RL(形式化安全屏蔽)", "kind": "paper", "tier": "B", "topic": "safety", "phase": "core", "year": 2018, "summary_zh": "屏蔽式强化学习在学习到的策略外面套一层用形式化方法或可达性分析得到的安全监督器:当策略想执行某动作时,屏蔽器检查该动作是否会进入预定义的不安全状态集,并在必要时替换为安全替代动作。这种结构让 RL 的探索能够在硬安全保证下进行。"}, + {"id": "paper:pebble", "label_en": "PEBBLE", "label_zh": "PEBBLE(无监督预训练 + 偏好反馈 RL)", "kind": "paper", "tier": "B", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "PEBBLE 先用无监督的内在奖励 RL 让智能体学到一组多样化策略,再向人类反复展示成对的轨迹片段征询偏好,用偏好数据训练一个奖励模型,最后再用这个奖励模型驱动正式的 RL 学习。它把偏好式 RL 的样本效率提升到能在几百次查询内学会复杂控制任务的水平。"}, + {"id": "paper:bpref", "label_en": "B-Pref", "label_zh": "B-Pref(偏好式 RL 基准)", "kind": "paper", "tier": "B", "topic": "deep_rl", "phase": "core", "year": 2021, "summary_zh": "B-Pref 提出了一组覆盖不同标注误差模式的人类偏好模拟器,并把当时所有主要的偏好式 RL 方法在统一接口下对照评估。它揭示了真实人类反馈中的不一致与延迟所带来的鲁棒性问题,是奖励建模与对齐研究的常用沙盒。"}, + {"id": "paper:trajeglish", "label_en": "Trajeglish", "label_zh": "Trajeglish(驾驶轨迹的 token 语言模型)", "kind": "paper", "tier": "B", "topic": "world_models", "phase": "frontier", "year": 2023, "summary_zh": "Trajeglish 把多智能体的连续驾驶轨迹离散化成动作 token 序列,然后训练一个像语言模型一样自回归预测下一步行为的 transformer。它生成的交通场景在 Waymo Open Sim Agents 基准上拿到当时的最佳成绩,把驾驶仿真智能体推向以序列建模为中心的范式。"}, + {"id": "paper:most_simagents", "label_en": "MoST", "label_zh": "MoST(多智能体场景 token 化生成)", "kind": "paper", "tier": "B", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "MoST 把仿真场景中所有车辆共同的状态序列编码成统一的 token 流,由一个 transformer 联合生成所有智能体未来若干秒的行为。这种把场景看成单一长序列的视角能够内生地捕捉车辆之间的交互,从而显著提升合成交通流的逼真度。"}, + {"id": "paper:codetraj", "label_en": "CodeTrajectory", "label_zh": "CodeTrajectory(以代码描述驾驶轨迹)", "kind": "paper", "tier": "B", "topic": "planning", "phase": "frontier", "year": 2024, "summary_zh": "CodeTrajectory 让大语言模型把规划任务表述成一段 Python 风格的轨迹生成代码,再交给数值求解器或仿真器执行得到具体轨迹。这种以代码作为中间表示的方法使 LLM 能在抽象语义层面思考决策,同时把数值优化的精度问题留给专门工具。"}, + {"id": "paper:diffusion_planner", "label_en": "Diffuser Planner extensions", "label_zh": "Diffuser 规划器扩展(条件扩散轨迹生成)", "kind": "paper", "tier": "B", "topic": "planning", "phase": "frontier", "year": 2023, "summary_zh": "在原始 Diffuser 的基础上,多项后续工作把条件扩散模型应用到驾驶轨迹规划:在采样过程中通过梯度引导加入碰撞代价、舒适度代价或目标到达约束,从而在不改动主干的情况下灵活注入安全和性能目标。"}, + {"id": "paper:mbrl_pets", "label_en": "PETS", "label_zh": "PETS(概率集成 + 轨迹采样的基于模型 RL)", "kind": "paper", "tier": "B", "topic": "world_models", "phase": "core", "year": 2018, "summary_zh": "PETS 用一个由概率神经网络集成构成的动力学模型,捕捉认知与随机两类不确定性,然后通过交叉熵方法对动作序列做采样规划。它证明只要正确建模不确定性,基于模型的 RL 在样本效率上可以和最强的无模型方法拉开数量级的差距。"}, + + {"id": "move:learn_world_model_then_plan_in_latent_imagination", "label_en": "Learn world model then plan in latent imagination", "label_zh": "学得世界模型后在隐空间想象中做规划", "kind": "move", "tier": "move", "topic": "world_models", "phase": "core", "year": 2018, "summary_zh": "训练一个能够预测未来隐状态、奖励和终止信号的世界模型,然后让策略或规划器完全在这个内部模拟器里展开多步推演。真实环境只用来收集数据和最终评估,从而把样本效率与昂贵交互彻底解耦。"}, + {"id": "move:plan_with_mcts_in_learned_model", "label_en": "Plan with MCTS in a learned model", "label_zh": "在学得模型上跑蒙特卡洛树搜索", "kind": "move", "tier": "move", "topic": "world_models", "phase": "frontier", "year": 2019, "summary_zh": "把传统上需要真实模拟器的蒙特卡洛树搜索接到一个学得的隐式动力学模型上,让搜索可以扩展到没有先验规则的领域。这个动作让 AlphaZero 的强大规划能力解锁到雅达利、机器人乃至驾驶任务。"}, + {"id": "move:discrete_latent_state_for_world_model", "label_en": "Discrete latent state for world model", "label_zh": "用离散隐变量构造世界模型状态", "kind": "move", "tier": "move", "topic": "world_models", "phase": "core", "year": 2020, "summary_zh": "把世界模型隐状态从连续高斯换成离散随机变量,再用直通梯度估计反向传播。这种结构能稳定捕捉多模态未来,是 DreamerV2 之后绝大多数世界模型的默认设计。"}, + {"id": "move:tokenize_pixel_frames_for_autoregressive_world_model", "label_en": "Tokenize pixel frames for autoregressive world model", "label_zh": "把像素帧 token 化成离散符号让 transformer 自回归预测", "kind": "move", "tier": "move", "topic": "world_models", "phase": "frontier", "year": 2022, "summary_zh": "先用 VQ-VAE 把每一帧图像压缩成几十到几百个离散 token,再让 transformer 把环境演化当成 token 序列预测。这把世界模型变成了一个像 GPT 一样可扩展、可条件化的生成模型。"}, + {"id": "move:replace_explicit_critic_with_diffusion_score", "label_en": "Replace explicit critic with diffusion score", "label_zh": "用扩散得分函数代替显式评论员", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "frontier", "year": 2022, "summary_zh": "把策略输出从一个简单高斯换成一个条件扩散模型,让动作分布的对数密度梯度自然承担起评论员的角色。这种动作让策略具备多模态表达能力,特别适合人类示教数据中常见的一态多动作场景。"}, + {"id": "move:bootstrap_target_network_to_stabilize_off_policy_learning", "label_en": "Use a target network to stabilize bootstrapping", "label_zh": "用慢更新目标网络稳定自举学习", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "prereq", "year": 2015, "summary_zh": "为价值函数维护一份缓慢更新或定期复制的目标参数,让 Bellman 备份的目标值在短时间内保持不变。这显著抑制了 Q 学习在深度网络上常见的发散现象。"}, + {"id": "move:add_entropy_bonus_to_encourage_exploration", "label_en": "Add entropy bonus to encourage exploration", "label_zh": "在目标函数里加策略熵奖励促进探索", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "在策略优化的目标里加入策略熵项,鼓励策略在高回报附近仍保留一定随机性。这一动作既缓解了局部最优陷阱,也让最大熵强化学习有了原则性的损失函数定义。"}, + {"id": "move:turn_offline_dataset_into_supervised_sequence_prediction", "label_en": "Turn offline dataset into supervised sequence prediction", "label_zh": "把离线数据集改写成监督序列预测", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "把回报到 go、状态、动作排成序列,让 transformer 用普通的下一个 token 监督学习目标来训练。这一动作绕开了价值函数与策略梯度,让强化学习直接受益于成熟的大模型工具链。"}, + {"id": "move:replace_value_function_with_implicit_max_via_expectile", "label_en": "Replace explicit value function with expectile implicit max", "label_zh": "用 expectile 回归隐式取最大值替代显式 Q 最大化", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "在离线 RL 中用 expectile 回归直接拟合 Q 在动作上的高分位,而不显式枚举或采样动作。这种动作让算法完全只看见数据集内的动作,自然回避了离线 RL 最棘手的分布外动作问题。"}, + {"id": "move:use_pretrained_language_model_as_action_prior", "label_en": "Use pretrained LLM as prior over action sequences", "label_zh": "把预训练大语言模型当作动作序列先验", "kind": "move", "tier": "move", "topic": "planning", "phase": "frontier", "year": 2023, "summary_zh": "用预训练大语言模型在符号或代码层面提出候选动作序列,再交给数值优化器或仿真器评估。这种动作把 LLM 的常识与场景理解嫁接到了细粒度控制,是 LLM 规划器的关键构件。"}, + {"id": "move:add_lagrangian_safety_constraint_to_actor_critic", "label_en": "Add Lagrangian safety constraint to actor-critic", "label_zh": "在 actor-critic 上挂安全约束的 Lagrangian", "kind": "move", "tier": "move", "topic": "safety", "phase": "core", "year": 2017, "summary_zh": "把碰撞概率或代价违规等安全指标写成不等式约束,引入对偶变量做交替优化,从而把安全 RL 直接架在已有的 PPO 或 SAC 上。它让带约束的策略改进有了一个工业可落地的实现路径。"}, + {"id": "move:treat_planning_as_conditional_generation", "label_en": "Treat planning as conditional generation", "label_zh": "把规划看作条件生成问题", "kind": "move", "tier": "move", "topic": "planning", "phase": "frontier", "year": 2022, "summary_zh": "把轨迹规划重新表述成给定当前状态、目标和约束的条件分布采样问题,让扩散模型或自回归 transformer 等生成模型直接输出整条候选轨迹。这一动作把规划从优化范式拉进了生成范式。"}, + {"id": "move:cast_continuous_action_as_discretized_token_sequence", "label_en": "Cast continuous action as discretized token sequence", "label_zh": "把连续动作离散化成 token 序列", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "把每一维连续动作按分位数或均匀离散化成有限种类的 token,从而让 transformer 用分类交叉熵学习。该动作打开了语言建模工具箱直接服务控制问题的大门。"}, + {"id": "move:use_n_step_returns_to_trade_bias_for_variance", "label_en": "Use n-step returns to trade bias for variance", "label_zh": "用 n 步回报折中偏差与方差", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "prereq", "year": 1988, "summary_zh": "把 1 步 TD 备份和蒙特卡洛回报间的 n 步混合作为价值学习目标,在偏差和方差之间显式调参。这一基本动作贯穿从经典 TD 到 Rainbow 再到 R2D2 的几乎所有改进。"}, + {"id": "move:add_intrinsic_motivation_via_novelty_or_curiosity", "label_en": "Add intrinsic motivation via novelty or curiosity", "label_zh": "通过新颖性或好奇心给探索加内在动机", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "在外部奖励之外再加一项基于预测误差、状态访问计数或表征相似度的内在奖励,鼓励智能体主动探索没见过的状态。这是稀疏奖励下让 RL 仍能学到东西的关键动作。"}, + {"id": "move:apply_gae_to_smooth_advantage_estimation", "label_en": "Apply GAE to smooth advantage estimation", "label_zh": "用广义优势估计平滑优势函数", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2015, "summary_zh": "用一个折扣系数 lambda 把不同 n 步优势的几何加权混合起来,在偏差和方差之间得到平滑的优势估计。这是 PPO、A3C 等所有现代策略梯度方法的标准配件。"}, + {"id": "move:use_prioritized_replay_buffer", "label_en": "Use priority-weighted replay buffer", "label_zh": "用优先级回放提升样本效率", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2015, "summary_zh": "在经验回放里按 TD 误差大小为转移分配采样概率,让网络更多见到学习信号大的样本。这一动作通常能让 DQN 系列在相同样本数下提升数倍的最终表现。"}, + {"id": "move:cotrain_dynamics_model_with_policy_to_share_representations", "label_en": "Co-train dynamics model with policy to share representations", "label_zh": "联合训练动力学模型与策略共享表示", "kind": "move", "tier": "move", "topic": "world_models", "phase": "core", "year": 2018, "summary_zh": "让感知主干同时承担预测下一时刻状态和输出动作两个任务,并共享中间表征。这迫使表征同时编码动力学相关和决策相关信息,从而比纯模仿学习更具结构。"}, + {"id": "move:warm_start_rl_with_imitation_then_anneal", "label_en": "Warm start RL with imitation data then anneal", "label_zh": "用模仿数据预热 RL 再退火到纯 RL", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2019, "summary_zh": "先用专家示范监督学习训练策略和评论员,再逐步降低模仿损失权重、提高 RL 损失权重,最后切换到纯环境奖励训练。这一动作显著降低早期探索失败和大规模分布式 RL 的冷启动成本。"}, + {"id": "move:double_q_to_reduce_overestimation", "label_en": "Double Q to reduce overestimation", "label_zh": "用双 Q 结构抑制过估计偏差", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2016, "summary_zh": "维护两个独立训练的 Q 网络,并用其中一个选取动作、另一个估值,或简单地在两者中取较小值。该动作有效地消除了 Q 学习以 max 操作为代价付出的系统性高估。"}, + {"id": "move:expert_iteration_self_distillation", "label_en": "Expert iteration via self-distillation", "label_zh": "专家迭代式自蒸馏", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "用一个慢但强的搜索专家(例如 MCTS)生成训练标签,让一个快但弱的网络去模仿;模仿后的网络又用作下一轮搜索的先验。这一循环是 AlphaZero 和后续多项工作的核心动作。"}, + {"id": "move:distill_privileged_teacher_to_sensor_student", "label_en": "Distill privileged teacher to sensor-only student", "label_zh": "将特权信息教师蒸馏到只看传感器的学生策略", "kind": "move", "tier": "move", "topic": "planning", "phase": "core", "year": 2021, "summary_zh": "先在仿真中训练一个能读到全部真值信息的教师策略,再让一个只能看相机或激光雷达的学生策略用监督学习去模仿。它把仿真特权信息变成了对真实部署可用的免费监督。"}, + {"id": "move:trust_region_step_for_monotonic_improvement", "label_en": "Trust region step for monotonic improvement", "label_zh": "用信赖域步长保证策略单调改进", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2015, "summary_zh": "把策略更新限制在新旧策略 KL 散度不超过给定阈值的范围内,给出有理论保证的近似单调改进。从 TRPO 的硬约束到 PPO 的截断比都是这一动作的不同实现形式。"}, + {"id": "move:expectile_or_quantile_target_for_distributional_robustness", "label_en": "Expectile or quantile target for distributional RL", "label_zh": "用 expectile 或 quantile 目标做分布式 RL", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "用 expectile 或 quantile 回归学习回报分布的高阶矩,而不仅是其均值。这一动作既是分布式 DQN 的基石,也启发了 IQL 等离线 RL 算法回避分布外动作。"}, + {"id": "move:hindsight_experience_relabeling", "label_en": "Hindsight experience relabeling", "label_zh": "用事后经验重标注扩充稀疏奖励数据", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "把失败轨迹的真实终点临时当作虚拟目标,让原本零奖励的转移变成对该虚拟任务的成功示例。该动作显著缓解了多目标稀疏奖励 RL 的样本效率问题。"}, + {"id": "move:safety_shield_filters_unsafe_actions", "label_en": "Safety shield filters unsafe actions", "label_zh": "用安全屏蔽器过滤不安全动作", "kind": "move", "tier": "move", "topic": "safety", "phase": "core", "year": 2018, "summary_zh": "在策略输出和执行之间插入一个由形式化验证或可达性分析得到的过滤器,对会进入不安全状态集的动作做替换。这是把硬安全保证与软学习策略结合的关键工程动作。"}, + {"id": "move:reward_model_from_pairwise_human_preferences", "label_en": "Reward model from pairwise human preferences", "label_zh": "从成对人类偏好学奖励模型", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "让人类标注员在两条候选轨迹之间二选一,用 Bradley-Terry 模型拟合一个标量奖励,再用这个奖励驱动后续 RL。该动作把人类直觉编码进了可微的奖励信号,是 RLHF 的核心。"}, + {"id": "move:guided_sampling_through_classifier_gradients_at_inference", "label_en": "Guided sampling via classifier gradients at inference", "label_zh": "在推理时用分类器梯度引导扩散采样", "kind": "move", "tier": "move", "topic": "planning", "phase": "frontier", "year": 2022, "summary_zh": "在扩散模型的反向采样过程中按某个外部代价函数的梯度对样本做扰动,从而把碰撞、舒适度、目标到达等约束在不重新训练模型的情况下注入轨迹生成。"}, + {"id": "move:plan_via_cross_entropy_method_on_dynamics_model", "label_en": "Plan via cross-entropy method on a learned dynamics model", "label_zh": "在学得动力学模型上用交叉熵方法做规划", "kind": "move", "tier": "move", "topic": "world_models", "phase": "core", "year": 2018, "summary_zh": "把动作序列看成一个分布,反复采样、按预测回报筛选精英、再拟合精英拟合一个新分布,反复迭代直至收敛。该动作让基于模型的 RL 不依赖反向传播也能做长程规划。"}, + {"id": "move:two_stage_coarse_to_fine_trajectory", "label_en": "Two-stage coarse-to-fine trajectory", "label_zh": "粗规划再细化的两阶段轨迹生成", "kind": "move", "tier": "move", "topic": "planning", "phase": "frontier", "year": 2023, "summary_zh": "先用一个简单解码器输出粗略未来轨迹,再把这条轨迹作为条件回到主干特征上做精细化预测。这一动作让模型既能高效地确定大致方向,又能在第二阶段集中算力处理障碍和交互细节。"}, + {"id": "move:league_play_for_policy_diversity", "label_en": "League play for policy diversity", "label_zh": "用联盟博弈维持策略多样性", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "frontier", "year": 2019, "summary_zh": "在自我对弈中显式维护主智能体、利用者和探索者等多种角色,让训练池涵盖多种风格的对手。这一动作有效防止了纯自我对弈陷入策略循环,是 AlphaStar 的核心创新。"}, + + {"id": "problem:reward_specification_for_safe_polite_driving", "label_en": "Reward specification for safe and polite driving", "label_zh": "如何为安全且礼让的驾驶设计奖励", "kind": "problem", "tier": "problem", "topic": "safety", "phase": "core", "year": 2018, "summary_zh": "驾驶任务的奖励既要鼓励到达目标,又要惩罚危险、不舒适和不文明的行为,而这些维度往往相互冲突且难以量化。如何设计一个数学上明确、又能在真实数据中可计算的奖励,是端到端规划长期未解的问题。"}, + {"id": "problem:long_horizon_credit_assignment_in_driving", "label_en": "Long-horizon credit assignment in driving", "label_zh": "驾驶任务中的长时信用分配", "kind": "problem", "tier": "problem", "topic": "rl_foundations", "phase": "core", "year": 2018, "summary_zh": "在城市驾驶里,一次错过的并道决策可能要几十秒后才在事故或拥堵中体现,使得 RL 的回报信号极度稀疏且延迟。如何在这种长视野下把奖励正确归因到关键决策,是把 RL 真正用于规划面临的核心难题。"}, + {"id": "problem:distributional_shift_between_offline_data_and_deployment", "label_en": "Distributional shift between offline data and deployment", "label_zh": "离线数据与在线部署之间的分布偏移", "kind": "problem", "tier": "problem", "topic": "deep_rl", "phase": "core", "year": 2020, "summary_zh": "用历史驾驶日志训练得到的策略一旦上路就会进入数据集没有覆盖的状态-动作组合,价值估计在这些点上往往严重高估。如何在不重新采集真实数据的情况下抑制这种偏移,是离线 RL 与模仿学习的共同难题。"}, + {"id": "problem:closed_loop_simulation_fidelity_gap", "label_en": "Closed-loop simulation fidelity gap", "label_zh": "闭环仿真与真实世界的逼真度差距", "kind": "problem", "tier": "problem", "topic": "planning", "phase": "core", "year": 2020, "summary_zh": "闭环仿真要同时模拟感知、预测和其它交通参与者的反应,任何一个环节失真都会让在仿真里训练或评估的策略在真实路况下表现迥异。如何刻画并缩小这种 sim-to-real 差距,是规划研究最核心的方法论瓶颈之一。"}, + {"id": "problem:multi_agent_interaction_modeling_in_dense_traffic", "label_en": "Multi-agent interaction modeling in dense traffic", "label_zh": "密集车流中的多智能体交互建模", "kind": "problem", "tier": "problem", "topic": "planning", "phase": "frontier", "year": 2021, "summary_zh": "在城市路口或匝道,自车决策与他车行为相互耦合,单边的预测或单边的规划都无法捕捉真实博弈过程。如何在不让模型规模与计算复杂度爆炸的情况下表达这种多智能体交互,是行为预测和规划的关键挑战。"}, + {"id": "problem:rare_event_evaluation_with_no_ground_truth", "label_en": "Rare-event evaluation without ground truth", "label_zh": "缺少真值的稀有事件评测", "kind": "problem", "tier": "problem", "topic": "safety", "phase": "frontier", "year": 2021, "summary_zh": "致命驾驶事故每数千万公里才发生一次,常规度量根本无法以统计置信度度量这种长尾,重要性采样和合成场景又面临真值缺失的问题。如何让稀有事件评测可信且可比较,是工业界与监管层的共同未解题。"}, + {"id": "problem:exploration_in_safety_critical_systems", "label_en": "Exploration in safety-critical systems", "label_zh": "安全关键系统中的探索问题", "kind": "problem", "tier": "problem", "topic": "safety", "phase": "core", "year": 2019, "summary_zh": "RL 需要主动尝试未知动作以学习更好的策略,但在驾驶或机器人手术等领域,错误探索代价无法承受。如何在硬安全约束下保持有效探索是安全 RL 与控制理论的长期共同难题。"}, + {"id": "problem:planning_horizon_vs_compute_budget_tradeoff", "label_en": "Planning horizon vs compute budget tradeoff", "label_zh": "规划时域与算力预算的权衡", "kind": "problem", "tier": "problem", "topic": "control", "phase": "core", "year": 2010, "summary_zh": "把规划做得更深可以预见更远的后果,但计算成本随时域指数或多项式增长,而车载算力又必须支持十赫兹以上的实时刷新。如何在不牺牲反应速度的前提下扩展有效规划时域,是车载规划系统的永恒折中。"}, + {"id": "problem:behavior_cloning_compounds_errors_over_time", "label_en": "Behavior cloning compounds errors over time", "label_zh": "行为克隆误差随时间复合", "kind": "problem", "tier": "problem", "topic": "deep_rl", "phase": "core", "year": 2010, "summary_zh": "纯监督式的行为克隆只看见专家轨迹,一旦在部署时偏离哪怕一点点就会进入训练分布之外,下一步又在更偏的位置预测,错误像雪球一样滚大。这是 DAgger、对抗模仿学习等大量后续方法所共同针对的问题。"}, + {"id": "problem:reward_hacking_in_learned_objectives", "label_en": "Reward hacking in learned objectives", "label_zh": "学习奖励容易被钻空子", "kind": "problem", "tier": "problem", "topic": "safety", "phase": "frontier", "year": 2020, "summary_zh": "无论是手写代价还是从人类偏好学到的奖励模型,都不可能完整反映真实意图,因此优化过头时策略会找到奇形怪状但高分的行为。如何检测并修补这种 reward hacking,是 RLHF 与控制对齐的共同未解问题。"}, + + {"id": "insight:imitation_learning_alone_cannot_recover_from_compounding_errors", "label_en": "Imitation learning alone cannot recover from compounding errors", "label_zh": "纯模仿学习无法从复合误差中自我恢复", "kind": "insight", "tier": "insight", "topic": "deep_rl", "phase": "core", "year": 2011, "summary_zh": "由于训练阶段只看专家状态而部署阶段必须自己应对自己产生的状态,模仿学习的误差会沿时间复合。任何想要把模仿学习推到长视野任务的方法都必须显式补救这一点,要么通过交互式重标注,要么通过引入价值或世界模型。"}, + {"id": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", "label_en": "World model as inner simulator unlocks long-horizon planning", "label_zh": "世界模型作为内部模拟器解锁长时规划", "kind": "insight", "tier": "insight", "topic": "world_models", "phase": "core", "year": 2018, "summary_zh": "一旦智能体内部拥有可微、可分支的环境近似,规划就不再受真实交互成本限制,可以在想象中跑成千上万次试错。这条洞见把 RL 从样本贵的范式带向了样本高效的范式。"}, + {"id": "insight:human_demonstrations_compress_implicit_reward_function", "label_en": "Human demonstrations compress an implicit reward function", "label_zh": "人类示教其实把隐式奖励压缩在轨迹里", "kind": "insight", "tier": "insight", "topic": "deep_rl", "phase": "core", "year": 2016, "summary_zh": "一组好的示教不只是状态-动作对,更是对某个未明说的奖励函数的优解。一旦认识到这点,逆强化学习、偏好学习、扩散策略都可以被理解为不同方式去解码这份隐式奖励。"}, + {"id": "insight:safety_emerges_from_constraint_lagrangian_not_reward_shaping", "label_en": "Safety emerges from constraint Lagrangian not reward shaping", "label_zh": "安全要靠约束 Lagrangian 而非奖励塑形", "kind": "insight", "tier": "insight", "topic": "safety", "phase": "core", "year": 2017, "summary_zh": "把碰撞惩罚塞进标量奖励里会被到达目标的回报抵消,策略仍可能选择高风险路径。把安全单独写成约束并用 Lagrangian 优化,让安全要求与性能要求分别有各自的对偶变量调控,是更稳健的安全 RL 范式。"}, + {"id": "insight:offline_rl_is_actually_constrained_dynamic_programming", "label_en": "Offline RL is actually constrained dynamic programming", "label_zh": "离线 RL 本质上是带约束的动态规划", "kind": "insight", "tier": "insight", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "之所以 CQL、IQL、Cal-QL 等离线 RL 方法都奏效,是因为它们用不同方式把价值迭代约束在数据集支撑内。一旦明白离线 RL 等价于在数据集分布上做带约束的动态规划,就能从单一框架推出大量算法变体。"}, + {"id": "insight:tokenized_trajectories_let_planning_borrow_from_language_modeling", "label_en": "Tokenized trajectories let planning borrow from language modeling", "label_zh": "把轨迹 token 化让规划可以借用语言模型工具", "kind": "insight", "tier": "insight", "topic": "planning", "phase": "frontier", "year": 2021, "summary_zh": "一旦把状态、动作和奖励都编码成离散 token,规划就变成了一个序列生成问题,可以直接套用 transformer、束搜索、扩散采样等成熟工具。这是 Decision Transformer、Trajeglish、CodeTrajectory 等工作背后的共同范式。"}, + {"id": "insight:bigger_model_plus_more_data_beats_clever_priors", "label_en": "Bigger model plus more data beats clever priors", "label_zh": "更大模型加更多数据胜过精巧先验", "kind": "insight", "tier": "insight", "topic": "deep_rl", "phase": "core", "year": 2019, "summary_zh": "在驾驶规划与 RL 的多项基准上,朴素的大模型 + 大数据组合一次次追上甚至超过手写规则与精巧结构。这一观察既是苦涩教训在决策领域的延伸,也是采取保守归纳偏置时必须正视的事实。"}, + {"id": "insight:control_theory_and_rl_meet_in_optimal_control", "label_en": "Control theory and RL meet in optimal control", "label_zh": "控制论与强化学习在最优控制处汇合", "kind": "insight", "tier": "insight", "topic": "control", "phase": "core", "year": 2000, "summary_zh": "LQR、iLQR、MPC 与 DP、值迭代、actor-critic 其实是从两条不同传统出发解决同一类带约束最优化问题。一旦认识到这一点,许多看似割裂的方法都能在 Bellman 方程的统一视角下被推导和对比。"}, + + {"id": "paradigm:model_based_rl", "label_en": "Model-based RL", "label_zh": "基于模型的强化学习范式", "kind": "paradigm", "tier": "paradigm", "topic": "world_models", "phase": "core", "year": 2018, "summary_zh": "基于模型的 RL 把环境的转移和奖励学习成一个可查询的模型,再在该模型内用规划或想象训练策略。它的关键卖点是样本效率,代价是要承担模型偏差带来的策略偏离。"}, + {"id": "paradigm:model_free_rl", "label_en": "Model-free RL", "label_zh": "无模型强化学习范式", "kind": "paradigm", "tier": "paradigm", "topic": "deep_rl", "phase": "core", "year": 1989, "summary_zh": "无模型 RL 不显式学习环境,而是直接从经验里估计价值或策略梯度。它实现简单、收敛保证清晰,但在样本成本高的真实任务中往往需要海量交互。"}, + {"id": "paradigm:offline_rl", "label_en": "Offline RL", "label_zh": "离线强化学习范式", "kind": "paradigm", "tier": "paradigm", "topic": "deep_rl", "phase": "core", "year": 2020, "summary_zh": "离线 RL 从一份固定的历史数据集中学习策略,部署前不再与环境交互。它把强化学习与监督式机器学习的实践模式拉近,但必须正面解决分布外动作带来的过估计问题。"}, + {"id": "paradigm:imitation_learning", "label_en": "Imitation learning", "label_zh": "模仿学习范式", "kind": "paradigm", "tier": "paradigm", "topic": "deep_rl", "phase": "prereq", "year": 1989, "summary_zh": "模仿学习把策略学习当作监督学习:用专家轨迹做标签训练策略去复现专家行为。它实现简单、训练稳定,但在分布偏移和奖励缺失两方面有原则性的限制。"}, + {"id": "paradigm:optimal_control", "label_en": "Optimal control", "label_zh": "最优控制范式", "kind": "paradigm", "tier": "paradigm", "topic": "control", "phase": "prereq", "year": 1960, "summary_zh": "最优控制把决策问题写成一个带动力学约束的最优化问题,用变分法或动态规划求解。LQR、iLQR、MPC、CILQR 都是它的不同求解策略,是工业自动驾驶规划栈的理论基石。"}, + {"id": "paradigm:safe_rl", "label_en": "Safe RL", "label_zh": "安全强化学习范式", "kind": "paradigm", "tier": "paradigm", "topic": "safety", "phase": "core", "year": 2017, "summary_zh": "安全 RL 把碰撞或代价违规等硬约束显式建模成约束马尔可夫决策过程,并用 Lagrangian、信赖域或形式化屏蔽确保策略改进的同时不破坏安全。它处于纯 RL 与控制论的交叉地带。"}, + {"id": "paradigm:sequence_modeling_for_decision", "label_en": "Sequence modeling for decision", "label_zh": "决策的序列建模范式", "kind": "paradigm", "tier": "paradigm", "topic": "planning", "phase": "frontier", "year": 2021, "summary_zh": "该范式把决策过程整个看成一个序列建模问题,让 transformer 或扩散模型直接学习状态-动作-奖励序列的联合分布。Decision Transformer、Trajeglish、CodeTrajectory 都是它在不同任务上的实例。"} + ], + + "edges": [ + {"source": "paper:silver2017_alphazero", "target": "paper:muzero", "rel": "extends"}, + {"source": "paper:world_models", "target": "paper:dreamer_v2", "rel": "prereq"}, + {"source": "paper:dreamer_v2", "target": "paper:dreamer_v3", "rel": "extends"}, + {"source": "paper:world_models", "target": "paper:iris_world_model", "rel": "prereq"}, + {"source": "paper:vaswani2017", "target": "paper:iris_world_model", "rel": "prereq"}, + {"source": "paper:iris_world_model", "target": "paper:dreamer_v3", "rel": "parallel"}, + {"source": "paper:muzero", "target": "paper:dreamer_v3", "rel": "parallel"}, + {"source": "paper:mnih2015_dqn", "target": "paper:sac", "rel": "prereq"}, + {"source": "paper:mnih2015_dqn", "target": "paper:td3", "rel": "prereq"}, + {"source": "paper:sac", "target": "paper:redq", "rel": "extends"}, + {"source": "paper:td3", "target": "paper:sac", "rel": "parallel"}, + {"source": "paper:mnih2015_dqn", "target": "paper:a3c_a2c", "rel": "parallel"}, + {"source": "paper:a3c_a2c", "target": "paper:impala", "rel": "extends"}, + {"source": "paper:schulman2017_ppo", "target": "paper:openai_five", "rel": "feeds"}, + {"source": "paper:a3c_a2c", "target": "paper:alphastar", "rel": "feeds"}, + {"source": "paper:sac", "target": "paper:mpo", "rel": "parallel"}, + {"source": "paper:schulman2017_ppo", "target": "paper:mpo", "rel": "contrasts"}, + {"source": "paper:gpt3", "target": "paper:decision_transformer", "rel": "prereq"}, + {"source": "paper:decision_transformer", "target": "paper:trajectory_transformer", "rel": "parallel"}, + {"source": "paper:vaswani2017", "target": "paper:trajectory_transformer", "rel": "prereq"}, + {"source": "paper:diffuser", "target": "paper:diffusion_policy_chi2023", "rel": "parallel"}, + {"source": "paper:diffuser", "target": "paper:diffusion_planner", "rel": "extends"}, + {"source": "paper:cql", "target": "paper:iql", "rel": "extends"}, + {"source": "paper:cql", "target": "paper:calql", "rel": "extends"}, + {"source": "paper:iql", "target": "paper:calql", "rel": "parallel"}, + {"source": "paper:cpo_safe_rl", "target": "paper:lagrangian_safe_rl", "rel": "parallel"}, + {"source": "paper:lagrangian_safe_rl", "target": "paper:shielded_rl", "rel": "contrasts"}, + {"source": "paper:rlhf_dpo", "target": "paper:pebble", "rel": "parallel"}, + {"source": "paper:pebble", "target": "paper:bpref", "rel": "parallel"}, + {"source": "paper:rlhf_dpo", "target": "paper:bpref", "rel": "parallel"}, + {"source": "paper:lqr_classic", "target": "paper:ilqr_classic", "rel": "prereq"}, + {"source": "paper:ilqr_classic", "target": "paper:cilqr", "rel": "extends"}, + {"source": "paper:lqr_classic", "target": "paper:mpc_book", "rel": "prereq"}, + {"source": "paper:mpc_book", "target": "paper:cilqr", "rel": "parallel"}, + {"source": "paper:transfuser", "target": "paper:interfuser", "rel": "parallel"}, + {"source": "paper:transfuser", "target": "paper:roach", "rel": "parallel"}, + {"source": "paper:roach", "target": "paper:thinktwice", "rel": "feeds"}, + {"source": "paper:interfuser", "target": "paper:thinktwice", "rel": "parallel"}, + {"source": "paper:world_models", "target": "paper:mile_driving", "rel": "prereq"}, + {"source": "paper:dreamer_v2", "target": "paper:mile_driving", "rel": "parallel"}, + {"source": "paper:ad_benchmarks", "target": "paper:nuplan_baselines", "rel": "covers"}, + {"source": "paper:nuplan_baselines", "target": "paper:interfuser", "rel": "contrasts"}, + {"source": "paper:vaswani2017", "target": "paper:trajeglish", "rel": "prereq"}, + {"source": "paper:trajeglish", "target": "paper:most_simagents", "rel": "parallel"}, + {"source": "paper:gpt3", "target": "paper:codetraj", "rel": "prereq"}, + {"source": "paper:2311.10813", "target": "paper:codetraj", "rel": "parallel"}, + {"source": "paper:world_models", "target": "paper:mbrl_pets", "rel": "parallel"}, + {"source": "paper:mbrl_pets", "target": "paper:dreamer_v2", "rel": "prereq"}, + + {"source": "move:learn_world_model_then_plan_in_latent_imagination", "target": "paper:world_models", "rel": "manifests"}, + {"source": "move:learn_world_model_then_plan_in_latent_imagination", "target": "paper:dreamer_v2", "rel": "manifests"}, + {"source": "move:learn_world_model_then_plan_in_latent_imagination", "target": "paper:dreamer_v3", "rel": "manifests"}, + {"source": "move:learn_world_model_then_plan_in_latent_imagination", "target": "paper:mile_driving", "rel": "manifests"}, + {"source": "move:plan_with_mcts_in_learned_model", "target": "paper:muzero", "rel": "manifests"}, + {"source": "move:plan_with_mcts_in_learned_model", "target": "paper:silver2017_alphazero", "rel": "extends"}, + {"source": "move:discrete_latent_state_for_world_model", "target": "paper:dreamer_v2", "rel": "manifests"}, + {"source": "move:discrete_latent_state_for_world_model", "target": "paper:dreamer_v3", "rel": "manifests"}, + {"source": "move:tokenize_pixel_frames_for_autoregressive_world_model", "target": "paper:iris_world_model", "rel": "manifests"}, + {"source": "move:tokenize_pixel_frames_for_autoregressive_world_model", "target": "paper:gaia1", "rel": "manifests"}, + {"source": "move:tokenize_pixel_frames_for_autoregressive_world_model", "target": "paper:drivedreamer", "rel": "manifests"}, + {"source": "move:replace_explicit_critic_with_diffusion_score", "target": "paper:diffusion_policy_chi2023", "rel": "manifests"}, + {"source": "move:replace_explicit_critic_with_diffusion_score", "target": "paper:diffuser", "rel": "manifests"}, + {"source": "move:bootstrap_target_network_to_stabilize_off_policy_learning", "target": "paper:mnih2015_dqn", "rel": "manifests"}, + {"source": "move:bootstrap_target_network_to_stabilize_off_policy_learning", "target": "paper:sac", "rel": "manifests"}, + {"source": "move:bootstrap_target_network_to_stabilize_off_policy_learning", "target": "paper:td3", "rel": "manifests"}, + {"source": "move:add_entropy_bonus_to_encourage_exploration", "target": "paper:sac", "rel": "manifests"}, + {"source": "move:add_entropy_bonus_to_encourage_exploration", "target": "paper:schulman2017_ppo", "rel": "manifests"}, + {"source": "move:add_entropy_bonus_to_encourage_exploration", "target": "paper:a3c_a2c", "rel": "manifests"}, + {"source": "move:turn_offline_dataset_into_supervised_sequence_prediction", "target": "paper:decision_transformer", "rel": "manifests"}, + {"source": "move:turn_offline_dataset_into_supervised_sequence_prediction", "target": "paper:trajectory_transformer", "rel": "manifests"}, + {"source": "move:replace_value_function_with_implicit_max_via_expectile", "target": "paper:iql", "rel": "manifests"}, + {"source": "move:use_pretrained_language_model_as_action_prior", "target": "paper:codetraj", "rel": "manifests"}, + {"source": "move:use_pretrained_language_model_as_action_prior", "target": "paper:2311.10813", "rel": "manifests"}, + {"source": "move:use_pretrained_language_model_as_action_prior", "target": "paper:2309.16292", "rel": "manifests"}, + {"source": "move:add_lagrangian_safety_constraint_to_actor_critic", "target": "paper:lagrangian_safe_rl", "rel": "manifests"}, + {"source": "move:add_lagrangian_safety_constraint_to_actor_critic", "target": "paper:cpo_safe_rl", "rel": "manifests"}, + {"source": "move:treat_planning_as_conditional_generation", "target": "paper:diffuser", "rel": "manifests"}, + {"source": "move:treat_planning_as_conditional_generation", "target": "paper:diffusion_policy_chi2023", "rel": "manifests"}, + {"source": "move:treat_planning_as_conditional_generation", "target": "paper:diffusion_planner", "rel": "manifests"}, + {"source": "move:treat_planning_as_conditional_generation", "target": "paper:vadv2", "rel": "manifests"}, + {"source": "move:cast_continuous_action_as_discretized_token_sequence", "target": "paper:trajectory_transformer", "rel": "manifests"}, + {"source": "move:cast_continuous_action_as_discretized_token_sequence", "target": "paper:trajeglish", "rel": "manifests"}, + {"source": "move:cast_continuous_action_as_discretized_token_sequence", "target": "paper:most_simagents", "rel": "manifests"}, + {"source": "move:use_n_step_returns_to_trade_bias_for_variance", "target": "paper:mnih2015_dqn", "rel": "enables"}, + {"source": "move:use_n_step_returns_to_trade_bias_for_variance", "target": "paper:schulman2017_ppo", "rel": "enables"}, + {"source": "move:add_intrinsic_motivation_via_novelty_or_curiosity", "target": "paper:pebble", "rel": "enables"}, + {"source": "move:apply_gae_to_smooth_advantage_estimation", "target": "paper:schulman2017_ppo", "rel": "manifests"}, + {"source": "move:apply_gae_to_smooth_advantage_estimation", "target": "paper:a3c_a2c", "rel": "manifests"}, + {"source": "move:apply_gae_to_smooth_advantage_estimation", "target": "paper:openai_five", "rel": "manifests"}, + {"source": "move:use_prioritized_replay_buffer", "target": "paper:mnih2015_dqn", "rel": "enables"}, + {"source": "move:cotrain_dynamics_model_with_policy_to_share_representations", "target": "paper:mile_driving", "rel": "manifests"}, + {"source": "move:cotrain_dynamics_model_with_policy_to_share_representations", "target": "paper:dreamer_v2", "rel": "manifests"}, + {"source": "move:warm_start_rl_with_imitation_then_anneal", "target": "paper:alphastar", "rel": "manifests"}, + {"source": "move:warm_start_rl_with_imitation_then_anneal", "target": "paper:roach", "rel": "manifests"}, + {"source": "move:warm_start_rl_with_imitation_then_anneal", "target": "paper:calql", "rel": "manifests"}, + {"source": "move:double_q_to_reduce_overestimation", "target": "paper:td3", "rel": "manifests"}, + {"source": "move:double_q_to_reduce_overestimation", "target": "paper:sac", "rel": "manifests"}, + {"source": "move:double_q_to_reduce_overestimation", "target": "paper:redq", "rel": "extends"}, + {"source": "move:expert_iteration_self_distillation", "target": "paper:silver2017_alphazero", "rel": "manifests"}, + {"source": "move:expert_iteration_self_distillation", "target": "paper:muzero", "rel": "manifests"}, + {"source": "move:distill_privileged_teacher_to_sensor_student", "target": "paper:roach", "rel": "manifests"}, + {"source": "move:distill_privileged_teacher_to_sensor_student", "target": "paper:transfuser", "rel": "manifests"}, + {"source": "move:trust_region_step_for_monotonic_improvement", "target": "paper:schulman2017_ppo", "rel": "manifests"}, + {"source": "move:trust_region_step_for_monotonic_improvement", "target": "paper:cpo_safe_rl", "rel": "manifests"}, + {"source": "move:trust_region_step_for_monotonic_improvement", "target": "paper:mpo", "rel": "manifests"}, + {"source": "move:expectile_or_quantile_target_for_distributional_robustness", "target": "paper:iql", "rel": "manifests"}, + {"source": "move:hindsight_experience_relabeling", "target": "paper:sac", "rel": "enables"}, + {"source": "move:safety_shield_filters_unsafe_actions", "target": "paper:shielded_rl", "rel": "manifests"}, + {"source": "move:reward_model_from_pairwise_human_preferences", "target": "paper:rlhf_dpo", "rel": "manifests"}, + {"source": "move:reward_model_from_pairwise_human_preferences", "target": "paper:pebble", "rel": "manifests"}, + {"source": "move:reward_model_from_pairwise_human_preferences", "target": "paper:bpref", "rel": "manifests"}, + {"source": "move:guided_sampling_through_classifier_gradients_at_inference", "target": "paper:diffuser", "rel": "manifests"}, + {"source": "move:guided_sampling_through_classifier_gradients_at_inference", "target": "paper:diffusion_planner", "rel": "manifests"}, + {"source": "move:plan_via_cross_entropy_method_on_dynamics_model", "target": "paper:mbrl_pets", "rel": "manifests"}, + {"source": "move:plan_via_cross_entropy_method_on_dynamics_model", "target": "paper:muzero", "rel": "parallel"}, + {"source": "move:two_stage_coarse_to_fine_trajectory", "target": "paper:thinktwice", "rel": "manifests"}, + {"source": "move:two_stage_coarse_to_fine_trajectory", "target": "paper:vadv2", "rel": "manifests"}, + {"source": "move:league_play_for_policy_diversity", "target": "paper:alphastar", "rel": "manifests"}, + + {"source": "move:learn_world_model_then_plan_in_latent_imagination", "target": "move:plan_with_mcts_in_learned_model", "rel": "composes"}, + {"source": "move:discrete_latent_state_for_world_model", "target": "move:learn_world_model_then_plan_in_latent_imagination", "rel": "composes"}, + {"source": "move:tokenize_pixel_frames_for_autoregressive_world_model", "target": "move:learn_world_model_then_plan_in_latent_imagination", "rel": "composes"}, + {"source": "move:cast_continuous_action_as_discretized_token_sequence", "target": "move:turn_offline_dataset_into_supervised_sequence_prediction", "rel": "composes"}, + {"source": "move:bootstrap_target_network_to_stabilize_off_policy_learning", "target": "move:double_q_to_reduce_overestimation", "rel": "composes"}, + {"source": "move:add_entropy_bonus_to_encourage_exploration", "target": "move:double_q_to_reduce_overestimation", "rel": "composes"}, + {"source": "move:trust_region_step_for_monotonic_improvement", "target": "move:add_lagrangian_safety_constraint_to_actor_critic", "rel": "composes"}, + {"source": "move:treat_planning_as_conditional_generation", "target": "move:guided_sampling_through_classifier_gradients_at_inference", "rel": "composes"}, + {"source": "move:use_pretrained_language_model_as_action_prior", "target": "move:treat_planning_as_conditional_generation", "rel": "composes"}, + {"source": "move:reward_model_from_pairwise_human_preferences", "target": "move:warm_start_rl_with_imitation_then_anneal", "rel": "composes"}, + {"source": "move:distill_privileged_teacher_to_sensor_student", "target": "move:warm_start_rl_with_imitation_then_anneal", "rel": "composes"}, + + {"source": "problem:behavior_cloning_compounds_errors_over_time", "target": "paper:ross2011_dagger", "rel": "motivates"}, + {"source": "problem:behavior_cloning_compounds_errors_over_time", "target": "paper:roach", "rel": "motivates"}, + {"source": "problem:behavior_cloning_compounds_errors_over_time", "target": "paper:transfuser", "rel": "motivates"}, + {"source": "problem:distributional_shift_between_offline_data_and_deployment", "target": "paper:cql", "rel": "motivates"}, + {"source": "problem:distributional_shift_between_offline_data_and_deployment", "target": "paper:iql", "rel": "motivates"}, + {"source": "problem:distributional_shift_between_offline_data_and_deployment", "target": "paper:calql", "rel": "motivates"}, + {"source": "problem:distributional_shift_between_offline_data_and_deployment", "target": "paper:decision_transformer", "rel": "motivates"}, + {"source": "problem:long_horizon_credit_assignment_in_driving", "target": "paper:muzero", "rel": "motivates"}, + {"source": "problem:long_horizon_credit_assignment_in_driving", "target": "paper:diffuser", "rel": "motivates"}, + {"source": "problem:reward_specification_for_safe_polite_driving", "target": "paper:rlhf_dpo", "rel": "motivates"}, + {"source": "problem:reward_specification_for_safe_polite_driving", "target": "paper:pebble", "rel": "motivates"}, + {"source": "problem:exploration_in_safety_critical_systems", "target": "paper:cpo_safe_rl", "rel": "motivates"}, + {"source": "problem:exploration_in_safety_critical_systems", "target": "paper:lagrangian_safe_rl", "rel": "motivates"}, + {"source": "problem:exploration_in_safety_critical_systems", "target": "paper:shielded_rl", "rel": "motivates"}, + {"source": "problem:closed_loop_simulation_fidelity_gap", "target": "paper:trajeglish", "rel": "motivates"}, + {"source": "problem:closed_loop_simulation_fidelity_gap", "target": "paper:most_simagents", "rel": "motivates"}, + {"source": "problem:closed_loop_simulation_fidelity_gap", "target": "paper:gaia1", "rel": "motivates"}, + {"source": "problem:closed_loop_simulation_fidelity_gap", "target": "paper:drivedreamer", "rel": "motivates"}, + {"source": "problem:multi_agent_interaction_modeling_in_dense_traffic", "target": "paper:trajeglish", "rel": "motivates"}, + {"source": "problem:multi_agent_interaction_modeling_in_dense_traffic", "target": "paper:nuplan_baselines", "rel": "motivates"}, + {"source": "problem:planning_horizon_vs_compute_budget_tradeoff", "target": "paper:cilqr", "rel": "motivates"}, + {"source": "problem:planning_horizon_vs_compute_budget_tradeoff", "target": "paper:mpc_book", "rel": "motivates"}, + {"source": "problem:rare_event_evaluation_with_no_ground_truth", "target": "paper:nuplan_baselines", "rel": "motivates"}, + {"source": "problem:rare_event_evaluation_with_no_ground_truth", "target": "paper:ad_benchmarks", "rel": "motivates"}, + {"source": "problem:reward_hacking_in_learned_objectives", "target": "paper:bpref", "rel": "motivates"}, + {"source": "problem:reward_hacking_in_learned_objectives", "target": "paper:rlhf_dpo", "rel": "motivates"}, + + {"source": "problem:behavior_cloning_compounds_errors_over_time", "target": "insight:imitation_learning_alone_cannot_recover_from_compounding_errors", "rel": "manifests"}, + {"source": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", "target": "paper:world_models", "rel": "manifests"}, + {"source": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", "target": "paper:dreamer_v3", "rel": "manifests"}, + {"source": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", "target": "paper:muzero", "rel": "manifests"}, + {"source": "insight:human_demonstrations_compress_implicit_reward_function", "target": "paper:diffusion_policy_chi2023", "rel": "manifests"}, + {"source": "insight:human_demonstrations_compress_implicit_reward_function", "target": "paper:ross2011_dagger", "rel": "manifests"}, + {"source": "insight:safety_emerges_from_constraint_lagrangian_not_reward_shaping", "target": "paper:cpo_safe_rl", "rel": "manifests"}, + {"source": "insight:safety_emerges_from_constraint_lagrangian_not_reward_shaping", "target": "paper:lagrangian_safe_rl", "rel": "manifests"}, + {"source": "insight:offline_rl_is_actually_constrained_dynamic_programming", "target": "paper:cql", "rel": "manifests"}, + {"source": "insight:offline_rl_is_actually_constrained_dynamic_programming", "target": "paper:iql", "rel": "manifests"}, + {"source": "insight:offline_rl_is_actually_constrained_dynamic_programming", "target": "paper:calql", "rel": "manifests"}, + {"source": "insight:tokenized_trajectories_let_planning_borrow_from_language_modeling", "target": "paper:decision_transformer", "rel": "manifests"}, + {"source": "insight:tokenized_trajectories_let_planning_borrow_from_language_modeling", "target": "paper:trajectory_transformer", "rel": "manifests"}, + {"source": "insight:tokenized_trajectories_let_planning_borrow_from_language_modeling", "target": "paper:trajeglish", "rel": "manifests"}, + {"source": "insight:bigger_model_plus_more_data_beats_clever_priors", "target": "paper:openai_five", "rel": "manifests"}, + {"source": "insight:bigger_model_plus_more_data_beats_clever_priors", "target": "paper:alphastar", "rel": "manifests"}, + {"source": "insight:bigger_model_plus_more_data_beats_clever_priors", "target": "essay:bitter_lesson", "rel": "extends"}, + {"source": "insight:control_theory_and_rl_meet_in_optimal_control", "target": "paper:lqr_classic", "rel": "manifests"}, + {"source": "insight:control_theory_and_rl_meet_in_optimal_control", "target": "paper:mpc_book", "rel": "manifests"}, + {"source": "insight:control_theory_and_rl_meet_in_optimal_control", "target": "concept:bellman_eq", "rel": "manifests"}, + + {"source": "paradigm:model_based_rl", "target": "paper:world_models", "rel": "manifests"}, + {"source": "paradigm:model_based_rl", "target": "paper:dreamer_v2", "rel": "manifests"}, + {"source": "paradigm:model_based_rl", "target": "paper:dreamer_v3", "rel": "manifests"}, + {"source": "paradigm:model_based_rl", "target": "paper:muzero", "rel": "manifests"}, + {"source": "paradigm:model_based_rl", "target": "paper:iris_world_model", "rel": "manifests"}, + {"source": "paradigm:model_based_rl", "target": "paper:mbrl_pets", "rel": "manifests"}, + {"source": "paradigm:model_free_rl", "target": "paper:mnih2015_dqn", "rel": "manifests"}, + {"source": "paradigm:model_free_rl", "target": "paper:schulman2017_ppo", "rel": "manifests"}, + {"source": "paradigm:model_free_rl", "target": "paper:sac", "rel": "manifests"}, + {"source": "paradigm:model_free_rl", "target": "paper:td3", "rel": "manifests"}, + {"source": "paradigm:model_free_rl", "target": "paper:a3c_a2c", "rel": "manifests"}, + {"source": "paradigm:model_free_rl", "target": "paper:impala", "rel": "manifests"}, + {"source": "paradigm:offline_rl", "target": "paper:cql", "rel": "manifests"}, + {"source": "paradigm:offline_rl", "target": "paper:iql", "rel": "manifests"}, + {"source": "paradigm:offline_rl", "target": "paper:calql", "rel": "manifests"}, + {"source": "paradigm:offline_rl", "target": "paper:decision_transformer", "rel": "manifests"}, + {"source": "paradigm:imitation_learning", "target": "paper:ross2011_dagger", "rel": "manifests"}, + {"source": "paradigm:imitation_learning", "target": "paper:diffusion_policy_chi2023", "rel": "manifests"}, + {"source": "paradigm:imitation_learning", "target": "paper:transfuser", "rel": "manifests"}, + {"source": "paradigm:imitation_learning", "target": "paper:interfuser", "rel": "manifests"}, + {"source": "paradigm:optimal_control", "target": "paper:lqr_classic", "rel": "manifests"}, + {"source": "paradigm:optimal_control", "target": "paper:ilqr_classic", "rel": "manifests"}, + {"source": "paradigm:optimal_control", "target": "paper:cilqr", "rel": "manifests"}, + {"source": "paradigm:optimal_control", "target": "paper:mpc_book", "rel": "manifests"}, + {"source": "paradigm:safe_rl", "target": "paper:cpo_safe_rl", "rel": "manifests"}, + {"source": "paradigm:safe_rl", "target": "paper:lagrangian_safe_rl", "rel": "manifests"}, + {"source": "paradigm:safe_rl", "target": "paper:shielded_rl", "rel": "manifests"}, + {"source": "paradigm:sequence_modeling_for_decision", "target": "paper:decision_transformer", "rel": "manifests"}, + {"source": "paradigm:sequence_modeling_for_decision", "target": "paper:trajectory_transformer", "rel": "manifests"}, + {"source": "paradigm:sequence_modeling_for_decision", "target": "paper:trajeglish", "rel": "manifests"}, + {"source": "paradigm:sequence_modeling_for_decision", "target": "paper:most_simagents", "rel": "manifests"}, + {"source": "paradigm:sequence_modeling_for_decision", "target": "paper:codetraj", "rel": "manifests"}, + + {"source": "paradigm:model_free_rl", "target": "paradigm:model_based_rl", "rel": "contrasts"}, + {"source": "paradigm:imitation_learning", "target": "paradigm:model_free_rl", "rel": "contrasts"}, + {"source": "paradigm:offline_rl", "target": "paradigm:model_free_rl", "rel": "contrasts"}, + {"source": "paradigm:sequence_modeling_for_decision", "target": "paradigm:model_free_rl", "rel": "contrasts"}, + {"source": "paradigm:optimal_control", "target": "paradigm:model_free_rl", "rel": "parallel"}, + {"source": "paradigm:safe_rl", "target": "paradigm:model_free_rl", "rel": "extends"}, + + {"source": "problem:reward_hacking_in_learned_objectives", "target": "insight:safety_emerges_from_constraint_lagrangian_not_reward_shaping", "rel": "motivates"}, + {"source": "problem:long_horizon_credit_assignment_in_driving", "target": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", "rel": "motivates"}, + {"source": "problem:distributional_shift_between_offline_data_and_deployment", "target": "insight:offline_rl_is_actually_constrained_dynamic_programming", "rel": "motivates"}, + {"source": "concept:mdp", "target": "paradigm:optimal_control", "rel": "prereq"}, + {"source": "concept:bellman_eq", "target": "paradigm:model_based_rl", "rel": "prereq"}, + {"source": "concept:bellman_eq", "target": "paradigm:model_free_rl", "rel": "prereq"}, + {"source": "concept:imitation_learning", "target": "paradigm:imitation_learning", "rel": "prereq"}, + {"source": "concept:replay_buffer", "target": "move:use_prioritized_replay_buffer", "rel": "prereq"}, + {"source": "concept:actor_critic", "target": "paper:sac", "rel": "prereq"}, + {"source": "concept:actor_critic", "target": "paper:a3c_a2c", "rel": "prereq"}, + {"source": "concept:actor_critic", "target": "paper:mpo", "rel": "prereq"}, + {"source": "concept:actor_critic", "target": "move:add_lagrangian_safety_constraint_to_actor_critic", "rel": "prereq"} + ] +} diff --git a/docs/data/generated/foundation_axis.json b/docs/data/generated/foundation_axis.json new file mode 100644 index 0000000..0ddda2e --- /dev/null +++ b/docs/data/generated/foundation_axis.json @@ -0,0 +1,356 @@ +{ + "$comment": "Foundation-models / VLA / world-model / agent axis expansion. Generated to extend docs/data/graph.json with paper / move / problem / insight / paradigm nodes covering closed/open LLMs, VLM, VLA, agent loops, video world models, and AD-specific instantiations.", + "nodes": [ + {"id": "paper:gpt4", "label_en": "GPT-4", "label_zh": "GPT-4(多模态闭源大模型)", "kind": "paper", "tier": "S", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "GPT-4 是 OpenAI 在 GPT-3 之后推出的更大规模混合专家架构语言模型,在专业考试、代码与多步推理上首次接近人类专家水平。它通过引入指令微调与基于人类反馈的强化学习,把模型行为从纯模仿语料压向有用、诚实与无害三个目标,并成为后续所有商用闭源大模型的对比基准。"}, + {"id": "paper:gpt4v", "label_en": "GPT-4V", "label_zh": "GPT-4V(视觉多模态扩展)", "kind": "paper", "tier": "S", "topic": "vlm_vla", "phase": "core", "year": 2023, "summary_zh": "GPT-4V 在 GPT-4 文本主干之上插入图像编码器,使闭源大模型第一次具备读图、读图表和读复杂场景照片的能力。它在驾驶研究中被广泛用作零样本场景描述与异常检测基线,但同时也暴露了视觉语言模型在精细空间几何与遮挡推理上的系统性弱点。"}, + {"id": "paper:claude", "label_en": "Claude", "label_zh": "Claude 系列(Anthropic 大模型)", "kind": "paper", "tier": "S", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "Claude 是 Anthropic 推出的对话式大模型,使用宪法式人工智能流程在没有人类逐条偏好打分的情况下完成自我对齐。它把一组自然语言原则当作显式宪法,让模型用自身改写答案以满足这些原则,是开源社区研究无人标注偏好对齐的主要参考实现。"}, + {"id": "paper:gemini", "label_en": "Gemini", "label_zh": "Gemini(Google 多模态原生大模型)", "kind": "paper", "tier": "S", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "Gemini 是 Google DeepMind 把文本、图像、音频、视频与代码端到端原生混合训练的大模型,强调真正的多模态联合预训练而不是事后拼装编码器。它的长上下文窗口与对工具调用的工程化支持,使其成为机器人与驾驶领域研究长时域决策时的常用闭源参考。"}, + {"id": "paper:llama", "label_en": "LLaMA family", "label_zh": "LLaMA 系列(Meta 开源大模型)", "kind": "paper", "tier": "S", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "LLaMA 是 Meta 发布的开源大语言模型权重族,第一次以学术许可让研究者获得百亿到千亿参数规模的强基线。它直接催生了 Alpaca、Vicuna、Llama-2-Chat 等指令微调分支,使 VLM 与 VLA 研究不必再依赖封闭 API 就能复现训练流程。"}, + {"id": "paper:mistral", "label_en": "Mistral / Mixtral", "label_zh": "Mistral 与 Mixtral(稀疏专家开源大模型)", "kind": "paper", "tier": "A", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "Mistral 用滑动窗口注意力和分组查询注意力把稠密小模型推到同尺寸 LLaMA 之上,Mixtral 进一步引入稀疏专家路由把推理成本控制在激活参数量上。这一线证明在固定算力下结构层面的工程优化仍然有显著回报,为车端部署小型 VLM 提供模板。"}, + {"id": "paper:qwen", "label_en": "Qwen series", "label_zh": "Qwen 与 Qwen-VL 系列", "kind": "paper", "tier": "A", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "Qwen 是阿里发布的中文与多语种开源大模型族,其 Qwen-VL 与 Qwen2-VL 分支把视觉编码器与语言主干联合微调到中文场景对话与文档理解上。它在国内驾驶相关 VLM 研究中是最常见的开源基座之一,许多 DriveLM 与 Senna 类工作直接基于 Qwen-VL 续训。"}, + {"id": "paper:instructgpt", "label_en": "InstructGPT", "label_zh": "InstructGPT(指令微调 + RLHF)", "kind": "paper", "tier": "S", "topic": "alignment", "phase": "core", "year": 2022, "summary_zh": "InstructGPT 是 OpenAI 把指令式数据集与人类偏好强化学习串成完整对齐流程的奠基论文。它证明了一个相对较小但经过 RLHF 的模型在用户偏好上能稳定击败更大却只做语言建模的基线,从此奠定了所有商用大模型的三段式训练范式。"}, + {"id": "paper:constitutional_ai", "label_en": "Constitutional AI", "label_zh": "Constitutional AI(宪法式自我对齐)", "kind": "paper", "tier": "A", "topic": "alignment", "phase": "core", "year": 2022, "summary_zh": "Constitutional AI 用一组写好的自然语言原则替代人类逐条偏好标注,让模型先自评再自改,最后再用强化学习把这些自评作为奖励信号。它的核心贡献是把对齐的人力瓶颈从打分员搬到原则书写者,给小团队复现 RLHF 提供了可行路径。"}, + {"id": "paper:react", "label_en": "ReAct", "label_zh": "ReAct(思考与行动交替)", "kind": "paper", "tier": "S", "topic": "llm_agent", "phase": "core", "year": 2022, "summary_zh": "ReAct 在提示中显式交替写出推理和工具调用两类 token,使语言模型既能像思维链一样自我推演,又能像工具使用者一样调用搜索与计算器纠正自身幻觉。它是绝大多数现代语言模型代理框架以及 Agent-Driver 这一类驾驶认知代理工作的提示骨架。"}, + {"id": "paper:reflexion", "label_en": "Reflexion", "label_zh": "Reflexion(语言式自我反思)", "kind": "paper", "tier": "A", "topic": "llm_agent", "phase": "core", "year": 2023, "summary_zh": "Reflexion 在每一轮失败之后让语言模型用自然语言写一段自我批评,并把这段批评作为下一轮提示的一部分,等价于一个文本梯度下降式的策略改进算子。它把强化学习里的奖励信号替换为自我生成的语言反馈,是 DiLu 等驾驶反思框架的直接灵感来源。"}, + {"id": "paper:tot", "label_en": "Tree-of-Thoughts", "label_zh": "Tree-of-Thoughts(思维树搜索)", "kind": "paper", "tier": "A", "topic": "reasoning", "phase": "core", "year": 2023, "summary_zh": "Tree-of-Thoughts 把单链思维链推广成对中间思考分支显式展开与回溯的搜索过程,让语言模型在每一步评估多条潜在思路再决定继续展开哪一条。它把推理本身建模为带启发式估值的搜索,为驾驶规划中的多假设展开提供了直接的语义类比。"}, + {"id": "paper:toolformer", "label_en": "Toolformer", "label_zh": "Toolformer(自监督学会工具调用)", "kind": "paper", "tier": "A", "topic": "llm_agent", "phase": "core", "year": 2023, "summary_zh": "Toolformer 让语言模型自己生成大量带工具调用标记的训练样本,并用调用结果是否降低后续 token 困惑度作为筛选信号,从而无需人工示范学会调用搜索、计算器与翻译器。这种自监督工具学习方法是把 API 集成进语言模型最干净的一条路径。"}, + {"id": "paper:voyager", "label_en": "VOYAGER", "label_zh": "VOYAGER(Minecraft 终身学习代理)", "kind": "paper", "tier": "A", "topic": "llm_agent", "phase": "frontier", "year": 2023, "summary_zh": "VOYAGER 在 Minecraft 中用 GPT-4 维护一个可执行技能库,把每个新学到的技能写成可调用的代码并不断累积成层次化课程,从而展示出无监督开放式探索能力。它是把语言模型作为持续学习元控制器的代表作,与驾驶中的长时域终身学习思路同构。"}, + {"id": "paper:swiftsage", "label_en": "SwiftSage", "label_zh": "SwiftSage(双系统语言代理)", "kind": "paper", "tier": "B", "topic": "llm_agent", "phase": "frontier", "year": 2023, "summary_zh": "SwiftSage 把代理拆成一个快速反应的小模型 Swift 与一个谨慎反思的大模型 Sage,按任务困难度在两者之间切换。它是 Daniel Kahneman 双系统假说在语言代理上的直接工程化,与 DriveVLM-Dual 在驾驶上的快慢双环架构思想完全一致。"}, + {"id": "paper:flamingo", "label_en": "Flamingo", "label_zh": "Flamingo(少样本视觉语言模型)", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "prereq", "year": 2022, "summary_zh": "Flamingo 在冻结的语言模型中插入交叉注意力门,让视觉特征以加性方式注入而不破坏原始语言能力,从而实现真正的少样本图文上下文学习。它是后来 LLaVA 与 BLIP-2 等开源 VLM 注入视觉特征到 LLM 的概念原型。"}, + {"id": "paper:palme", "label_en": "PaLM-E", "label_zh": "PaLM-E(具身多模态大模型)", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "core", "year": 2023, "summary_zh": "PaLM-E 把视觉、状态向量与语言一起编码成连续 token 序列送入 PaLM 主干,使一个模型可以同时回答视觉问答与生成机器人控制指令。它第一次系统性地展示了大语言模型预训练带来的常识可以正迁移到具身控制策略上。"}, + {"id": "paper:rt1", "label_en": "RT-1", "label_zh": "RT-1(机器人 transformer 1)", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "core", "year": 2022, "summary_zh": "RT-1 是 Google 用机器人采集数据训练的端到端 transformer 控制器,把图像与语言指令编码后离散化输出末端动作。它建立了机器人通用控制器的训练管线模板,也是 RT-2 与 OpenVLA 一系列后续视觉语言动作模型的直接前身。"}, + {"id": "paper:rt2", "label_en": "RT-2", "label_zh": "RT-2(视觉语言动作模型)", "kind": "paper", "tier": "S", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "RT-2 把动作输出重新编码成 PaLI-X 视觉语言模型词表中的文本 token,让一个原本只做视觉问答的大模型直接生成机器人动作序列。这一动作 token 化思想把控制问题转写成统一的自回归生成,是当前 VLA 范式的奠基工作。"}, + {"id": "paper:rtx", "label_en": "RT-X / Open X-Embodiment", "label_zh": "RT-X 与 Open X-Embodiment", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "RT-X 把来自二十多家实验室、几十种机器人本体的演示数据统一成 OpenX 格式并联合训练,证明跨本体训练能换来正迁移而非冲突。它把机器人视觉语言动作模型推进到大规模联合预训练阶段,是开源机器人基础模型的数据基石。"}, + {"id": "paper:openvla", "label_en": "OpenVLA", "label_zh": "OpenVLA(开源视觉语言动作模型)", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "OpenVLA 在 LLaMA-2 与 SigLIP 之上复现并开放了 RT-2 风格的动作 token 训练管线,给学术界第一份可下载权重的 7B 级 VLA 基线。它把 VLA 研究的入门门槛从大厂资源拉回到单机多卡,是开源机器人控制基础模型的标志性节点。"}, + {"id": "paper:octo", "label_en": "Octo", "label_zh": "Octo(开源跨本体策略)", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "Octo 用基于 transformer 的扩散动作头在 Open X-Embodiment 数据上训练出一个支持多本体多视角接口的通用策略。它的核心贡献是把多模态条件、多种动作空间和多个传感器配置统一进同一个可微管线,是另一条与 OpenVLA 并行的开源 VLA 路线。"}, + {"id": "paper:florence", "label_en": "Florence-2", "label_zh": "Florence-2(统一视觉基础模型)", "kind": "paper", "tier": "B", "topic": "vlm_vla", "phase": "core", "year": 2023, "summary_zh": "Florence-2 把检测、分割、关键点、描述等多种视觉任务都改写为 prompt 引导的文本生成,并在大规模图文数据上联合训练。它代表了一条把视觉任务全部归约为语言生成的统一基础模型路线,与 DETR 类纯结构通用化形成对照。"}, + {"id": "paper:internvl", "label_en": "InternVL", "label_zh": "InternVL(大规模开源 VLM)", "kind": "paper", "tier": "B", "topic": "vlm_vla", "phase": "core", "year": 2023, "summary_zh": "InternVL 把视觉编码器规模拉到与语言模型对齐的数十亿参数级别,并采用渐进式对齐策略训练,使开源 VLM 在中英文基准上首次接近 GPT-4V。它是后续国内驾驶领域 VLM 续训和评估的常用基座之一。"}, + {"id": "paper:cambrian", "label_en": "Cambrian-1", "label_zh": "Cambrian-1(以视觉为中心的 VLM)", "kind": "paper", "tier": "B", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "Cambrian-1 系统性比较了 20 多种视觉编码器接入大语言模型的方式,并提出空间视觉聚合连接器以缓解高分辨率视觉特征的稀释。它强调当前 VLM 瓶颈不在语言侧而在视觉侧,对驾驶等需要精细空间理解的领域有直接指导意义。"}, + {"id": "paper:sora", "label_en": "Sora", "label_zh": "Sora(视频扩散基础模型)", "kind": "paper", "tier": "S", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "Sora 用扩散 transformer 在压缩潜空间中生成长时高分辨率视频,把图像扩散模型的范式推广到时空联合建模。OpenAI 把它定位为通用物理世界模拟器原型,也使视频生成与世界模型的边界开始消失。"}, + {"id": "paper:veo", "label_en": "Veo", "label_zh": "Veo(Google 视频生成模型)", "kind": "paper", "tier": "B", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "Veo 是 Google DeepMind 推出的高分辨率长视频生成模型,强调对相机轨迹和场景指令的可控条件输入。它与 Sora 并列代表视频生成迈向可控物理模拟器的工业实践,对驾驶仿真数据合成有直接溢出价值。"}, + {"id": "paper:cosmos", "label_en": "NVIDIA Cosmos", "label_zh": "NVIDIA Cosmos(物理 AI 世界基础模型)", "kind": "paper", "tier": "A", "topic": "world_models", "phase": "frontier", "year": 2025, "summary_zh": "NVIDIA Cosmos 是面向具身与自动驾驶的视频世界基础模型族,提供扩散与自回归两条主干以及驾驶专用的条件控制接口。它把视频生成模型显式定位为机器人与车辆训练用的反事实数据合成器,是世界模型工程化最完整的工业方案之一。"}, + {"id": "paper:dit", "label_en": "DiT", "label_zh": "DiT(扩散 transformer)", "kind": "paper", "tier": "A", "topic": "world_models", "phase": "core", "year": 2022, "summary_zh": "DiT 用纯 transformer 主干替换 U-Net 作为扩散模型的去噪网络,证明在大规模图像扩散上 transformer 同样能享受 scaling law。它是 Sora、Stable Diffusion 3 与 Cosmos 等视频世界模型的共同结构基础。"}, + {"id": "paper:svd", "label_en": "Stable Video Diffusion", "label_zh": "Stable Video Diffusion", "kind": "paper", "tier": "B", "topic": "world_models", "phase": "core", "year": 2023, "summary_zh": "Stable Video Diffusion 把图像扩散模型在时间维度上加入卷积与注意力扩展,并发布开源权重,使学术界能够在视频生成与世界模型之间自由切换。它是 GAIA-1、DriveDreamer 等驾驶世界模型在权重层面常用的开源参考点。"}, + {"id": "paper:senna", "label_en": "Senna", "label_zh": "Senna(驾驶 VLM 元动作框架)", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "Senna 把驾驶决策拆成 VLM 输出语言元动作和小型端到端模型把元动作翻译成具体轨迹两层,让大模型只承担需要常识与意图理解的高层判断。它是驾驶领域元动作中介范式的代表,与 DriveVLM-Dual 并列。"}, + {"id": "paper:emma", "label_en": "EMMA", "label_zh": "EMMA(端到端多模态 Waymo 模型)", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "EMMA 是 Waymo 基于 Gemini 主干的端到端驾驶模型,将图像、地图与文本指令编码为 token,让 VLM 直接输出未来轨迹与解释性思维链。它展示了把闭源大模型作为驾驶单一神经栈的工业级可行性,但同时也带来推理延迟与可解释性新的挑战。"}, + {"id": "paper:drivelm", "label_en": "DriveLM", "label_zh": "DriveLM(图结构问答驾驶数据集与模型)", "kind": "paper", "tier": "A", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "DriveLM 在 nuScenes 上构造按感知、预测、规划组织的图结构问答数据集,并提出在 VLM 上做图链式推理的训练范式。它把驾驶任务显式建模成一个可被语言模型展开的有向因果图,为驾驶领域思维链评测提供基准。"}, + {"id": "paper:drivemlm", "label_en": "DriveMLM", "label_zh": "DriveMLM(多模态闭环驾驶大模型)", "kind": "paper", "tier": "B", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "DriveMLM 在 CARLA 闭环中把多视角图像、激光点云与语言指令一起送入 LLM,并输出与规则栈对接的离散行为决策。它是较早系统验证闭环条件下 LLM 决策可用性的工作,也暴露了模型延迟与决策一致性之间的硬约束。"}, + {"id": "paper:gpt_driver", "label_en": "GPT-Driver", "label_zh": "GPT-Driver(语言模型驾驶规划)", "kind": "paper", "tier": "B", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "GPT-Driver 把感知输出序列化为文本场景描述送入 GPT,让大语言模型以坐标 token 形式输出未来轨迹。它把驾驶规划完全压成自回归文本生成任务,为后续轨迹 token 化与动作 token 化研究提供了最简版本的概念证明。"}, + {"id": "paper:lmdrive", "label_en": "LMDrive", "label_zh": "LMDrive(闭环语言驾驶代理)", "kind": "paper", "tier": "B", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "LMDrive 在 CARLA 中接入 LLaMA 类大模型,用自然语言指令实时驱动车辆,并提出语言指令到航点的端到端转换机制。它是开源社区可复现的闭环语言驾驶代理基线,与商用闭环系统形成对照实验。"}, + {"id": "paper:prism1", "label_en": "Wayve PRISM-1", "label_zh": "Wayve PRISM-1(神经场驾驶仿真)", "kind": "paper", "tier": "B", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "Wayve PRISM-1 用四维神经场把真实采集的驾驶日志重建成可任意视角与可干预的动态场景,是 GAIA 系列世界模型的仿真互补工具。它把世界模型分成生成式与重建式两条线索,使闭环训练在没有外部模拟器的情况下也能进行。"}, + {"id": "paper:cot_wei2022", "label_en": "Chain-of-Thought", "label_zh": "Chain-of-Thought 提示", "kind": "paper", "tier": "S", "topic": "reasoning", "phase": "core", "year": 2022, "summary_zh": "Chain-of-Thought 论文系统性展示了在提示中加入中间推理步骤可以让大语言模型在算术、常识与符号推理任务上获得显著质变。它是后续 ReAct、Tree-of-Thoughts、Reflexion 以及所有 VLM 驾驶解释链工作的概念起点。"}, + {"id": "paper:self_consistency", "label_en": "Self-Consistency", "label_zh": "自一致性解码", "kind": "paper", "tier": "A", "topic": "reasoning", "phase": "core", "year": 2022, "summary_zh": "自一致性解码在思维链之上采样多条独立推理路径然后对最终答案投票,把单点推理升级为蒙特卡洛式集成。它是几乎无成本就能显著提升语言模型推理可靠性的标准技术,也常被驾驶 VLM 用作多假设规划的轻量化版本。"}, + {"id": "paper:debate", "label_en": "Multi-Agent Debate", "label_zh": "多智能体辩论", "kind": "paper", "tier": "B", "topic": "reasoning", "phase": "frontier", "year": 2023, "summary_zh": "多智能体辩论让多个语言模型实例针对同一问题相互质询并修正,最终由裁判模型综合出结论。它把对齐与可靠性问题转写为可扩展的多角色协作过程,为高风险驾驶决策提供另一条不依赖外部验证器的纠错机制。"}, + {"id": "paper:verifier", "label_en": "Process / Outcome Verifier", "label_zh": "过程验证器与结果验证器", "kind": "paper", "tier": "B", "topic": "reasoning", "phase": "frontier", "year": 2023, "summary_zh": "过程验证器对推理链的每一步分别打分,结果验证器只评估最终答案;二者都用监督数据训练成单独模型并作为生成器的搜索向导。它在 OpenAI 与 DeepMind 的数学推理工作中被反复验证,是把强化学习风格价值函数引回语言推理的关键桥梁。"}, + + {"id": "move:scale_data_then_let_emergent_capabilities_appear", "label_en": "Scale then let capabilities emerge", "label_zh": "先把数据与算力堆上去再观察涌现能力", "kind": "move", "tier": "move", "topic": "foundation_models", "phase": "core", "year": 2020, "summary_zh": "这是一种刻意把规模放在结构创新之前的研究姿态,方法者承认无法预测哪些能力会突然出现,因此先沿着已知 scaling law 把训练规模拉到下一级,再回过头去刻画新冒出来的能力。它是 GPT-3、PaLM 和 Sora 等里程碑工作背后共同的方法论起手式。"}, + {"id": "move:pretrain_with_contrastive_alignment_between_modalities", "label_en": "Cross-modal contrastive pretraining", "label_zh": "用跨模态对比学习把不同模态对齐到同一空间", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "prereq", "year": 2021, "summary_zh": "这一招把两条模态的编码器训练成把语义对应的样本互相拉近、把无关样本互相推远,从而在零样本任务中复用语言空间。CLIP 是其代表,但同样的对比对齐思想也驱动了 SigLIP、ImageBind 等多模态基础模型,是把视觉信号桥接到语言模型的最简洁可扩展手段。"}, + {"id": "move:fine_tune_with_instruction_data_then_align_with_preferences", "label_en": "Instruction tune then preference align", "label_zh": "先用指令数据微调再用偏好数据对齐", "kind": "move", "tier": "move", "topic": "alignment", "phase": "core", "year": 2022, "summary_zh": "这一招把对齐过程拆成两段:先用大量带格式的指令对让模型学会服从任务约定,再用人类或宪法式偏好对让模型学会在多个合规候选中挑出更好的那一个。它已经成为所有商用大模型的默认训练阶段划分,也是把基座模型变成可交付产品的最小工艺集。"}, + {"id": "move:plug_in_modality_encoder_to_frozen_language_model_via_projection", "label_en": "Plug modality encoder via projection", "label_zh": "用一个投影头把模态编码器插进冻结语言模型", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "core", "year": 2023, "summary_zh": "这一招保留预训练语言模型权重不动,只训练一个把视觉或音频特征线性映射到语言 token 空间的小投影模块,从而最大化复用昂贵语言能力。LLaVA、BLIP-2 与 MiniGPT-4 都是这一移动的实例,它也成为学术实验室能用单卡复现多模态大模型的关键工艺。"}, + {"id": "move:wrap_language_model_with_tool_calling_loop", "label_en": "Wrap LM with tool-calling loop", "label_zh": "在语言模型外面套一层工具调用循环", "kind": "move", "tier": "move", "topic": "llm_agent", "phase": "core", "year": 2023, "summary_zh": "把语言模型当成可以生成函数调用 token 的策略,再把调用结果回灌成新的上下文,循环直到任务完成。这一招把无状态生成器升级成可以与环境交互的代理,是 ReAct、Toolformer、Agent-Driver 等大量代理框架的共有骨架。"}, + {"id": "move:add_reflection_step_so_agent_critiques_its_own_output", "label_en": "Add reflection step for self-critique", "label_zh": "在代理循环中加入自我反思与批评步骤", "kind": "move", "tier": "move", "topic": "llm_agent", "phase": "core", "year": 2023, "summary_zh": "在每一轮行动之后专门留一段空间让模型回顾刚才的决策并写出文字批评,然后把这段批评作为下一轮的额外输入。它把强化学习中的奖励信号换成自然语言反馈,是 Reflexion、DiLu 等让代理在不更新权重的情况下持续改进的关键招式。"}, + {"id": "move:replace_explicit_action_head_with_tokenized_action_sequence", "label_en": "Tokenize action sequence", "label_zh": "把显式动作头替换为离散化的动作 token 序列", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "把连续控制量量化成有限词表,再让语言模型像写句子一样自回归生成动作 token,从而把控制与语言生成统一进同一套预训练。RT-2、OpenVLA、GPT-Driver 都用这一招把驾驶或机械臂控制问题改写为大模型友好的形式。"}, + {"id": "move:augment_supervised_training_with_counterfactual_or_synthetic_data", "label_en": "Augment with counterfactual synthetic data", "label_zh": "用反事实或合成数据扩充监督训练", "kind": "move", "tier": "move", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "这一招用世界模型或仿真器生成被有意修改过条件的反事实数据,让监督训练接触到现实数据里从未出现但物理上合理的极端情形。它是缓解长尾分布、解耦虚假相关、训练稳健驾驶策略的关键工具,CF-VLA 与 Cosmos 都把它作为核心机制。"}, + {"id": "move:condition_video_generative_model_on_control_action_for_world_model", "label_en": "Condition video generator on action", "label_zh": "用控制动作条件化视频生成模型构成世界模型", "kind": "move", "tier": "move", "topic": "world_models", "phase": "frontier", "year": 2023, "summary_zh": "在视频扩散或自回归生成模型上加入未来动作或控制信号作为条件输入,使模型能回答如果车辆这样行动场景会如何演化。GAIA-1、DriveDreamer 与 Cosmos 都用这一招把生成模型升级成可被规划器查询的隐式物理引擎。"}, + {"id": "move:use_retrieval_augmented_memory_to_extend_context", "label_en": "Retrieval-augmented memory", "label_zh": "用检索增强的外部记忆扩展上下文", "kind": "move", "tier": "move", "topic": "llm_agent", "phase": "core", "year": 2022, "summary_zh": "在固定窗口的语言模型外面挂一个可读写的向量索引,把长期经验存到外部库里,需要时检索回来拼进上下文。这一招把有限上下文窗口扩展成事实上的无限记忆,是构建终身学习代理与驾驶经验库的标准工程模式。"}, + {"id": "move:cast_reasoning_as_search_over_thought_tree", "label_en": "Cast reasoning as tree search", "label_zh": "把推理过程刻画为思维树上的搜索", "kind": "move", "tier": "move", "topic": "reasoning", "phase": "frontier", "year": 2023, "summary_zh": "在每一步推理之后保留多个候选展开方向并用启发式或验证器评估,再选择有前途的分支继续展开,必要时回溯。Tree-of-Thoughts、AlphaCode 与各类数学推理工作都用这一招把语言生成提升到带显式状态搜索的层级。"}, + {"id": "move:co_finetune_language_model_with_action_data_jointly", "label_en": "Co-finetune LM with action data", "label_zh": "把动作数据与语言数据联合微调到同一模型", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "在 VLA 训练中把机器人或驾驶动作数据与通用图文问答数据按比例混合进行联合微调,避免动作微调灾难性遗忘语言能力。RT-2、OpenVLA 与 EMMA 都依赖这一招在不丧失视觉问答能力的前提下学会输出动作。"}, + {"id": "move:use_self_play_to_generate_unlimited_training_signal", "label_en": "Self-play for unlimited signal", "label_zh": "用自博弈生成无尽训练信号", "kind": "move", "tier": "move", "topic": "reasoning", "phase": "core", "year": 2017, "summary_zh": "让模型与自己博弈或互相批评,从而把训练信号的瓶颈从人工标注搬到计算量上。AlphaZero 是其经典例证,多智能体辩论与宪法式自我改写则是其在语言模型时代的等价物,对扩展驾驶 corner case 训练同样有启示。"}, + {"id": "move:distill_large_model_into_specialist_for_deployment", "label_en": "Distill into deployable specialist", "label_zh": "把大模型蒸馏成可部署的专用模型", "kind": "move", "tier": "move", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "把缓慢但能干的大模型当成教师生成行为数据或软标签,再用一个小很多的学生网络去拟合,使最终上车的模型既保留大模型常识又能满足实时约束。它是把 VLM 与 VLA 落到车端控制器的几乎所有工业线的必走步骤。"}, + {"id": "move:rewrite_continuous_video_as_token_sequence_for_transformer_world_model", "label_en": "Tokenize video for transformer world model", "label_zh": "把连续视频改写成离散 token 喂给 transformer 世界模型", "kind": "move", "tier": "move", "topic": "world_models", "phase": "frontier", "year": 2023, "summary_zh": "用 VAE 或 VQ-VAE 把每一帧压缩成离散视觉 token,再把整段视频拼成 token 序列让 transformer 像处理语言那样建模。GAIA-1、Genie 与 Cosmos 自回归分支都依赖这一招把世界模型嵌入到统一 token 化框架。"}, + {"id": "move:condition_on_language_meta_action_then_emit_low_level_action", "label_en": "Language meta-action then low-level action", "label_zh": "先输出语言元动作再翻译成低层动作", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "在驾驶或机械臂决策中先让 VLM 输出诸如让行、跟车、避让等少量语言元动作,再让小型专用模型把元动作转译为连续轨迹或电机指令。Senna 与 DriveVLM-Dual 都是这一招的代表,把语义判断与几何细化解耦。"}, + {"id": "move:cache_kv_state_to_amortize_long_context", "label_en": "Cache KV state to amortize context", "label_zh": "缓存键值状态以摊销长上下文成本", "kind": "move", "tier": "move", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "把推理时的注意力键值张量保存下来,下一步只追加新的 token 并复用旧缓存,把长序列推理成本从二次降到线性。它是把大模型推理时延控制到车端可接受范围的最常用工程招式,也是 vLLM 与 TensorRT-LLM 的核心机制。"}, + {"id": "move:speculative_decoding_with_draft_model", "label_en": "Speculative decoding with draft model", "label_zh": "用小草稿模型推测式解码加速大模型", "kind": "move", "tier": "move", "topic": "foundation_models", "phase": "frontier", "year": 2023, "summary_zh": "用一个便宜的小模型先生成若干候选 token,再让大模型一次性并行验证这些候选并接受其中可被复现的前缀,从而在不改变分布的前提下显著降低延迟。它是让大模型在驾驶等实时场景里勉强可用的关键推理优化手段。"}, + {"id": "move:freeze_visual_encoder_and_only_train_connector", "label_en": "Freeze visual encoder, train connector", "label_zh": "冻结视觉编码器只训练连接器", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "core", "year": 2023, "summary_zh": "把代价昂贵的视觉编码器和语言模型都冻结,仅训练中间的小型连接器或交叉注意力,使多模态对齐可以在很小算力下完成。这是 BLIP-2、LLaVA 初代以及众多驾驶 VLM 落地工作的事实标准入门姿态。"}, + {"id": "move:use_diffusion_head_for_continuous_action", "label_en": "Diffusion head for continuous action", "label_zh": "用扩散头建模连续动作分布", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "在策略网络末端用条件扩散模型采样连续控制量,从而自然刻画多模态动作分布和不确定性,避开离散 token 量化误差。Octo、Diffusion Policy 与多种轨迹规划工作都采用这一招在精细控制场景下替代单点回归。"}, + {"id": "move:treat_planning_as_autoregressive_trajectory_generation", "label_en": "Planning as autoregressive trajectory", "label_zh": "把规划问题视为自回归轨迹生成", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "把未来一段时间的航点序列当作 token 序列让 transformer 一步一步生成,把规划与语言生成统一进同一接口。GPT-Driver、EMMA 与多种端到端规划网络都用这一招把规划架在大模型生成能力之上。"}, + {"id": "move:use_world_model_rollout_as_critic_for_policy", "label_en": "World-model rollout as critic", "label_zh": "用世界模型滚动作为策略的批评者", "kind": "move", "tier": "move", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "让世界模型对策略提议的多个候选动作分别模拟若干步未来,再用一个评估函数比较结果,从而把世界模型当成可微或可查询的批评者。Dreamer 家族、CF-VLA 与基于 Sora 类模型的规划研究都共享这一招的思想。"}, + {"id": "move:long_horizon_via_hierarchical_subgoal", "label_en": "Long-horizon via hierarchical subgoal", "label_zh": "用层次化子目标处理长时域决策", "kind": "move", "tier": "move", "topic": "llm_agent", "phase": "frontier", "year": 2023, "summary_zh": "把长时任务先用语言模型分解成有序子目标列表,再为每个子目标调用低层策略或者再次递归分解。VOYAGER、SwiftSage 与多种规划代理都用这一招对抗有限上下文窗口与稀疏奖励,把任务难度向下展开成可解决的小步。"}, + {"id": "move:prompt_chain_with_explicit_persona_roles", "label_en": "Prompt chain with persona roles", "label_zh": "用显式角色化的提示链分工", "kind": "move", "tier": "move", "topic": "llm_agent", "phase": "core", "year": 2023, "summary_zh": "把同一个大模型用不同系统提示扮演规划者、批评者、执行者等角色再彼此对话,把单体语言模型变成可解释的多角色管线。它把对齐与协作问题搬到提示工程层面,是大量驾驶认知代理工作的低成本起步形态。"}, + {"id": "move:contrast_corner_case_against_normal_case_in_training", "label_en": "Contrast corner vs normal cases", "label_zh": "在训练中显式对比 corner case 与常规情况", "kind": "move", "tier": "move", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "在数据组织阶段刻意把同一场景的正常版本与被扰动出 corner case 的版本配对呈现,使模型学到的不只是平均行为而是变化的边界。这一招在驾驶安全和反事实训练里尤为关键,是 CF-VLA 与某些 Cosmos 子任务的核心设计。"}, + {"id": "move:evaluate_open_loop_then_close_loop_for_realism", "label_en": "Open-loop then closed-loop eval", "label_zh": "先做开环评估再切到闭环验证现实性", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "core", "year": 2022, "summary_zh": "先在静态数据集上跑指标筛掉明显不合格的模型,再在仿真或路测里做闭环以暴露分布偏移与累积误差。它是驾驶研究从论文走向工程必经的两阶段评估姿态,避免开环指标好看但闭环失稳的常见陷阱。"}, + {"id": "move:use_language_explanation_as_auxiliary_supervision", "label_en": "Language explanation as auxiliary supervision", "label_zh": "把语言解释作为辅助监督信号", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "core", "year": 2024, "summary_zh": "在训练驾驶策略或感知网络时加入对决策原因的自然语言解释作为附加输出,要求模型同时回答做了什么和为什么。LINGO、DriveLM 与多种带解释 VLA 工作都用这一招把模型逼向人类可审计的内部表征。"}, + + {"id": "problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop", "label_en": "Hallucinated action in safety-critical loop", "label_zh": "VLM 在安全关键回路中产生幻觉动作", "kind": "problem", "tier": "problem", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "视觉语言模型擅长生成看起来合理的解释和动作,但这种合理性并不蕴含物理可执行性,一旦把它直接闭环到车辆控制就可能输出违反约束甚至危险的动作。如何在保持表达力的同时给 VLA 加上硬约束验证,是当前安全可部署的核心难题。"}, + {"id": "problem:grounding_language_token_to_continuous_physical_world", "label_en": "Grounding language to physical world", "label_zh": "把离散语言 token 接地到连续物理世界", "kind": "problem", "tier": "problem", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "语言模型在离散符号空间中训练,但要驱动机械臂或车辆就必须输出有度量含义的连续量,这之间存在天然的语义到物理的接地鸿沟。如何系统性地学到这种接地,而不是依赖手工动作 token 表,是 VLA 与具身智能的根本未解问题。"}, + {"id": "problem:latency_budget_for_large_model_in_realtime_control", "label_en": "Latency budget for large model control", "label_zh": "大模型在实时控制中的延迟预算", "kind": "problem", "tier": "problem", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "车辆控制循环需要在十毫秒量级响应,而当前 VLM 与 VLA 在车端硬件上的单次推理常常超过这个预算几个数量级。如何用蒸馏、缓存、推测式解码与双系统架构把大模型推理压进控制周期,决定了 VLA 是否真能上车。"}, + {"id": "problem:long_horizon_reasoning_with_finite_context_window", "label_en": "Long-horizon reasoning with finite context", "label_zh": "有限上下文窗口下的长时域推理", "kind": "problem", "tier": "problem", "topic": "llm_agent", "phase": "core", "year": 2023, "summary_zh": "驾驶任务跨越分钟甚至小时尺度,但语言模型的注意力代价随窗口长度二次增长,必须在固定窗口内表达远超窗口的历史信息。如何用外部记忆、层次化总结与世界模型来弥补这一矛盾,是把代理范式推进到真实长时任务的瓶颈。"}, + {"id": "problem:zero_shot_generalization_to_unseen_driving_scenes", "label_en": "Zero-shot generalization to unseen scenes", "label_zh": "对未见驾驶场景的零样本泛化", "kind": "problem", "tier": "problem", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "驾驶世界几乎不可能被采集穷尽,模型必须在训练分布之外保持合理行为。VLM 因为预训练数据广泛而被寄予厚望,但实证显示它们在精细几何与新国家路况上仍频繁失败,因此零样本泛化的真正机制与边界还远未被理解。"}, + {"id": "problem:fine_grained_spatial_understanding_in_vision_language_model", "label_en": "Fine-grained spatial understanding in VLM", "label_zh": "视觉语言模型的精细空间理解", "kind": "problem", "tier": "problem", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "当前 VLM 在描述场景大意时表现良好,但要它说出两辆车之间的精确距离、车道几何或视差关系时几乎一律失败。这种细粒度空间推理的缺失直接限制了 VLM 在驾驶感知与规划中的可信度,是 Cambrian 等工作集中攻击的方向。"}, + {"id": "problem:counterfactual_reasoning_about_other_agents_intent", "label_en": "Counterfactual reasoning about other agents", "label_zh": "对他车意图的反事实推理", "kind": "problem", "tier": "problem", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "安全驾驶要求模型能想象其他交通参与者在不同假设下会怎样反应,而不是只对实际观察到的轨迹做拟合。如何让 VLA 或世界模型显式表示并搜索这些反事实分支,是 CF-VLA、Cosmos 等工作正在试图把握的研究问题。"}, + {"id": "problem:open_world_corner_case_synthesis_for_training", "label_en": "Open-world corner case synthesis", "label_zh": "开放世界 corner case 的合成", "kind": "problem", "tier": "problem", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "现实道路里真正危险的 corner case 罕见到无法靠采集解决,必须由生成模型在保持物理合理性的前提下大量合成。如何衡量合成 corner case 的覆盖率与有效性,又如何避免模型只学到合成数据上的虚假相关,是世界模型工程化的核心未解问题。"}, + {"id": "problem:evaluation_gap_between_offline_benchmark_and_closed_loop", "label_en": "Offline-to-closed-loop evaluation gap", "label_zh": "离线基准与闭环表现之间的评估鸿沟", "kind": "problem", "tier": "problem", "topic": "vlm_vla", "phase": "core", "year": 2023, "summary_zh": "目前的离线 VQA 与轨迹相似度指标无法可靠预测模型在闭环里的真实安全性,许多在 DriveLM 上得高分的 VLM 在 CARLA 闭环里反而失败。如何设计能贯通开闭环的评估指标,是 VLA 研究是否能形成可累积进展的方法学前提。"}, + {"id": "problem:catastrophic_forgetting_after_action_finetuning", "label_en": "Catastrophic forgetting after action finetune", "label_zh": "动作微调后的灾难性遗忘", "kind": "problem", "tier": "problem", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "当 VLM 被进一步用动作数据微调成 VLA 时,原本强大的视觉问答与语言推理能力常常显著退化,使得模型在需要常识的边界情况下反而比微调前更糟。如何在保持语言能力的同时引入动作能力,是 RT-2、OpenVLA 等工作面对的工程难题。"}, + + {"id": "insight:language_is_compressed_world_model_for_human_priors", "label_en": "Language as compressed world model", "label_zh": "语言是人类先验的压缩世界模型", "kind": "insight", "tier": "insight", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "人类用语言把对世界因果、物理和社会的判断压缩在文本里,使得在大规模文本上预训练得到的模型自动继承了这些先验。这一洞察解释了为什么纯文本预训练的 LLM 在驾驶常识、社会博弈等领域不需要进一步训练就能给出合理答案,也支撑了语言作为通用世界模型代理的研究路线。"}, + {"id": "insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models", "label_en": "Scaling unlocks emergent capabilities", "label_zh": "扩大规模解锁小模型上不存在的能力", "kind": "insight", "tier": "insight", "topic": "foundation_models", "phase": "core", "year": 2022, "summary_zh": "经验上某些能力例如多步推理、零样本指令跟随只在模型规模与数据量越过某个阈值后才突然出现,而无法在小模型上靠精调获得。这一洞察支撑了苦涩教训所倡导的把工程资源投入到通用扩展而非领域规则上的方法论选择。"}, + {"id": "insight:world_model_video_diffusion_is_implicit_physics_engine", "label_en": "Video diffusion as implicit physics engine", "label_zh": "视频扩散模型是隐式物理引擎", "kind": "insight", "tier": "insight", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "在足够大的视频数据上训练的扩散模型自动学到了惯性、碰撞、光照与物体持久性等近似物理规律,使它们成为不需要显式方程的可查询世界模型。这一洞察使 Sora、GAIA、Cosmos 等模型被重新定位为机器人和驾驶的通用模拟器,而不仅仅是内容生成器。"}, + {"id": "insight:agent_loop_is_just_iterated_conditional_generation", "label_en": "Agent loop is iterated conditional generation", "label_zh": "代理循环本质是反复条件生成", "kind": "insight", "tier": "insight", "topic": "llm_agent", "phase": "core", "year": 2023, "summary_zh": "ReAct、Reflexion、VOYAGER 等代理框架虽然形态各异,但都可以被统一理解为把工具结果与历史观察拼回上下文后再次条件生成下一段动作。这一洞察把代理设计简化为如何构造每一步的条件输入与如何聚合长期记忆两件事,是搭建驾驶认知代理的统一抽象。"}, + {"id": "insight:tool_use_extends_language_model_into_environment_grounded_actor", "label_en": "Tool use extends LM into actor", "label_zh": "工具使用把语言模型扩展为接地于环境的执行者", "kind": "insight", "tier": "insight", "topic": "llm_agent", "phase": "core", "year": 2023, "summary_zh": "一旦语言模型可以生成结构化函数调用并消费其返回值,它就从无状态文本生成器变成了可以查询事实、操作仿真器与控制车辆的环境接地执行者。这一洞察是 Toolformer、ReAct 与 Agent-Driver 共同的方法论核心,也把对齐重心从输出文本搬到了选择动作。"}, + {"id": "insight:counterfactual_replanning_separates_intent_from_execution", "label_en": "Counterfactual replanning separates intent from execution", "label_zh": "反事实重规划把意图与执行解耦", "kind": "insight", "tier": "insight", "topic": "world_models", "phase": "frontier", "year": 2025, "summary_zh": "通过让模型对相同高层意图模拟多种动作轨迹再选择最优,可以把语义层面的我想做什么与几何层面的我该怎么做分离开来。CF-VLA 把这一思想工程化,使 VLM 输出的意图可以经由世界模型反事实验证后再交给底层控制器执行。"}, + {"id": "insight:foundation_model_decouples_perception_from_task_specific_training", "label_en": "Foundation model decouples perception from task training", "label_zh": "基础模型把感知与任务特定训练解耦", "kind": "insight", "tier": "insight", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "DINOv2、SAM、Florence 等通用感知基础模型把以前各任务都要重复训练的视觉骨干变成可复用的冻结服务,使下游 AD 工作只需要在轻量任务头上微调。这一洞察重塑了感知研究的劳动分工,把绝大多数科研价值压到任务设计与数据策展上。"}, + {"id": "insight:dual_system_fast_slow_loop_marries_reactive_and_deliberative_control", "label_en": "Dual-system fast-slow loop", "label_zh": "快慢双系统循环融合反应式与审议式控制", "kind": "insight", "tier": "insight", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "Kahneman 的快慢思维启发的双系统架构在 SwiftSage 与 DriveVLM-Dual 中被工程化为高频小模型与低频大模型并行的控制结构。这一洞察既保留了大模型常识又满足车端实时性,是当前 VLA 落地的事实标准形态。"}, + {"id": "insight:emergent_planning_from_next_token_prediction_alone", "label_en": "Emergent planning from next-token prediction", "label_zh": "纯下一 token 预测中涌现出的规划能力", "kind": "insight", "tier": "insight", "topic": "reasoning", "phase": "frontier", "year": 2023, "summary_zh": "尽管语言模型只优化下一个 token 的对数似然,但在足够规模与数据下它们表现出对未来若干步的隐式规划行为,包括在数学证明与代码生成中明显的目标驱动结构。这一洞察暗示规划与生成在足够大的模型上趋同,对端到端驾驶研究有深远启示。"}, + {"id": "insight:alignment_is_constraint_satisfaction_over_generation", "label_en": "Alignment as constraint satisfaction", "label_zh": "对齐本质是对生成的约束满足", "kind": "insight", "tier": "insight", "topic": "alignment", "phase": "core", "year": 2023, "summary_zh": "RLHF、DPO 与宪法式自我对齐都可以被看作在已经强大的生成分布上添加额外约束,让满足偏好或原则的样本概率上升而不满足的样本概率下降。这一视角统一了多种对齐技术,也把安全驾驶决策的对齐问题嵌入相同的形式框架。"}, + {"id": "insight:open_weight_release_compounds_research_velocity", "label_en": "Open weights compound research velocity", "label_zh": "开放权重发布复利化研究速度", "kind": "insight", "tier": "insight", "topic": "foundation_models", "phase": "core", "year": 2023, "summary_zh": "LLaMA、Mistral、Qwen、OpenVLA 等开源权重的发布让任何研究者都能在强基线上做受控实验,而不必从零训练,使整个领域的实验速度形成正反馈。这一洞察解释了为什么开源策略在长期上对方法论进步的贡献往往超过单点最强闭源模型。"}, + + {"id": "paradigm:foundation_model_axis", "label_en": "Foundation Model Axis", "label_zh": "基础模型与 VLA 轴范式总览", "kind": "paradigm", "tier": "paradigm", "topic": "foundation_models", "phase": "core", "year": 2024, "summary_zh": "这一范式把所有具备通用能力的视觉、语言、视频、动作模型组织在同一研究轴上,强调它们共享 transformer 主干、自监督预训练与下游联合微调的相同方法论。它是组织从 GPT-3 到 Cosmos、再到 EMMA 与 CF-VLA 这一巨大谱系的统一坐标系。"}, + {"id": "paradigm:world_model_paradigm", "label_en": "World Model Paradigm", "label_zh": "世界模型范式", "kind": "paradigm", "tier": "paradigm", "topic": "world_models", "phase": "frontier", "year": 2024, "summary_zh": "世界模型范式主张把环境动力学独立学到一个可被规划器查询或可被策略联合训练的生成模型里,从而把决策与感知解耦。从 Ha 与 Schmidhuber 的 World Models 到 Dreamer、GAIA-1、DriveDreamer 和 Cosmos,这一范式贯穿强化学习与自动驾驶研究的多个时代。"}, + {"id": "paradigm:llm_agent_paradigm", "label_en": "LLM Agent Paradigm", "label_zh": "大模型代理范式", "kind": "paradigm", "tier": "paradigm", "topic": "llm_agent", "phase": "frontier", "year": 2024, "summary_zh": "大模型代理范式把语言模型当成具备状态、工具、记忆的通用决策器,并通过工具循环、反思与层次分解构造长时域行为。它在驾驶领域以 Agent-Driver、DiLu、DriveVLM-Dual 等形态出现,是把语言模型常识转化为可执行驾驶策略的主要研究路径。"}, + {"id": "paradigm:vla_paradigm", "label_en": "VLA Paradigm", "label_zh": "视觉语言动作范式", "kind": "paradigm", "tier": "paradigm", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "视觉语言动作范式把视觉、语言与控制动作统一进同一个自回归或扩散生成模型,让一个网络同时承担感知、推理与控制三个传统上分离的角色。RT-2、OpenVLA、EMMA、CF-VLA 都是这一范式在机器人与驾驶上的代表实例。"} + ], + + "edges": [ + {"source": "paper:gpt3", "target": "paper:gpt4", "rel": "extends"}, + {"source": "paper:gpt4", "target": "paper:gpt4v", "rel": "extends"}, + {"source": "paper:gpt4", "target": "paper:claude", "rel": "parallel"}, + {"source": "paper:gpt4", "target": "paper:gemini", "rel": "parallel"}, + {"source": "paper:gpt3", "target": "paper:llama", "rel": "parallel"}, + {"source": "paper:llama", "target": "paper:mistral", "rel": "extends"}, + {"source": "paper:llama", "target": "paper:qwen", "rel": "parallel"}, + {"source": "paper:gpt3", "target": "paper:instructgpt", "rel": "extends"}, + {"source": "paper:instructgpt", "target": "paper:gpt4", "rel": "feeds"}, + {"source": "paper:instructgpt", "target": "paper:rlhf_dpo", "rel": "covers"}, + {"source": "paper:rlhf_dpo", "target": "paper:constitutional_ai", "rel": "parallel"}, + {"source": "paper:constitutional_ai", "target": "paper:claude", "rel": "feeds"}, + + {"source": "paper:gpt3", "target": "paper:cot_wei2022", "rel": "extends"}, + {"source": "paper:cot_wei2022", "target": "paper:self_consistency", "rel": "extends"}, + {"source": "paper:cot_wei2022", "target": "paper:react", "rel": "extends"}, + {"source": "paper:cot_wei2022", "target": "paper:tot", "rel": "extends"}, + {"source": "paper:react", "target": "paper:reflexion", "rel": "extends"}, + {"source": "paper:react", "target": "paper:toolformer", "rel": "parallel"}, + {"source": "paper:react", "target": "paper:voyager", "rel": "feeds"}, + {"source": "paper:reflexion", "target": "paper:voyager", "rel": "feeds"}, + {"source": "paper:tot", "target": "paper:verifier", "rel": "parallel"}, + {"source": "paper:self_consistency", "target": "paper:verifier", "rel": "parallel"}, + {"source": "paper:debate", "target": "paper:verifier", "rel": "parallel"}, + {"source": "paper:react", "target": "paper:swiftsage", "rel": "feeds"}, + + {"source": "paper:llava", "target": "paper:flamingo", "rel": "parallel"}, + {"source": "paper:flamingo", "target": "paper:llava", "rel": "prereq"}, + {"source": "paper:llava", "target": "paper:internvl", "rel": "extends"}, + {"source": "paper:llava", "target": "paper:cambrian", "rel": "extends"}, + {"source": "paper:florence", "target": "paper:cambrian", "rel": "parallel"}, + {"source": "paper:qwen", "target": "paper:internvl", "rel": "parallel"}, + + {"source": "paper:llava", "target": "paper:palme", "rel": "parallel"}, + {"source": "paper:palme", "target": "paper:rt2", "rel": "prereq"}, + {"source": "paper:rt1", "target": "paper:rt2", "rel": "extends"}, + {"source": "paper:rt2", "target": "paper:rtx", "rel": "extends"}, + {"source": "paper:rtx", "target": "paper:openvla", "rel": "feeds"}, + {"source": "paper:rt2", "target": "paper:openvla", "rel": "extends"}, + {"source": "paper:rtx", "target": "paper:octo", "rel": "feeds"}, + {"source": "paper:openvla", "target": "paper:octo", "rel": "parallel"}, + + {"source": "paper:vaswani2017", "target": "paper:dit", "rel": "prereq"}, + {"source": "paper:dit", "target": "paper:sora", "rel": "prereq"}, + {"source": "paper:dit", "target": "paper:svd", "rel": "prereq"}, + {"source": "paper:svd", "target": "paper:sora", "rel": "parallel"}, + {"source": "paper:sora", "target": "paper:veo", "rel": "parallel"}, + {"source": "paper:sora", "target": "paper:cosmos", "rel": "feeds"}, + {"source": "paper:svd", "target": "paper:cosmos", "rel": "prereq"}, + {"source": "paper:gaia1", "target": "paper:cosmos", "rel": "parallel"}, + {"source": "paper:drivedreamer", "target": "paper:cosmos", "rel": "parallel"}, + {"source": "paper:gaia1", "target": "paper:prism1", "rel": "parallel"}, + {"source": "paper:drivedreamer", "target": "paper:prism1", "rel": "parallel"}, + + {"source": "paper:2402.12289", "target": "paper:senna", "rel": "parallel"}, + {"source": "paper:gemini", "target": "paper:emma", "rel": "prereq"}, + {"source": "paper:rt2", "target": "paper:emma", "rel": "parallel"}, + {"source": "paper:2402.12289", "target": "paper:emma", "rel": "parallel"}, + {"source": "paper:2311.10813", "target": "paper:drivemlm", "rel": "parallel"}, + {"source": "paper:2402.12289", "target": "paper:drivemlm", "rel": "parallel"}, + {"source": "paper:llava", "target": "paper:drivelm", "rel": "feeds"}, + {"source": "paper:drivelm", "target": "paper:senna", "rel": "feeds"}, + {"source": "paper:gpt_driver", "target": "paper:lmdrive", "rel": "parallel"}, + {"source": "paper:gpt3", "target": "paper:gpt_driver", "rel": "feeds"}, + {"source": "paper:llama", "target": "paper:lmdrive", "rel": "feeds"}, + {"source": "paper:lingo2", "target": "paper:senna", "rel": "parallel"}, + {"source": "paper:lingo2", "target": "paper:prism1", "rel": "parallel"}, + + {"source": "paper:emma", "target": "paper:2512.24426", "rel": "parallel"}, + {"source": "paper:senna", "target": "paper:2512.24426", "rel": "feeds"}, + {"source": "paper:cosmos", "target": "paper:2512.24426", "rel": "feeds"}, + {"source": "paper:openvla", "target": "paper:2512.24426", "rel": "parallel"}, + + {"source": "paper:gpt3", "target": "move:scale_data_then_let_emergent_capabilities_appear", "rel": "covers"}, + {"source": "paper:sora", "target": "move:scale_data_then_let_emergent_capabilities_appear", "rel": "covers"}, + {"source": "essay:bitter_lesson", "target": "move:scale_data_then_let_emergent_capabilities_appear", "rel": "covers"}, + {"source": "paper:llava", "target": "move:pretrain_with_contrastive_alignment_between_modalities", "rel": "covers"}, + {"source": "paper:flamingo", "target": "move:pretrain_with_contrastive_alignment_between_modalities", "rel": "covers"}, + {"source": "paper:instructgpt", "target": "move:fine_tune_with_instruction_data_then_align_with_preferences", "rel": "covers"}, + {"source": "paper:constitutional_ai", "target": "move:fine_tune_with_instruction_data_then_align_with_preferences", "rel": "covers"}, + {"source": "paper:rlhf_dpo", "target": "move:fine_tune_with_instruction_data_then_align_with_preferences", "rel": "covers"}, + {"source": "paper:llava", "target": "move:plug_in_modality_encoder_to_frozen_language_model_via_projection", "rel": "covers"}, + {"source": "paper:flamingo", "target": "move:plug_in_modality_encoder_to_frozen_language_model_via_projection", "rel": "covers"}, + {"source": "paper:palme", "target": "move:plug_in_modality_encoder_to_frozen_language_model_via_projection", "rel": "covers"}, + + {"source": "paper:react", "target": "move:wrap_language_model_with_tool_calling_loop", "rel": "covers"}, + {"source": "paper:toolformer", "target": "move:wrap_language_model_with_tool_calling_loop", "rel": "covers"}, + {"source": "paper:2311.10813", "target": "move:wrap_language_model_with_tool_calling_loop", "rel": "covers"}, + {"source": "paper:reflexion", "target": "move:add_reflection_step_so_agent_critiques_its_own_output", "rel": "covers"}, + {"source": "paper:2309.16292", "target": "move:add_reflection_step_so_agent_critiques_its_own_output", "rel": "covers"}, + {"source": "paper:voyager", "target": "move:add_reflection_step_so_agent_critiques_its_own_output", "rel": "covers"}, + + {"source": "paper:rt2", "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", "rel": "covers"}, + {"source": "paper:openvla", "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", "rel": "covers"}, + {"source": "paper:gpt_driver", "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", "rel": "covers"}, + {"source": "paper:emma", "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", "rel": "covers"}, + + {"source": "paper:2512.24426", "target": "move:augment_supervised_training_with_counterfactual_or_synthetic_data", "rel": "covers"}, + {"source": "paper:cosmos", "target": "move:augment_supervised_training_with_counterfactual_or_synthetic_data", "rel": "covers"}, + {"source": "paper:drivedreamer", "target": "move:augment_supervised_training_with_counterfactual_or_synthetic_data", "rel": "covers"}, + + {"source": "paper:gaia1", "target": "move:condition_video_generative_model_on_control_action_for_world_model", "rel": "covers"}, + {"source": "paper:drivedreamer", "target": "move:condition_video_generative_model_on_control_action_for_world_model", "rel": "covers"}, + {"source": "paper:cosmos", "target": "move:condition_video_generative_model_on_control_action_for_world_model", "rel": "covers"}, + + {"source": "paper:voyager", "target": "move:use_retrieval_augmented_memory_to_extend_context", "rel": "covers"}, + {"source": "paper:2309.16292", "target": "move:use_retrieval_augmented_memory_to_extend_context", "rel": "covers"}, + + {"source": "paper:tot", "target": "move:cast_reasoning_as_search_over_thought_tree", "rel": "covers"}, + {"source": "paper:silver2017_alphazero", "target": "move:cast_reasoning_as_search_over_thought_tree", "rel": "covers"}, + {"source": "paper:verifier", "target": "move:cast_reasoning_as_search_over_thought_tree", "rel": "covers"}, + + {"source": "paper:rt2", "target": "move:co_finetune_language_model_with_action_data_jointly", "rel": "covers"}, + {"source": "paper:openvla", "target": "move:co_finetune_language_model_with_action_data_jointly", "rel": "covers"}, + {"source": "paper:emma", "target": "move:co_finetune_language_model_with_action_data_jointly", "rel": "covers"}, + + {"source": "paper:silver2017_alphazero", "target": "move:use_self_play_to_generate_unlimited_training_signal", "rel": "covers"}, + {"source": "paper:debate", "target": "move:use_self_play_to_generate_unlimited_training_signal", "rel": "covers"}, + {"source": "paper:constitutional_ai", "target": "move:use_self_play_to_generate_unlimited_training_signal", "rel": "covers"}, + + {"source": "paper:gpt4", "target": "move:distill_large_model_into_specialist_for_deployment", "rel": "covers"}, + {"source": "paper:2402.12289", "target": "move:distill_large_model_into_specialist_for_deployment", "rel": "covers"}, + {"source": "paper:senna", "target": "move:distill_large_model_into_specialist_for_deployment", "rel": "covers"}, + + {"source": "paper:gaia1", "target": "move:rewrite_continuous_video_as_token_sequence_for_transformer_world_model", "rel": "covers"}, + {"source": "paper:cosmos", "target": "move:rewrite_continuous_video_as_token_sequence_for_transformer_world_model", "rel": "covers"}, + + {"source": "paper:senna", "target": "move:condition_on_language_meta_action_then_emit_low_level_action", "rel": "covers"}, + {"source": "paper:2402.12289", "target": "move:condition_on_language_meta_action_then_emit_low_level_action", "rel": "covers"}, + {"source": "paper:2512.24426", "target": "move:condition_on_language_meta_action_then_emit_low_level_action", "rel": "covers"}, + + {"source": "paper:gpt4", "target": "move:cache_kv_state_to_amortize_long_context", "rel": "covers"}, + {"source": "paper:mistral", "target": "move:cache_kv_state_to_amortize_long_context", "rel": "covers"}, + {"source": "paper:mistral", "target": "move:speculative_decoding_with_draft_model", "rel": "covers"}, + {"source": "paper:gemini", "target": "move:speculative_decoding_with_draft_model", "rel": "covers"}, + + {"source": "paper:llava", "target": "move:freeze_visual_encoder_and_only_train_connector", "rel": "covers"}, + {"source": "paper:flamingo", "target": "move:freeze_visual_encoder_and_only_train_connector", "rel": "covers"}, + {"source": "paper:cambrian", "target": "move:freeze_visual_encoder_and_only_train_connector", "rel": "covers"}, + + {"source": "paper:octo", "target": "move:use_diffusion_head_for_continuous_action", "rel": "covers"}, + {"source": "paper:diffuser", "target": "move:use_diffusion_head_for_continuous_action", "rel": "covers"}, + + {"source": "paper:gpt_driver", "target": "move:treat_planning_as_autoregressive_trajectory_generation", "rel": "covers"}, + {"source": "paper:emma", "target": "move:treat_planning_as_autoregressive_trajectory_generation", "rel": "covers"}, + {"source": "paper:vadv2", "target": "move:treat_planning_as_autoregressive_trajectory_generation", "rel": "parallel"}, + + {"source": "paper:2512.24426", "target": "move:use_world_model_rollout_as_critic_for_policy", "rel": "covers"}, + {"source": "paper:world_models", "target": "move:use_world_model_rollout_as_critic_for_policy", "rel": "covers"}, + {"source": "paper:cosmos", "target": "move:use_world_model_rollout_as_critic_for_policy", "rel": "covers"}, + + {"source": "paper:voyager", "target": "move:long_horizon_via_hierarchical_subgoal", "rel": "covers"}, + {"source": "paper:swiftsage", "target": "move:long_horizon_via_hierarchical_subgoal", "rel": "covers"}, + {"source": "paper:2311.10813", "target": "move:long_horizon_via_hierarchical_subgoal", "rel": "covers"}, + + {"source": "paper:debate", "target": "move:prompt_chain_with_explicit_persona_roles", "rel": "covers"}, + {"source": "paper:swiftsage", "target": "move:prompt_chain_with_explicit_persona_roles", "rel": "covers"}, + {"source": "paper:2311.10813", "target": "move:prompt_chain_with_explicit_persona_roles", "rel": "covers"}, + + {"source": "paper:2512.24426", "target": "move:contrast_corner_case_against_normal_case_in_training", "rel": "covers"}, + {"source": "paper:cosmos", "target": "move:contrast_corner_case_against_normal_case_in_training", "rel": "covers"}, + + {"source": "paper:ad_benchmarks", "target": "move:evaluate_open_loop_then_close_loop_for_realism", "rel": "covers"}, + {"source": "paper:drivemlm", "target": "move:evaluate_open_loop_then_close_loop_for_realism", "rel": "covers"}, + {"source": "paper:lmdrive", "target": "move:evaluate_open_loop_then_close_loop_for_realism", "rel": "covers"}, + + {"source": "paper:drivelm", "target": "move:use_language_explanation_as_auxiliary_supervision", "rel": "covers"}, + {"source": "paper:lingo2", "target": "move:use_language_explanation_as_auxiliary_supervision", "rel": "covers"}, + {"source": "paper:emma", "target": "move:use_language_explanation_as_auxiliary_supervision", "rel": "covers"}, + + {"source": "problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop", "target": "paper:2512.24426", "rel": "addresses"}, + {"source": "problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop", "target": "paper:senna", "rel": "addresses"}, + {"source": "problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop", "target": "move:add_reflection_step_so_agent_critiques_its_own_output", "rel": "addresses"}, + {"source": "problem:grounding_language_token_to_continuous_physical_world", "target": "paper:rt2", "rel": "addresses"}, + {"source": "problem:grounding_language_token_to_continuous_physical_world", "target": "paper:palme", "rel": "addresses"}, + {"source": "problem:grounding_language_token_to_continuous_physical_world", "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", "rel": "addresses"}, + {"source": "problem:latency_budget_for_large_model_in_realtime_control", "target": "paper:2402.12289", "rel": "addresses"}, + {"source": "problem:latency_budget_for_large_model_in_realtime_control", "target": "move:distill_large_model_into_specialist_for_deployment", "rel": "addresses"}, + {"source": "problem:latency_budget_for_large_model_in_realtime_control", "target": "move:speculative_decoding_with_draft_model", "rel": "addresses"}, + {"source": "problem:latency_budget_for_large_model_in_realtime_control", "target": "move:cache_kv_state_to_amortize_long_context", "rel": "addresses"}, + {"source": "problem:long_horizon_reasoning_with_finite_context_window", "target": "paper:gemini", "rel": "addresses"}, + {"source": "problem:long_horizon_reasoning_with_finite_context_window", "target": "move:use_retrieval_augmented_memory_to_extend_context", "rel": "addresses"}, + {"source": "problem:long_horizon_reasoning_with_finite_context_window", "target": "move:long_horizon_via_hierarchical_subgoal", "rel": "addresses"}, + {"source": "problem:zero_shot_generalization_to_unseen_driving_scenes", "target": "paper:gpt4v", "rel": "addresses"}, + {"source": "problem:zero_shot_generalization_to_unseen_driving_scenes", "target": "paper:emma", "rel": "addresses"}, + {"source": "problem:fine_grained_spatial_understanding_in_vision_language_model", "target": "paper:cambrian", "rel": "addresses"}, + {"source": "problem:fine_grained_spatial_understanding_in_vision_language_model", "target": "paper:florence", "rel": "addresses"}, + {"source": "problem:counterfactual_reasoning_about_other_agents_intent", "target": "paper:2512.24426", "rel": "addresses"}, + {"source": "problem:counterfactual_reasoning_about_other_agents_intent", "target": "paper:gaia1", "rel": "addresses"}, + {"source": "problem:counterfactual_reasoning_about_other_agents_intent", "target": "move:use_world_model_rollout_as_critic_for_policy", "rel": "addresses"}, + {"source": "problem:open_world_corner_case_synthesis_for_training", "target": "paper:cosmos", "rel": "addresses"}, + {"source": "problem:open_world_corner_case_synthesis_for_training", "target": "paper:prism1", "rel": "addresses"}, + {"source": "problem:open_world_corner_case_synthesis_for_training", "target": "move:contrast_corner_case_against_normal_case_in_training", "rel": "addresses"}, + {"source": "problem:evaluation_gap_between_offline_benchmark_and_closed_loop", "target": "paper:drivemlm", "rel": "addresses"}, + {"source": "problem:evaluation_gap_between_offline_benchmark_and_closed_loop", "target": "paper:lmdrive", "rel": "addresses"}, + {"source": "problem:evaluation_gap_between_offline_benchmark_and_closed_loop", "target": "move:evaluate_open_loop_then_close_loop_for_realism", "rel": "addresses"}, + {"source": "problem:catastrophic_forgetting_after_action_finetuning", "target": "paper:rt2", "rel": "addresses"}, + {"source": "problem:catastrophic_forgetting_after_action_finetuning", "target": "paper:openvla", "rel": "addresses"}, + {"source": "problem:catastrophic_forgetting_after_action_finetuning", "target": "move:co_finetune_language_model_with_action_data_jointly", "rel": "addresses"}, + + {"source": "insight:language_is_compressed_world_model_for_human_priors", "target": "paper:gpt3", "rel": "covers"}, + {"source": "insight:language_is_compressed_world_model_for_human_priors", "target": "paper:2309.16292", "rel": "covers"}, + {"source": "insight:language_is_compressed_world_model_for_human_priors", "target": "paper:palme", "rel": "covers"}, + {"source": "insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models", "target": "paper:gpt4", "rel": "covers"}, + {"source": "insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models", "target": "essay:bitter_lesson", "rel": "covers"}, + {"source": "insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models", "target": "move:scale_data_then_let_emergent_capabilities_appear", "rel": "covers"}, + {"source": "insight:world_model_video_diffusion_is_implicit_physics_engine", "target": "paper:sora", "rel": "covers"}, + {"source": "insight:world_model_video_diffusion_is_implicit_physics_engine", "target": "paper:gaia1", "rel": "covers"}, + {"source": "insight:world_model_video_diffusion_is_implicit_physics_engine", "target": "paper:cosmos", "rel": "covers"}, + {"source": "insight:agent_loop_is_just_iterated_conditional_generation", "target": "paper:react", "rel": "covers"}, + {"source": "insight:agent_loop_is_just_iterated_conditional_generation", "target": "paper:reflexion", "rel": "covers"}, + {"source": "insight:agent_loop_is_just_iterated_conditional_generation", "target": "paper:2311.10813", "rel": "covers"}, + {"source": "insight:tool_use_extends_language_model_into_environment_grounded_actor", "target": "paper:toolformer", "rel": "covers"}, + {"source": "insight:tool_use_extends_language_model_into_environment_grounded_actor", "target": "paper:react", "rel": "covers"}, + {"source": "insight:tool_use_extends_language_model_into_environment_grounded_actor", "target": "paper:2311.10813", "rel": "covers"}, + {"source": "insight:counterfactual_replanning_separates_intent_from_execution", "target": "paper:2512.24426", "rel": "covers"}, + {"source": "insight:counterfactual_replanning_separates_intent_from_execution", "target": "move:use_world_model_rollout_as_critic_for_policy", "rel": "covers"}, + {"source": "insight:foundation_model_decouples_perception_from_task_specific_training", "target": "paper:dinov2", "rel": "covers"}, + {"source": "insight:foundation_model_decouples_perception_from_task_specific_training", "target": "paper:sam", "rel": "covers"}, + {"source": "insight:foundation_model_decouples_perception_from_task_specific_training", "target": "paper:florence", "rel": "covers"}, + {"source": "insight:dual_system_fast_slow_loop_marries_reactive_and_deliberative_control", "target": "paper:swiftsage", "rel": "covers"}, + {"source": "insight:dual_system_fast_slow_loop_marries_reactive_and_deliberative_control", "target": "paper:2402.12289", "rel": "covers"}, + {"source": "insight:emergent_planning_from_next_token_prediction_alone", "target": "paper:gpt4", "rel": "covers"}, + {"source": "insight:emergent_planning_from_next_token_prediction_alone", "target": "paper:cot_wei2022", "rel": "covers"}, + {"source": "insight:alignment_is_constraint_satisfaction_over_generation", "target": "paper:instructgpt", "rel": "covers"}, + {"source": "insight:alignment_is_constraint_satisfaction_over_generation", "target": "paper:rlhf_dpo", "rel": "covers"}, + {"source": "insight:alignment_is_constraint_satisfaction_over_generation", "target": "paper:constitutional_ai", "rel": "covers"}, + {"source": "insight:open_weight_release_compounds_research_velocity", "target": "paper:llama", "rel": "covers"}, + {"source": "insight:open_weight_release_compounds_research_velocity", "target": "paper:openvla", "rel": "covers"}, + {"source": "insight:open_weight_release_compounds_research_velocity", "target": "paper:qwen", "rel": "covers"}, + {"source": "insight:open_weight_release_compounds_research_velocity", "target": "paper:mistral", "rel": "covers"}, + + {"source": "paradigm:foundation_model_axis", "target": "paper:gpt4", "rel": "covers"}, + {"source": "paradigm:foundation_model_axis", "target": "paper:llama", "rel": "covers"}, + {"source": "paradigm:foundation_model_axis", "target": "paper:llava", "rel": "covers"}, + {"source": "paradigm:foundation_model_axis", "target": "paradigm:vla_paradigm", "rel": "parallel"}, + {"source": "paradigm:foundation_model_axis", "target": "paradigm:world_model_paradigm", "rel": "parallel"}, + {"source": "paradigm:foundation_model_axis", "target": "paradigm:llm_agent_paradigm", "rel": "parallel"}, + {"source": "paradigm:world_model_paradigm", "target": "paper:world_models", "rel": "covers"}, + {"source": "paradigm:world_model_paradigm", "target": "paper:gaia1", "rel": "covers"}, + {"source": "paradigm:world_model_paradigm", "target": "paper:cosmos", "rel": "covers"}, + {"source": "paradigm:world_model_paradigm", "target": "paper:sora", "rel": "covers"}, + {"source": "paradigm:llm_agent_paradigm", "target": "paper:react", "rel": "covers"}, + {"source": "paradigm:llm_agent_paradigm", "target": "paper:2311.10813", "rel": "covers"}, + {"source": "paradigm:llm_agent_paradigm", "target": "paper:2309.16292", "rel": "covers"}, + {"source": "paradigm:llm_agent_paradigm", "target": "paper:voyager", "rel": "covers"}, + {"source": "paradigm:vla_paradigm", "target": "paper:rt2", "rel": "covers"}, + {"source": "paradigm:vla_paradigm", "target": "paper:openvla", "rel": "covers"}, + {"source": "paradigm:vla_paradigm", "target": "paper:emma", "rel": "covers"}, + {"source": "paradigm:vla_paradigm", "target": "paper:2512.24426", "rel": "covers"}, + {"source": "paradigm:vla_paradigm", "target": "paper:2402.12289", "rel": "covers"} + ] +} diff --git a/docs/data/generated/insights_and_validations.json b/docs/data/generated/insights_and_validations.json new file mode 100644 index 0000000..cca57c7 --- /dev/null +++ b/docs/data/generated/insights_and_validations.json @@ -0,0 +1,486 @@ +{ + "$comment": "Cross-disciplinary insight paths, validation case trace nodes, and paradigm meta-nodes. Generated to let researchers re-derive seminal works by following their building blocks (concepts + moves + insights).", + "nodes": [ + + {"id": "move:residual_connection", "label_en": "Residual Connection", "label_zh": "残差连接(恒等捷径)", "kind": "move", "tier": "move", "topic": "math_foundations", "phase": "prereq", "year": 2015, "summary_zh": "残差连接是一种方法学原语,它让网络的每一层学习相对于输入的偏差而非完整映射,从而把恒等函数作为优化的默认起点。这一移动将深度网络的可训练性问题转化为残差函数的学习问题,使梯度可以沿恒等通路直达浅层。它最早在 ResNet 中被系统化,随后被 Transformer、扩散模型、Diffusion Policy 和残差策略学习反复借用。在自动驾驶中,残差策略可以叠加在规则规划器之上,从而在保留可解释安全行为的同时让数据驱动模块进行局部修正。", "building_blocks": []}, + + {"id": "move:patchify_tokenization", "label_en": "Patch / Token Reshaping", "label_zh": "Patch 分块与 token 化", "kind": "move", "tier": "move", "topic": "math_foundations", "phase": "prereq", "year": 2020, "summary_zh": "Patch 化是将连续高维信号(图像、点云、视频、语音)切分为离散 token 序列的方法学动作,目的是让 transformer 这类与序列长度近似线性相关的架构能够直接处理非语言模态。这一移动在 ViT 中第一次把图像视为 16×16 的 patch 序列,在 VideoMAE 中扩展到时空管道,在驾驶感知中扩展到 BEV 网格 token 与点云体素 token。它揭示了一个普适规律,即模态之间的差异往往可以通过 tokenizer 的设计而非主干结构的差异来消化。", "building_blocks": []}, + + {"id": "move:masking_for_pretext", "label_en": "Masked Prediction Pretext", "label_zh": "掩码预测自监督任务", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "core", "year": 2018, "summary_zh": "掩码并预测是一类基础的自监督移动,其核心是把输入的一部分隐藏起来并要求模型从可见部分重构被掩盖的部分,从而免费获得无穷多的监督信号。BERT 把它用于文本片段,MAE 把它用于图像块,VideoMAE 把它用于时空体,占据栅格预训练把它用于驾驶 BEV 占据。在自动驾驶中,将未来帧的占据栅格或邻车轨迹掩码起来再预测,可以作为大规模无标签预训练任务来缓解长尾驾驶事件的标注稀缺。", "building_blocks": []}, + + {"id": "move:contrastive_alignment", "label_en": "Contrastive Alignment", "label_zh": "对比对齐", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "core", "year": 2020, "summary_zh": "对比对齐通过把成对的样本在嵌入空间拉近、把非成对样本推远来学习一个共享表征,这一移动在 SimCLR 中用于图像增广对,在 CLIP 中用于图文配对,在 Audio-CLIP 中扩展到音频。其本质是用相对的相似性结构而非绝对标签来定义任务,从而能够利用互联网规模的天然配对数据。在自动驾驶中,将驾驶日志片段与人类自然语言解释做对比对齐,可以使模型实现零样本场景检索和自然语言条件下的轨迹生成。", "building_blocks": []}, + + {"id": "move:cross_attention_query", "label_en": "Typed Cross-Attention Query", "label_zh": "类型化 cross-attention query", "kind": "move", "tier": "move", "topic": "math_foundations", "phase": "core", "year": 2020, "summary_zh": "把任何关心的实体表达为一组可学习的 query 向量,让它们通过 cross-attention 从一个公共特征记忆中拉取信息,是一类极其普适的方法学移动。DETR 把它用于目标检测的对象 query,BEVFormer 把它用于鸟瞰图网格 query,UniAD 进一步把它用于代理 query 与地图 query。在自动驾驶中,每一种司机关心的对象都可以被参数化为 query,并通过 cross-attention 实现感知、预测、规划在共享潜空间中的对话。", "building_blocks": []}, + + {"id": "move:lift_2d_to_3d", "label_en": "Lift-Splat to BEV", "label_zh": "2D 升至 3D 的 lift-splat", "kind": "move", "tier": "move", "topic": "e2e_ad", "phase": "prereq", "year": 2020, "summary_zh": "Lift-Splat 是一类把二维图像特征通过深度概率或可学习投影抬升到三维空间、再投影到鸟瞰图的方法学移动。该移动建立了一个跨视角共享的几何一致表征,使得多相机感知、占据预测与运动规划可以在同一坐标系中协作。它最早在 Lift-Splat-Shoot 中明确提出,在 BEVDet、BEVFormer 中演化出深度监督与时空 query 变体,并成为现代端到端自动驾驶的事实标准前端。", "building_blocks": []}, + + {"id": "move:set_prediction_with_hungarian", "label_en": "Set Prediction with Hungarian Matching", "label_zh": "匈牙利匹配下的集合预测", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "core", "year": 2020, "summary_zh": "集合预测把检测、跟踪、规划等任务统一为输出一个无序集合,并通过匈牙利算法在预测与真值之间做最优二分匹配以计算损失,从而消除了对手工设计的非极大值抑制和 anchor 的依赖。DETR 把它用于目标检测,DETR3D 把它用于三维检测,PlanT 与 UniAD 把它用于代理级的规划预测。这一移动在自动驾驶中提供了端到端可微的输出层范式,使后续模块可以无歧义地拼接到感知输出之后。", "building_blocks": []}, + + {"id": "move:diffusion_denoise_sampling", "label_en": "Score-based Denoising Sampling", "label_zh": "基于得分的去噪采样", "kind": "move", "tier": "move", "topic": "math_foundations", "phase": "core", "year": 2020, "summary_zh": "扩散方法把生成问题转化为反向去噪过程,模型只需学习预测每一步的噪声或得分函数,并在推断时迭代采样得到样本。这一移动在图像生成中由 DDPM 奠基,在视频中扩展为视频扩散,在控制领域被 Diffusion Policy 重新解释为以条件为状态、以动作为样本的策略学习。在自动驾驶中,扩散过程可以同时生成多模态轨迹候选和反事实场景,统一了生成与决策两个传统上分离的问题。", "building_blocks": []}, + + {"id": "move:dual_system_fast_slow", "label_en": "Dual System Fast-Slow Decomposition", "label_zh": "快慢双系统分解", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "快慢双系统是一种受 Kahneman 双过程理论启发的方法学分解,让一个低延迟的小模型负责常规情况下的实时响应,让一个高延迟的大模型在异常或复杂情境下提供慢速推理。AlphaGo 用快速 policy 加 MCTS 体现了它,OpenAI o1 用思考链体现了它,DriveVLM-Dual 用快规划加 VLM 慢系统体现了它。在自动驾驶中这一移动直接回应了延迟和质量之间的硬性权衡,是落地系统中最实用的范式之一。", "building_blocks": []}, + + {"id": "move:tool_use_function_calling", "label_en": "Tool Use / Function Calling", "label_zh": "工具调用与函数调用", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "core", "year": 2023, "summary_zh": "工具调用让语言模型把外部 API、地图查询、轨迹优化器、规则检查器视为可调用函数,从而把符号系统的精确性嫁接到神经语言模型的灵活推理之上。Toolformer 演示了从无监督语料学习工具触发,ReAct 引入了思考与行动交替的循环,Agent-Driver 则把它落到驾驶决策栈中。在自动驾驶里这一移动使 LLM 不需要内化所有数值能力,而可以通过调用专家组件解决精确控制和约束验证问题。", "building_blocks": []}, + + {"id": "move:counterfactual_replan", "label_en": "Counterfactual Replan", "label_zh": "反事实重规划", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "frontier", "year": 2025, "summary_zh": "反事实重规划让系统主动构造若干并未发生的对照场景并比较其后果,从而把决策从被动响应升级为对潜在后果的主动评估。它在因果推断、强化学习中的 model-based rollout、对抗训练中都有体现,在 CF-VLA 中被显式实现为对替代轨迹的 VLM 评分。在自动驾驶中这一移动让系统能够回答如果我变道而非保持车道会发生什么这类问题,是迈向人类级因果驾驶推理的关键路径。", "building_blocks": []}, + + {"id": "move:tokenize_modalities", "label_en": "Universal Modality Tokenization", "label_zh": "模态统一 tokenize", "kind": "move", "tier": "move", "topic": "vlm_vla", "phase": "core", "year": 2022, "summary_zh": "模态统一 tokenize 是一种深刻的简化:与其为每种模态设计独立的编码器,不如把图像、语音、动作、点云都映射到同一个离散或连续 token 空间,然后让一个统一的序列模型处理它们。Gato 在多模态控制中实现了它,RT-1、RT-2 在机器人动作中扩展了它,Wayve LINGO-2 在驾驶中把传感、语言、动作放进同一个序列。在自动驾驶中这一移动允许同一个基础模型在驾驶日志、人类解说、控制信号上联合训练。", "building_blocks": []}, + + {"id": "move:replay_and_target_net", "label_en": "Replay Buffer with Target Network", "label_zh": "经验回放与目标网络", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2015, "summary_zh": "经验回放与目标网络是稳定值函数学习的两个方法学原语,前者通过从大池子均匀采样打破数据的时序相关性,后者通过周期同步的复制网络打破贝尔曼自举的反馈循环。DQN 同时引入它们使深度强化学习首次稳定收敛,DDPG、TD3、Rainbow 等继承并扩展了这一移动。在自动驾驶中这一移动是任何基于值函数的离线强化学习方法的基础前置条件。", "building_blocks": []}, + + {"id": "move:dataset_aggregation", "label_en": "Iterative Dataset Aggregation", "label_zh": "迭代数据集聚合", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2011, "summary_zh": "迭代数据集聚合让学习到的策略与环境交互、由专家在新访问到的状态上重新标注、再把这些状态与标签加入训练集进行下一轮训练。这一移动直接对应于模仿学习中协变量偏移的成因并提供了理论上有界的解决方案,是 DAgger 的核心。在自动驾驶中这一移动启发了影子模式数据收集、规划失败案例自动挖掘等数据飞轮设计。", "building_blocks": []}, + + {"id": "move:clipped_surrogate_objective", "label_en": "Clipped Surrogate Objective", "label_zh": "剪裁替代目标", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "剪裁替代目标用一个对策略比率施加上下界限制的目标函数来近似信赖域约束,从而在保留高效一阶优化的同时避免单步策略更新过大。这是 PPO 的方法学心脏,被广泛复用于 RLHF、DPO 的早期变体和很多机器人在线策略优化。在自动驾驶相关的训练后期,这一移动是把模仿学习初始化的策略安全地推向更高奖励的标准工具。", "building_blocks": []}, + + {"id": "move:self_play_with_search", "label_en": "Self-Play with Tree Search", "label_zh": "自对弈与蒙特卡洛树搜索", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "自对弈让一个策略对抗自身的副本以生成无穷数据,蒙特卡洛树搜索利用学习到的价值与策略进行未来推演,二者结合形成正反馈式的能力提升。AlphaZero 在棋类中实现了它,MuZero 把它推广到无模型环境,最近的 Multi-Agent Driving Self-Play 把它探索性地引入交通博弈。在自动驾驶中这一移动是大规模仿真器中多智能体策略学习的核心范式。", "building_blocks": []}, + + {"id": "move:latent_imagination_rollout", "label_en": "Latent Imagination Rollout", "label_zh": "潜空间想象 rollout", "kind": "move", "tier": "move", "topic": "deep_rl", "phase": "frontier", "year": 2018, "summary_zh": "潜空间想象让策略和价值在一个学到的紧凑潜世界模型中进行多步推演,而非在像素或原始传感空间中代价高昂地展开。Ha 与 Schmidhuber 的 World Models 首次系统化它,Dreamer 系列把它做到大规模可训练,GAIA-1 与 DriveDreamer 把它扩展到驾驶视频。在自动驾驶中这一移动是稀缺真实事故数据条件下大规模 model-based 强化学习的关键。", "building_blocks": []}, + + {"id": "move:spike_event_compute", "label_en": "Spike / Event-driven Compute", "label_zh": "脉冲与事件驱动计算", "kind": "move", "tier": "move", "topic": "brain_inspired", "phase": "frontier", "year": 1997, "summary_zh": "脉冲与事件驱动计算让神经元只有在阈值被跨越时才发放离散事件,从而把信息表示为稀疏时序脉冲流,大幅降低能耗和延迟。脉冲神经网络在神经形态硬件、事件相机视觉、Spike-driven Transformer 中反复出现。在自动驾驶中这一移动直接对应车载端能耗、延迟与冗余度的硬性约束,是边缘端低功耗感知与决策的方向之一。", "building_blocks": []}, + + + {"id": "insight:residual_learning_unlocks_arbitrary_depth", "label_en": "Residual Learning Unlocks Arbitrary Depth", "label_zh": "残差学习解锁任意深度", "kind": "insight", "tier": "insight", "topic": "math_foundations", "phase": "prereq", "year": 2015, "summary_zh": "残差学习是一条跨领域的普适洞见,其抽象内核是把任意复杂的映射改写为恒等加上一个小的偏差并训练这个偏差。这一洞见在视觉中由 ResNet 首先打通了百层网络的训练,在 Transformer 中以残差子层与层归一化的组合再次出现,在扩散模型中以预测噪声残差的形式作为生成的方法学基石,在强化学习中以残差策略叠加在规则控制器之上而获得安全可控的微调。在自动驾驶研究中这意味着规则规划器并不需要被推翻,而是可以作为残差策略的恒等基线,由数据驱动模块只学习相对修正,从而把传统工程与神经端到端融合。", "building_blocks": ["move:residual_connection", "paper:he2015_resnet"]}, + + {"id": "insight:masked_prediction_yields_self_supervised_signal", "label_en": "Masked Prediction Yields Self-Supervised Signal", "label_zh": "掩码预测提取自监督信号", "kind": "insight", "tier": "insight", "topic": "ssl_vision", "phase": "core", "year": 2018, "summary_zh": "掩码并预测是一条贯穿语言、视觉、视频、机器人等模态的洞见,其内核是任何含有冗余结构的数据都可以通过遮蔽一部分并让模型重构被遮蔽部分而提供任务无关的监督。BERT 把它带入语言并奠定了预训练后微调的范式,MAE 把它带入图像并显著降低了视觉预训练的标签成本,VideoMAE 与 OmniMAE 把它扩展到时空,占据栅格掩码预训练则把它引入自动驾驶。对于自动驾驶研究的含义是,可以将下一秒 BEV 占据或邻车轨迹作为掩码目标进行大规模无标签预训练,从而把街道上海量的未标注行驶数据转化为通用驾驶表征。", "building_blocks": ["move:masking_for_pretext", "concept:ssl"]}, + + {"id": "insight:attention_is_typed_entity_communication", "label_en": "Attention is Communication Between Typed Entities", "label_zh": "注意力是类型化实体之间的通信", "kind": "insight", "tier": "insight", "topic": "math_foundations", "phase": "core", "year": 2017, "summary_zh": "注意力机制可以被理解为一群类型化的实体在共享潜空间中互相发送和接收消息的协议,每个 query 是一个有指定意图的探针,每个 key-value 对是一个可被检索的事实。DETR 把检测对象编码为 query,BEVFormer 把鸟瞰网格点编码为 query,UniAD 把代理、地图元素与运动模式全部编码为 query 并让它们在同一注意力栈中对话。在自动驾驶研究的含义是,任何被司机关心的实体(路口、信号灯、邻车意图、占据网格、自车未来)都可以被声明为 query 并接入一个统一的注意力工厂,从而把感知、预测、规划写成同一种结构。", "building_blocks": ["move:cross_attention_query", "concept:self_attention", "concept:detr_query"]}, + + {"id": "insight:contrastive_alignment_creates_zero_shot_transfer", "label_en": "Contrastive Alignment Creates Zero-shot Transfer", "label_zh": "对比对齐造就零样本迁移", "kind": "insight", "tier": "insight", "topic": "ssl_vision", "phase": "core", "year": 2021, "summary_zh": "对比对齐告诉我们任何两种自然成对出现的模态都可以通过对齐它们的嵌入空间而获得相互检索和零样本分类能力。CLIP 在图文上展示了这一点,ALIGN 用更脏的网络数据进一步扩大规模,CLAP 与 AudioCLIP 把它推广到音频文本,DriveCLIP 与场景文本对齐工作把它带入驾驶日志检索。对于自动驾驶研究的含义是,可以把驾驶视频与人类驾驶员的解说、事故报告、城市规则文本进行对比对齐,从而获得零样本的场景检索能力和自然语言条件下的轨迹生成基础。", "building_blocks": ["move:contrastive_alignment", "concept:vlm"]}, + + {"id": "insight:diffusion_unifies_generation_and_decision", "label_en": "Diffusion Unifies Generation and Decision", "label_zh": "扩散统一生成与决策", "kind": "insight", "tier": "insight", "topic": "math_foundations", "phase": "core", "year": 2022, "summary_zh": "扩散与得分匹配揭示了一条把生成、推断、控制统一起来的洞见,其内核是任何复杂分布的样本都可以通过反向去噪过程逐步生成,而条件信息可以作为得分网络的额外输入控制采样。它在图像生成中由 DDPM 与 Stable Diffusion 推广,在视频中通过 Video Diffusion 实现,在控制中由 Diffusion Policy、Decision Diffuser 把动作序列视为待去噪样本,在驾驶中由轨迹扩散与场景扩散世界模型同时使用。对于自动驾驶研究的含义是,多模态轨迹候选与反事实场景可以从同一个扩散过程中以不同条件采样得到,从而消除生成模块和规划模块之间的人为边界。", "building_blocks": ["move:diffusion_denoise_sampling", "paper:diffuser"]}, + + {"id": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", "label_en": "End-to-End Differentiable Beats Handcraft When Signal is Strong", "label_zh": "信号足够强时端到端可微胜过手工中间表示", "kind": "insight", "tier": "insight", "topic": "e2e_ad", "phase": "core", "year": 2017, "summary_zh": "端到端可微优化在监督信号足够强、数据足够多时往往优于由人类手工设计的中间接口,因为后者人为限制了表征空间。Listen-Attend-Spell 在语音识别中消化了声学加发音加语言模型的级联,seq2seq 在机器翻译中消化了短语对齐与句法分析的级联,UniAD 在驾驶中消化了感知到预测到规划的级联。对于自动驾驶研究的含义是,当具备完整的端到端反馈(仿真器奖励、真实日志的人类轨迹、安全度量)时,应当主动溶解模块边界、让梯度自由流动;同时需要清醒地承认在反馈稀疏或安全敏感的子任务上模块化仍是稳健的折衷。", "building_blocks": ["paper:2212.10156", "essay:bitter_lesson"]}, + + {"id": "insight:dual_system_handles_latency_quality_tradeoff", "label_en": "Dual System Handles the Latency-Quality Tradeoff", "label_zh": "双系统化解延迟与质量权衡", "kind": "insight", "tier": "insight", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "Kahneman 的快慢系统给出了一条跨学科洞见,复杂决策系统应当用一条低延迟廉价的反应通路覆盖大多数常规情境,并用一条高延迟昂贵的反思通路在异常或风险情境下提供慢速推理。AlphaGo 用快速 policy 加 MCTS 实现了它,OpenAI o1 用 chain-of-thought 思考时间换质量,DriveVLM-Dual 用快规划加 VLM 慢系统融合实现了它。对于自动驾驶研究的含义是,应当在生产堆栈中显式分离 30 Hz 控制循环和 1 Hz 反思循环,并把后者用于罕见且高代价的决策点,而非试图用单一大模型既快又好。", "building_blocks": ["move:dual_system_fast_slow", "paper:2402.12289"]}, + + {"id": "insight:symbolic_intermediate_enables_interpretability_and_transfer", "label_en": "Symbolic Intermediate Enables Interpretability and Transfer", "label_zh": "符号中间表示提供可解释性与可迁移性", "kind": "insight", "tier": "insight", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "在两个神经组件之间插入一个人类可读的符号中间表示往往同时提升可解释性和跨任务迁移能力,因为符号既约束了表达空间又对人类审计开放。Code-as-Policies 用 Python 代码作为机器人计划的中间表示,LMDrive 与 DriveLM 用自然语言作为感知与规划之间的接口,PROGPROMPT 用结构化提示驱动家庭机器人。对于自动驾驶研究的含义是,把语言或结构化场景描述显式插入感知到规划之间,可以在解释失败案例、跨车型迁移、安全审计上带来巨大杠杆,而代价仅仅是中间一次额外的模型推理。", "building_blocks": ["concept:cot", "paper:2309.16292"]}, + + {"id": "insight:long_tail_solved_by_synthesis_not_data_alone", "label_en": "Long Tail Solved by Synthesis Rather than Data Alone", "label_zh": "长尾问题靠合成而非单纯增加数据解决", "kind": "insight", "tier": "insight", "topic": "e2e_ad", "phase": "frontier", "year": 2023, "summary_zh": "长尾问题的解决依赖主动合成而非被动收集,因为现实世界中真正稀有的情境出现频率太低使得纯数据扩张的边际收益递减。机器人控制中通过域随机化合成了千万级仿真轨迹,文本到 3D 中通过扩散世界模型合成了不存在的物体,自动驾驶中通过场景生成器和反事实重写合成了边角案例。对于自动驾驶研究的含义是,每多收集一千小时真实数据可能不如训练一个高质量的反事实场景生成器更划算,反事实重规划与合成驱动的安全验证应当成为核心工具链而非辅助工具。", "building_blocks": ["move:counterfactual_replan", "paper:2512.24426", "paper:drivedreamer"]}, + + {"id": "insight:scaling_laws_predict_capability_emergence", "label_en": "Scaling Laws Predict Capability Emergence", "label_zh": "Scaling 定律预测能力涌现", "kind": "insight", "tier": "insight", "topic": "meta_philosophy", "phase": "core", "year": 2020, "summary_zh": "Kaplan 等人的 scaling 定律显示模型损失对参数量、数据量、计算量呈幂律下降,并允许研究者在做小实验后预测大规模训练的能力。它在语言模型中由 GPT-3 验证、在视觉中由 DINOv2 验证、在多模态中由 Flamingo 与 Gemini 验证。对于自动驾驶研究的含义是,应当在每一项关键架构决策之前先做小规模 scaling 扫描以验证其外推性,而非在小模型上得到的局部最优结论上做大规模工程投入。", "building_blocks": ["essay:bitter_lesson", "concept:scaling_vs_knowledge", "paper:gpt3"]}, + + {"id": "insight:foundation_pretraining_decouples_data_from_task", "label_en": "Foundation Pretraining Decouples Data from Task", "label_zh": "基础模型预训练把数据与任务解耦", "kind": "insight", "tier": "insight", "topic": "ssl_vision", "phase": "core", "year": 2021, "summary_zh": "基础模型范式揭示了把任务无关的大规模预训练与任务相关的小规模微调彻底分离的可行性,由此使下游任务的边际数据需求大幅下降。BERT 与 GPT 在语言上演示了它,CLIP 与 DINOv2、DINOv3 在视觉上演示了它,SAM 在分割上演示了它。对于自动驾驶研究的含义是,应当首先投入资源训练驾驶通用主干(视觉、占据、轨迹)而非每个任务从零训练,并通过适配器和提示工程在下游低成本部署。", "building_blocks": ["paper:dinov2", "paper:2508.10104", "concept:ssl"]}, + + {"id": "insight:test_time_compute_substitutes_train_time_via_search", "label_en": "Test-Time Compute Can Substitute Train-Time via Search", "label_zh": "测试时计算可经搜索替代训练时计算", "kind": "insight", "tier": "insight", "topic": "deep_rl", "phase": "frontier", "year": 2017, "summary_zh": "测试时计算与训练时计算之间存在可替换关系,通过在推断时引入搜索、采样、自洽性投票等开销可以补偿训练阶段的能力不足。AlphaGo 用 MCTS 在弱策略上做搜索拿到超人类表现,OpenAI o1 用更长思维链交换更高准确率,AlphaCode 用大规模采样加过滤换取代码竞赛能力。对于自动驾驶研究的含义是,在生产端可以将昂贵车队训练与可控的车载推断搜索(轨迹采样加约束检查、VLM 反思)结合起来,从而以可承担的车载预算获得更高的决策质量。", "building_blocks": ["move:self_play_with_search", "paper:silver2017_alphazero"]}, + + {"id": "insight:imitation_data_compresses_unspecified_reward", "label_en": "Imitation Data Compresses an Unspecified Reward", "label_zh": "模仿数据压缩了未明示的奖励函数", "kind": "insight", "tier": "insight", "topic": "deep_rl", "phase": "core", "year": 2011, "summary_zh": "模仿学习的洞见是专家演示隐式编码了一个研究者难以手工指定的奖励函数,从而避开了奖励设计的难题。它在 ALVINN 中首次用于驾驶,在 GAIL、AIRL 中通过对抗逆强化学习显式提取了奖励,在 DriveGPT 系列中被推到大规模驾驶日志预训练。对于自动驾驶研究的含义是,与其试图手工写出涵盖舒适、安全、效率、社会礼仪的复合奖励,不如先用大规模驾驶日志做模仿预训练,再用偏好对齐与少量精细奖励做微调。", "building_blocks": ["concept:imitation_learning", "paper:ross2011_dagger"]}, + + {"id": "insight:world_models_let_planning_be_done_in_imagination", "label_en": "World Models Let Planning Be Done in Imagination", "label_zh": "世界模型让规划在想象中进行", "kind": "insight", "tier": "insight", "topic": "deep_rl", "phase": "core", "year": 2018, "summary_zh": "世界模型把环境动力学压缩进一个学到的潜空间模型,使策略和价值可以在该潜空间里以极低代价进行多步推演而非每次都与昂贵或危险的真实环境交互。Ha 与 Schmidhuber 第一次系统化了它,Dreamer 系列把它扩展到 Atari 与机器人,GAIA-1 与 DriveDreamer 把它扩展到驾驶视频。对于自动驾驶研究的含义是,未来的安全验证、罕见事故训练、反事实问答都可以建立在驾驶世界模型之上,把今天昂贵的实车测试逐步替换为模型内想象。", "building_blocks": ["move:latent_imagination_rollout", "paper:world_models", "paper:gaia1"]}, + + {"id": "insight:tokenization_collapses_modality_gap", "label_en": "Tokenization Collapses the Modality Gap", "label_zh": "Tokenize 抹平模态差距", "kind": "insight", "tier": "insight", "topic": "vlm_vla", "phase": "frontier", "year": 2022, "summary_zh": "把每种模态都映射为一个共享的 token 序列后,处理它们的主干网络几乎可以是同一个,模态差异退化为 tokenizer 的设计差异。ViT 把图像 tokenize 为 patch,VQ-VAE 把语音和视频 tokenize 为离散码本,Gato 与 RT-2 把动作 tokenize 为符号,使同一个 transformer 既能聊天又能操作。对于自动驾驶研究的含义是,可以把视觉、点云、HD 地图、控制信号统一 tokenize 进同一序列,从而让基础模型在驾驶日志、人类讲解、控制信号上联合训练,向真正的多模态驾驶基础模型迈进。", "building_blocks": ["move:tokenize_modalities", "move:patchify_tokenization"]}, + + {"id": "insight:set_prediction_eliminates_postprocessing_heuristics", "label_en": "Set Prediction Eliminates Post-processing Heuristics", "label_zh": "集合预测消除后处理启发式", "kind": "insight", "tier": "insight", "topic": "ssl_vision", "phase": "core", "year": 2020, "summary_zh": "把多对象输出建模为集合并用匈牙利匹配计算损失,可以一次性消除诸如非极大值抑制、锚框设计、阈值调节等大量手工后处理。这一洞见在 DETR 中第一次系统化,在 DETR3D、Sparse R-CNN 中推广到三维与稀疏检测,在 PlanT 与 UniAD 中扩展为对智能体级别规划的集合预测。对于自动驾驶研究的含义是,每当下游任务可以被表述为一个无序的输出集合时,集合预测往往能用更少的代码、更直接的梯度信号取代级联的启发式后处理,使整个 stack 真正端到端可微。", "building_blocks": ["move:set_prediction_with_hungarian", "paper:carion2020"]}, + + {"id": "insight:in_context_learning_emerges_at_scale", "label_en": "In-Context Learning Emerges at Scale", "label_zh": "上下文学习在规模上涌现", "kind": "insight", "tier": "insight", "topic": "vlm_vla", "phase": "core", "year": 2020, "summary_zh": "大规模自回归语言模型展示了在不更新参数的前提下从提示中的少量例子中学习新任务的能力,这是参数规模与数据规模累积之后的涌现现象。GPT-3 第一次把它放到聚光灯下,Flamingo 把它扩展到图像文本交错,VLM 工具如 Gemini 与 Claude 已经在驾驶问答中演示了少样本场景理解。对于自动驾驶研究的含义是,许多边角案例可以通过精心设计的提示中包含少量类似情境的示例与解决方案而被解决,而不必为每一种长尾微调一个新模型。", "building_blocks": ["paper:gpt3", "concept:cot"]}, + + {"id": "insight:safety_constraints_via_lagrangian_dual", "label_en": "Safety Constraints via Lagrangian Duality", "label_zh": "安全约束借助拉格朗日对偶实现", "kind": "insight", "tier": "insight", "topic": "deep_rl", "phase": "core", "year": 2019, "summary_zh": "拉格朗日对偶为把硬安全约束嵌入到策略优化提供了一条原则化道路,做法是把约束乘以可学习的乘子并将其加入目标函数,乘子由约束违反程度驱动上升。Constrained Policy Optimization 把它和信赖域结合,RCPO 把它和 PPO 结合,自动驾驶的 Safe-DRL 工作把它用于限制碰撞概率和加速度。对于自动驾驶研究的含义是,可以用拉格朗日方法把碰撞率、加速度、横向加速度等硬指标作为约束接入端到端策略优化,从而让性能与安全在统一框架下被对齐。", "building_blocks": ["concept:policy_gradient", "concept:actor_critic"]}, + + {"id": "insight:event_sparse_compute_matches_energy_budget", "label_en": "Event-Sparse Compute Matches Edge Energy Budget", "label_zh": "事件稀疏计算匹配边缘能耗预算", "kind": "insight", "tier": "insight", "topic": "brain_inspired", "phase": "frontier", "year": 2023, "summary_zh": "事件驱动稀疏计算的核心洞见是大多数自然信号在时间或空间上都是高度稀疏的,因此只在事件发生时进行计算可以节省数量级的能耗。事件相机在视觉中演示了它,脉冲神经网络在感知中演示了它,Spike-driven Transformer 把它与注意力结构融合。对于自动驾驶研究的含义是,车端有严格能耗与延迟预算,事件稀疏感知与决策可以作为补充昂贵密集 GPU 推断的轻量备份,特别适用于高频低复杂度的旁路任务。", "building_blocks": ["move:spike_event_compute", "concept:spiking_nn", "paper:2307.01694"]}, + + + {"id": "validation:trace_unified_planning_oriented_e2e_driving", "label_en": "Trace: Unified Planning-Oriented E2E Driving", "label_zh": "再发现:UniAD(统一规划导向端到端驾驶)", "kind": "validation", "tier": "validation", "topic": "e2e_ad", "phase": "core", "year": 2022, "summary_zh": "如果 UniAD 不存在,要从零再发明它,研究者必须在图谱中具备以下完整的构件链。第一是 BEVFormer 提供的时空鸟瞰图特征作为统一坐标系,第二是 DETR 风格的对象 query 与集合预测作为可微输出接口,第三是把每一种子任务(检测、跟踪、地图、运动预测、占据预测、规划)都改写为 query 在共享 BEV 上 cross-attention 的统一语言。第四是端到端可微即在信号足够强时溶解模块边界的洞见,使最终规划损失能够把梯度回传到感知。第五是模仿学习作为大规模驾驶日志监督的载体。把这五条放在一起,研究者就能自然推导出 UniAD 这种以规划为目标、各感知模块通过 query 协作的统一架构。", "building_blocks": ["paper:li2022bevformer", "paper:carion2020", "concept:bev", "concept:detr_query", "move:cross_attention_query", "move:set_prediction_with_hungarian", "concept:imitation_learning", "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", "insight:attention_is_typed_entity_communication"]}, + + {"id": "validation:trace_object_level_planning_transformer", "label_en": "Trace: Object-Level Planning Transformer", "label_zh": "再发现:PlanT(对象级规划 transformer)", "kind": "validation", "tier": "validation", "topic": "e2e_ad", "phase": "core", "year": 2022, "summary_zh": "如果 PlanT 不存在,要重新发明它,研究者需要的构件是:第一,已经具备成熟的检测器输出对象级别的紧凑场景表示(位置、速度、类别);第二,Transformer 作为一种把变长无序对象集合编码并产生序列化输出的通用工具;第三,集合预测与匈牙利匹配作为损失结构以处理对象的无序性;第四,模仿学习把人类驾驶轨迹作为目标输出;第五,对端到端模仿在 CARLA 等闭环仿真中可验证的认识。把这些组合起来便可自然得到一个用 transformer 直接消费对象 token 并输出未来路点的对象级规划器,并理解为何它比像素级端到端更样本高效与可解释。", "building_blocks": ["paper:vaswani2017", "paper:carion2020", "concept:transformer", "move:cross_attention_query", "move:set_prediction_with_hungarian", "concept:imitation_learning", "paper:ad_benchmarks", "paper:transfuser"]}, + + {"id": "validation:trace_vision_language_action_dual_loop", "label_en": "Trace: Vision-Language-Action Dual Loop", "label_zh": "再发现:DriveVLM-Dual(VLA 双系统)", "kind": "validation", "tier": "validation", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "如果 DriveVLM 与其双系统变体不存在,要把它再发明出来,研究者需要具备的构件是:一个能将视觉与语言对齐并产生符号级输出的 VLM 主干(如 LLaVA);BEV 与代理 query 作为快规划器的几何输入;chain-of-thought 推理作为慢系统的核心能力;快慢双系统范式作为对延迟与质量权衡的方法学回答;以及一个把语言场景描述与轨迹候选互相校验的接口。把它们组合起来,研究者会自然提出一个 30 Hz 的快规划器加 1 Hz 的 VLM 反思回路在关键时刻提供高阶决策的双重架构。", "building_blocks": ["paper:llava", "concept:vlm", "concept:vla", "concept:cot", "move:dual_system_fast_slow", "move:cross_attention_query", "insight:dual_system_handles_latency_quality_tradeoff", "insight:symbolic_intermediate_enables_interpretability_and_transfer", "paper:li2022bevformer"]}, + + {"id": "validation:trace_llm_decision_agent_for_driving", "label_en": "Trace: LLM Decision Agent for Driving", "label_zh": "再发现:Agent-Driver(LLM 决策智能体)", "kind": "validation", "tier": "validation", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "如果 Agent-Driver 不存在,要从零再发明它,研究者需要的构件是:第一,GPT-3 量级的语言模型展示的少样本上下文学习能力;第二,工具调用范式让 LLM 能调用外部的检测器、地图查询、轨迹优化器以补偿其几何与控制的不足;第三,ReAct 形式的交替思考与行动循环;第四,对象级感知作为对环境的紧凑描述以馈入语言提示;第五,模仿学习专家轨迹作为对比基线以评估 LLM 的决策质量。组合起来即可推出一个把 LLM 作为高阶决策核心、把所有数值技能委托给外部工具的驾驶认知智能体。", "building_blocks": ["paper:gpt3", "concept:vlm", "concept:cot", "concept:tool_use", "move:tool_use_function_calling", "paper:2210.14222", "insight:in_context_learning_emerges_at_scale", "insight:symbolic_intermediate_enables_interpretability_and_transfer"]}, + + {"id": "validation:trace_knowledge_driven_reflective_agent", "label_en": "Trace: Knowledge-Driven Reflective Agent", "label_zh": "再发现:DiLu(知识驱动反思智能体)", "kind": "validation", "tier": "validation", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "如果 DiLu 不存在,要重新发明它,研究者需要的构件是:GPT-3 级别的语言推理能力作为通用决策器;chain-of-thought 让模型显式写出选择车道、避让的理由;类似 Reflexion 的外部记忆与反思机制把过去错误的决策回溯并修正;驾驶仿真器作为决策闭环;以及对照 Sutton 苦涩教训的批判性立场,承认在数据稀缺的小型试点中知识与符号反思可以暂时弥补 scaling 的不足。把这些组合起来便能得到一个写自然语言决策日志、维护可检索经验池、用反思修正错误的知识驱动驾驶 agent。", "building_blocks": ["paper:gpt3", "concept:vlm", "concept:cot", "concept:tool_use", "essay:bitter_lesson", "paper:ad_benchmarks", "insight:symbolic_intermediate_enables_interpretability_and_transfer", "insight:in_context_learning_emerges_at_scale"]}, + + {"id": "validation:trace_brain_inspired_spike_attention", "label_en": "Trace: Brain-inspired Spike Attention", "label_zh": "再发现:Spike-driven Transformer", "kind": "validation", "tier": "validation", "topic": "brain_inspired", "phase": "frontier", "year": 2023, "summary_zh": "如果 Spike-driven Transformer 不存在,要把它再发明出来,研究者需要的构件是:脉冲神经网络的发放-阈值-膜电位计算模型;transformer 中的自注意力作为通用 mixer;ResNet 启发的残差结构保证深层可训练性;把 query-key 点积重写为脉冲触发的稀疏外积的代数技巧;事件稀疏计算可匹配边缘能耗预算的洞见以及对苦涩教训的逆向思考即接受在能耗约束硬性主导的边缘场景里仍需引入硬件友好的归纳偏置。组合起来即可得到一个用脉冲事件代替密集激活、保留注意力表达力又显著节能的模型。", "building_blocks": ["concept:spiking_nn", "paper:vaswani2017", "paper:vit", "paper:he2015_resnet", "concept:self_attention", "move:residual_connection", "move:spike_event_compute", "insight:event_sparse_compute_matches_energy_budget", "essay:bitter_lesson"]}, + + {"id": "validation:trace_scalable_self_supervised_vision_backbone", "label_en": "Trace: Scalable Self-Supervised Vision Backbone", "label_zh": "再发现:DINOv3(可规模化自监督视觉主干)", "kind": "validation", "tier": "validation", "topic": "ssl_vision", "phase": "frontier", "year": 2025, "summary_zh": "如果 DINOv3 不存在,要把它再发明出来,研究者需要的构件是:DINOv2 提供的自蒸馏多裁剪自监督训练配方;ViT 提供的可扩展架构;掩码图像建模作为辅助监督;scaling 定律告诉我们更大的模型与数据继续提升下游能力;基础模型预训练把数据与任务解耦的洞见使得驾驶等下游可以零样本受益;以及对苦涩教训的承诺即长期看大规模无监督预训练胜过手工特征。把这些组合起来便可推出一个把数据规模、模型规模、训练步数同时再推一档以提供下一代驾驶视觉表征的工作。", "building_blocks": ["paper:dinov2", "paper:vit", "concept:ssl", "move:masking_for_pretext", "essay:bitter_lesson", "insight:foundation_pretraining_decouples_data_from_task", "insight:scaling_laws_predict_capability_emergence", "insight:masked_prediction_yields_self_supervised_signal"]}, + + {"id": "validation:trace_counterfactual_vla_replanner", "label_en": "Trace: Counterfactual VLA Replanner", "label_zh": "再发现:CF-VLA(反事实 VLA 重规划器)", "kind": "validation", "tier": "validation", "topic": "vlm_vla", "phase": "frontier", "year": 2025, "summary_zh": "如果 CF-VLA 不存在,要把它再发明出来,研究者需要的构件是:DriveVLM 等已经稳定的 VLA 主干提供基础的多模态感知与符号决策;meta-action 作为高层语义动作的离散接口以便枚举反事实;驾驶世界模型用以快速模拟反事实轨迹的结果;RLHF 与偏好对齐作为对反事实之间偏好打分的训练手段;反事实重规划作为方法学移动;以及长尾要靠合成而非更多真实数据解决的洞见。组合起来即可推出一个在每个决策点枚举若干 meta-action 替代方案、用世界模型推演并用偏好对齐挑选最优方案的反事实 VLA 重规划框架。", "building_blocks": ["paper:2402.12289", "paper:llava", "paper:world_models", "paper:gaia1", "paper:drivedreamer", "paper:rlhf_dpo", "concept:vla", "concept:counterfactual", "concept:meta_action", "move:counterfactual_replan", "move:latent_imagination_rollout", "insight:long_tail_solved_by_synthesis_not_data_alone", "insight:world_models_let_planning_be_done_in_imagination"]}, + + {"id": "validation:trace_set_prediction_with_object_queries", "label_en": "Trace: Set Prediction with Object Queries", "label_zh": "再发现:DETR(基于对象 query 的集合预测)", "kind": "validation", "tier": "validation", "topic": "ssl_vision", "phase": "prereq", "year": 2020, "summary_zh": "如果 DETR 不存在,要再发明它,研究者需要的构件是:Transformer 的编码器解码器作为对图像 patch 与对象之间任意远依赖的建模工具;ViT 启发的把图像切成 patch 的 tokenize 习惯(或 CNN 主干输出特征图);匈牙利算法在二分图上做最优分配的经典工具;模仿与监督学习中对集合输出的损失设计经验;以及把每个待检测对象抽象为一个 query 与图像 cross-attention 的关键洞见。组合起来即可得到一个用固定数量的 query、通过 cross-attention 从图像中拉取对象信息并经匈牙利匹配计算损失的检测器,从而彻底去除锚框和非极大值抑制。", "building_blocks": ["paper:vaswani2017", "concept:transformer", "concept:self_attention", "move:cross_attention_query", "move:set_prediction_with_hungarian", "insight:attention_is_typed_entity_communication", "insight:set_prediction_eliminates_postprocessing_heuristics"]}, + + {"id": "validation:trace_self_attention_replaces_recurrence", "label_en": "Trace: Self-Attention Replaces Recurrence", "label_zh": "再发现:Transformer(自注意力取代循环)", "kind": "validation", "tier": "validation", "topic": "math_foundations", "phase": "prereq", "year": 2017, "summary_zh": "如果 Transformer 不存在,要把它再发明出来,研究者需要的构件是:seq2seq 翻译框架已经引入的注意力作为序列对齐机制;ResNet 已经普及的残差连接与层归一化作为深网络训练的稳定子;多头机制对应了把若干并行子空间的检索合并起来的直觉;位置编码作为对序列顺序的可加表示;以及把所有时间步的计算并行化以摆脱循环网络瓶颈的工程动机。组合起来便可推出一个完全靠 self-attention 与残差子层堆叠、能完全并行训练、对长距离依赖一阶建模的纯注意力网络。", "building_blocks": ["paper:he2015_resnet", "concept:self_attention", "concept:transformer", "move:residual_connection", "move:cross_attention_query", "insight:residual_learning_unlocks_arbitrary_depth", "insight:attention_is_typed_entity_communication"]}, + + {"id": "validation:trace_image_transformer_via_patch_tokenization", "label_en": "Trace: Image Transformer via Patch Tokenization", "label_zh": "再发现:ViT(patch tokenize 的图像 transformer)", "kind": "validation", "tier": "validation", "topic": "ssl_vision", "phase": "prereq", "year": 2020, "summary_zh": "如果 ViT 不存在,要再发明它,研究者需要的构件是:Transformer 已经在文本上证明其在大规模数据上的优势;ResNet 时代积累的图像分类基准与数据集;把图像切成 16×16 patch 并展平为序列作为 tokenize 移动;位置编码引入图像中的空间结构;以及对 scaling 定律的信念使得研究者愿意接受 ViT 在小数据上不如 CNN 但在大数据上会反超的实验直觉。组合起来即可得到一个把图像视作 patch 序列、用纯 transformer 主干完成分类的架构,由此打开视觉基础模型的大门。", "building_blocks": ["paper:vaswani2017", "concept:transformer", "concept:self_attention", "move:patchify_tokenization", "insight:tokenization_collapses_modality_gap", "insight:scaling_laws_predict_capability_emergence"]}, + + {"id": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "label_en": "Trace: BEV Transformer with Temporal Aggregation", "label_zh": "再发现:BEVFormer(时空 BEV transformer)", "kind": "validation", "tier": "validation", "topic": "e2e_ad", "phase": "prereq", "year": 2022, "summary_zh": "如果 BEVFormer 不存在,要再发明它,研究者需要的构件是:DETR 提供的 query-based 检测范式;ViT 提供的图像 patch 编码主干;Lift-Splat-Shoot 思想的 2D 升 3D 投影;多相机标定与时序对齐的工程基础;以及把每个 BEV 网格点作为一个 query、对各相机特征做 deformable cross-attention、并把上一帧的 BEV 作为时间记忆做时序自注意力这一关键设计。组合起来即可推出一个直接在鸟瞰图坐标系下统一多相机感知、时序传递场景信息的 transformer 主干,为下游统一规划提供共享底座。", "building_blocks": ["paper:vaswani2017", "paper:vit", "paper:carion2020", "concept:transformer", "concept:detr_query", "concept:bev", "move:cross_attention_query", "move:lift_2d_to_3d", "move:patchify_tokenization", "insight:attention_is_typed_entity_communication"]}, + + {"id": "validation:trace_few_shot_in_context_learning_at_scale", "label_en": "Trace: Few-shot In-Context Learning at Scale", "label_zh": "再发现:GPT-3(大规模 few-shot 上下文学习)", "kind": "validation", "tier": "validation", "topic": "vlm_vla", "phase": "prereq", "year": 2020, "summary_zh": "如果 GPT-3 不存在,要再发明它,研究者需要的构件是:Transformer 解码器作为自回归语言建模主干;BERT 与 GPT-2 已经验证的预训练加微调范式;CommonCrawl 量级的网络文本作为预训练数据;模型与数据的 scaling 定律告诉我们继续把规模再放大一两个数量级会带来质变;以及对苦涩教训的承诺即放弃精细任务工程、把所有任务用统一的语言模型预训练目标处理。组合起来即可得到一个一百多亿到千亿参数级、在不更新参数的前提下从提示中学习新任务的通用语言模型,为后来所有 LLM 应用奠基。", "building_blocks": ["paper:vaswani2017", "concept:transformer", "essay:bitter_lesson", "concept:scaling_vs_knowledge", "insight:scaling_laws_predict_capability_emergence", "insight:in_context_learning_emerges_at_scale"]}, + + {"id": "validation:trace_clipped_policy_gradient_surrogate", "label_en": "Trace: Clipped Policy Gradient Surrogate", "label_zh": "再发现:PPO(剪裁策略梯度)", "kind": "validation", "tier": "validation", "topic": "deep_rl", "phase": "core", "year": 2017, "summary_zh": "如果 PPO 不存在,要再发明它,研究者需要的构件是:REINFORCE 形式的策略梯度作为基础;TRPO 提出的信赖域约束保证单步更新不过大;Actor-Critic 框架以共享值函数减小方差;广义优势估计 GAE 用以平衡偏差与方差;以及对工程友好性的强烈关注,希望避免 TRPO 的二阶共轭梯度计算。把这些组合起来便可推出一个用对策略比率施加上下界 clip 来近似信赖域、可以纯靠 SGD 训练、超参数少的策略梯度方法,并显著降低 RL 算法的工程门槛。", "building_blocks": ["concept:policy_gradient", "concept:actor_critic", "concept:ppo", "move:clipped_surrogate_objective", "course:zhao_rl", "paper:sutton_barto"]}, + + {"id": "validation:trace_deep_q_network_with_replay_and_target", "label_en": "Trace: Deep Q Network with Replay and Target", "label_zh": "再发现:DQN(深度 Q 网络)", "kind": "validation", "tier": "validation", "topic": "deep_rl", "phase": "prereq", "year": 2015, "summary_zh": "如果 DQN 不存在,要再发明它,研究者需要的构件是:Q-learning 与 Bellman 最优方程作为值函数学习的目标;CNN 作为从像素到动作值的可学习函数逼近;经验回放打破数据时序相关性;目标网络打破贝尔曼自举反馈回路;以及 Atari 仿真器作为高频可重复的环境。组合起来即可推出一个用 CNN 估计 Q 值、用经验回放和目标网络稳定训练、能够从纯像素学习多种 Atari 游戏的端到端深度强化学习算法。", "building_blocks": ["concept:mdp", "concept:bellman_eq", "concept:td_learning", "concept:dqn", "concept:replay_buffer", "move:replay_and_target_net", "course:zhao_rl", "paper:sutton_barto"]}, + + {"id": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", "label_en": "Trace: AlphaZero Self-Play with MCTS-Guided Policy", "label_zh": "再发现:AlphaZero(自对弈与 MCTS 指导策略)", "kind": "validation", "tier": "validation", "topic": "deep_rl", "phase": "prereq", "year": 2017, "summary_zh": "如果 AlphaZero 不存在,要再发明它,研究者需要的构件是:策略网络与价值网络作为一对联合训练的近似器;蒙特卡洛树搜索作为基于先验策略和价值估计的前向推演;自对弈作为生成无穷训练数据的源头;明确的、零和、完全信息的棋类规则作为闭环奖励;以及对苦涩教训的承诺即弃用人类棋谱与手工特征。组合起来即可推出一个仅靠自对弈数据、用 MCTS 改良策略、把 MCTS 改良后的访问分布作为策略训练目标的通用棋类超人系统。", "building_blocks": ["concept:mdp", "concept:policy_gradient", "concept:actor_critic", "essay:bitter_lesson", "move:self_play_with_search", "insight:test_time_compute_substitutes_train_time_via_search"]}, + + {"id": "validation:trace_dataset_aggregation_for_imitation", "label_en": "Trace: Dataset Aggregation for Imitation", "label_zh": "再发现:DAgger(模仿学习中的数据集聚合)", "kind": "validation", "tier": "validation", "topic": "deep_rl", "phase": "core", "year": 2011, "summary_zh": "如果 DAgger 不存在,要再发明它,研究者需要的构件是:行为克隆作为最朴素的监督模仿基线;对监督学习中分布偏移即协变量偏移的深入认识;专家策略可以被反复查询这一假设;以及在线学习中的 Follow-the-Leader 等理论思想给出聚合策略可以获得 no-regret 保证的灵感。组合起来即可推出一种迭代算法,让当前策略与环境交互、由专家在新状态上重新标注、把新数据加入训练集,从而解决纯 BC 在长视野任务上的崩塌问题。", "building_blocks": ["concept:imitation_learning", "concept:covariate_shift", "move:dataset_aggregation", "course:cs285", "insight:imitation_data_compresses_unspecified_reward"]}, + + {"id": "validation:trace_world_model_in_latent_imagination", "label_en": "Trace: World Model in Latent Imagination", "label_zh": "再发现:World Models / Dreamer(潜空间想象世界模型)", "kind": "validation", "tier": "validation", "topic": "deep_rl", "phase": "core", "year": 2018, "summary_zh": "如果 World Models 与 Dreamer 类工作不存在,要再发明它们,研究者需要的构件是:VAE 等概率潜变量模型作为对感知的紧凑编码;循环或状态空间网络作为对动力学的预测;策略与价值函数能够在潜空间 rollout 中训练;强化学习的策略梯度作为最终优化算法;以及把规划在想象中进行可以大幅降低环境样本复杂度的洞见。组合起来即可推出把感知压缩为潜表示、用 RNN 建模潜动力学、在潜空间中并行多步 rollout 训练策略的世界模型方法。", "building_blocks": ["concept:mdp", "concept:policy_gradient", "concept:actor_critic", "move:latent_imagination_rollout", "insight:world_models_let_planning_be_done_in_imagination", "paper:world_models"]}, + + {"id": "validation:trace_vision_language_pretrained_dual_encoder", "label_en": "Trace: Vision-Language Pretrained Dual Encoder", "label_zh": "再发现:CLIP(视觉语言对比预训练双编码器)", "kind": "validation", "tier": "validation", "topic": "ssl_vision", "phase": "prereq", "year": 2021, "summary_zh": "如果 CLIP 不存在,要再发明它,研究者需要的构件是:ViT 或 ResNet 作为可扩展图像编码器;Transformer 作为文本编码器;互联网规模的图文配对作为天然监督;InfoNCE 等对比学习损失把成对样本拉近、非成对样本推远;以及对比对齐可造就零样本迁移的洞见。组合起来即可推出一个由图像编码器、文本编码器和 InfoNCE 对齐目标构成的双塔模型,并显示其在零样本图像分类与跨模态检索上的全面突破。", "building_blocks": ["paper:vit", "paper:vaswani2017", "concept:vlm", "move:contrastive_alignment", "concept:ssl", "insight:contrastive_alignment_creates_zero_shot_transfer", "insight:foundation_pretraining_decouples_data_from_task"]}, + + {"id": "validation:trace_diffusion_policy_as_score_based_action_sampler", "label_en": "Trace: Diffusion Policy as Score-Based Action Sampler", "label_zh": "再发现:Diffusion Policy(基于得分的动作采样策略)", "kind": "validation", "tier": "validation", "topic": "deep_rl", "phase": "frontier", "year": 2022, "summary_zh": "如果 Diffusion Policy 不存在,要再发明它,研究者需要的构件是:DDPM 等图像扩散模型已经验证的反向去噪生成范式;模仿学习提供的状态动作配对数据;条件生成的标准化做法即把观测嵌入作为去噪网络的条件;多模态行为分布无法被单峰高斯捕捉这一经验观察;以及扩散统一生成与决策的洞见。组合起来即可推出一种把动作序列视为待去噪样本、把当前观测作为条件、在推断时迭代采样得到多模态动作分布的模仿策略。", "building_blocks": ["concept:imitation_learning", "move:diffusion_denoise_sampling", "paper:diffuser", "insight:diffusion_unifies_generation_and_decision", "insight:imitation_data_compresses_unspecified_reward"]}, + + {"id": "validation:trace_modular_perception_pipeline_with_bev_fusion", "label_en": "Trace: Modular Perception Pipeline with BEV Fusion", "label_zh": "再发现:BEV 融合模块化感知流水线(LSS / BEVFormer 家族)", "kind": "validation", "tier": "validation", "topic": "e2e_ad", "phase": "prereq", "year": 2020, "summary_zh": "如果 Lift-Splat-Shoot 与现代 BEV 融合流水线不存在,要再发明它们,研究者需要的构件是:多相机外参标定经验;每像素的深度分布预测的可学习方法;体素和栅格表示作为统一坐标系;ViT 或 CNN 作为图像编码主干;DETR 风格的 query 用以从 BEV 特征中拉取下游任务;以及 2D 升 3D 投影是 AD 中跨视角融合的天然语言这一洞见。组合起来即可推出一个把每相机图像特征沿深度概率展平为视锥体素、splat 到 BEV 栅格、再供下游检测和规划共享的统一感知流水线。", "building_blocks": ["paper:vit", "paper:carion2020", "paper:li2022bevformer", "concept:bev", "concept:detr_query", "move:lift_2d_to_3d", "move:cross_attention_query", "paper:tesla_ai_day"]}, + + {"id": "validation:trace_neural_field_for_dynamic_driving_scene", "label_en": "Trace: Neural Field for Dynamic Driving Scene", "label_zh": "再发现:EmerNeRF / DrivingGaussian(动态驾驶场神经场)", "kind": "validation", "tier": "validation", "topic": "e2e_ad", "phase": "frontier", "year": 2023, "summary_zh": "如果 EmerNeRF 与 DrivingGaussian 类方法不存在,要把它们再发明出来,研究者需要的构件是:NeRF 与 3D Gaussian Splatting 作为对静态三维场景的可微表示;驾驶日志中多相机加 LiDAR 加自车位姿作为输入;时空分解把场分为静态背景与动态对象的设计思想;以及把神经场视为可被仿真器消费的可微世界模型这一立场。组合起来即可推出一个能从真实驾驶日志中重建可视、可编辑、可重放的动态驾驶场,为大规模反事实生成和闭环仿真提供基础。", "building_blocks": ["paper:vit", "paper:gaia1", "paper:drivedreamer", "paper:world_models", "move:latent_imagination_rollout", "insight:long_tail_solved_by_synthesis_not_data_alone", "insight:world_models_let_planning_be_done_in_imagination"]}, + + {"id": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "label_en": "Trace: Decision Transformer (Offline RL via Sequence Modeling)", "label_zh": "再发现:Decision Transformer(序列建模式离线 RL)", "kind": "validation", "tier": "validation", "topic": "deep_rl", "phase": "frontier", "year": 2021, "summary_zh": "如果 Decision Transformer 不存在,要再发明它,研究者需要的构件是:GPT 风格的自回归 transformer 作为序列建模主干;离线强化学习的轨迹数据;把 return-to-go、状态、动作三元组拼接为 token 序列的关键 tokenize 移动;模仿学习作为监督信号;以及一个简单但有力的认识即只要把目标回报作为提示输入,自回归预测下一动作就等价于条件策略学习。组合起来即可推出一个完全用监督学习训练、却能在推断时通过指定高目标回报而得到高质量策略的离线强化学习方法。", "building_blocks": ["paper:vaswani2017", "paper:gpt3", "concept:transformer", "concept:imitation_learning", "move:tokenize_modalities", "insight:tokenization_collapses_modality_gap", "insight:imitation_data_compresses_unspecified_reward"]}, + + {"id": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", "label_en": "Trace: Safe RL via Lagrangian Constrained Optimization", "label_zh": "再发现:拉格朗日约束的安全强化学习", "kind": "validation", "tier": "validation", "topic": "deep_rl", "phase": "frontier", "year": 2019, "summary_zh": "如果拉格朗日安全强化学习方法不存在,要再发明它们,研究者需要的构件是:约束马尔可夫决策过程作为对硬安全限制的形式化;PPO 或 TRPO 等策略优化算法;拉格朗日对偶把约束转化为乘子调节的可微目标;估计成本函数与价值函数的方法;以及把安全视为与回报对偶而非奖励内塞软项的认识立场。组合起来即可推出一类把碰撞概率、加速度上限等约束乘以可学习乘子并与策略联合优化、在保证安全约束被满足前提下最大化驾驶性能的安全强化学习算法。", "building_blocks": ["concept:mdp", "concept:policy_gradient", "concept:actor_critic", "concept:ppo", "move:clipped_surrogate_objective", "insight:safety_constraints_via_lagrangian_dual"]}, + + + {"id": "paradigm:modular_perception_to_planning_pipeline", "label_en": "Paradigm: Modular Perception-to-Planning Pipeline", "label_zh": "范式:模块化感知到规划流水线", "kind": "paradigm", "tier": "paradigm", "topic": "e2e_ad", "phase": "prereq", "year": 2018, "summary_zh": "模块化感知到规划范式把自动驾驶分解为检测、跟踪、预测、规划、控制等明确的模块并通过结构化中间表示串联,押注于每个模块都可以独立验证、独立迭代、独立替换。其立场是把工程可维护性与可解释性置于联合优化之前,代表工作包括 Tesla 早期 AI Day 架构、Waymo 经典分层、以及任何在 BEVFormer 上接非神经规划器的产品系统。它的主要局限是模块边界压抑了端到端反馈、人为接口可能丢失下游任务真正需要的信息,并且每个模块的最优解不一定加起来等于系统最优。", "building_blocks": ["paper:tesla_ai_day", "paper:li2022bevformer", "paper:transfuser", "concept:bev", "move:lift_2d_to_3d", "insight:set_prediction_eliminates_postprocessing_heuristics"]}, + + {"id": "paradigm:differentiable_end_to_end_imitation", "label_en": "Paradigm: Differentiable End-to-End Imitation", "label_zh": "范式:可微端到端模仿", "kind": "paradigm", "tier": "paradigm", "topic": "e2e_ad", "phase": "core", "year": 2022, "summary_zh": "可微端到端模仿范式把感知到规划写成一个完全可微的神经网络并用人类驾驶日志作为监督,押注于在数据规模足够大时端到端反馈能够发现比手工接口更优的表征。代表工作包括 UniAD、PlanT、VADv2、TransFuser,它们的共同立场是模块化的人工接口最终会成为性能瓶颈。它的主要局限是难以验证、对协变量偏移敏感、奖励隐式且难以注入硬安全约束,且对车载推断与传感器变化的鲁棒性需要额外的工程努力。", "building_blocks": ["paper:2212.10156", "paper:2210.14222", "paper:vadv2", "paper:transfuser", "concept:imitation_learning", "move:cross_attention_query", "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong"]}, + + {"id": "paradigm:model_based_world_imagination_planning", "label_en": "Paradigm: Model-Based World Imagination Planning", "label_zh": "范式:基于世界模型的想象规划", "kind": "paradigm", "tier": "paradigm", "topic": "deep_rl", "phase": "frontier", "year": 2018, "summary_zh": "基于世界模型的想象规划范式把环境动力学学习成一个可微生成模型,再让策略在想象的潜空间或像素空间中以低代价 rollout,押注于这种方式可以把样本复杂度从昂贵的真实交互转移到便宜的想象推演。代表工作包括 World Models、Dreamer 系列、GAIA-1、DriveDreamer 与 CF-VLA 中的扩散世界模型。它的主要局限是世界模型的预测漂移会污染长视野规划,且对稀有事件的覆盖仍然取决于训练分布,导致罕见但关键的事故场景未必能被想象出来。", "building_blocks": ["paper:world_models", "paper:gaia1", "paper:drivedreamer", "move:latent_imagination_rollout", "move:diffusion_denoise_sampling", "insight:world_models_let_planning_be_done_in_imagination", "insight:long_tail_solved_by_synthesis_not_data_alone"]}, + + {"id": "paradigm:foundation_model_zero_shot_driving_agent", "label_en": "Paradigm: Foundation Model Zero-shot Driving Agent", "label_zh": "范式:基础模型零样本驾驶智能体", "kind": "paradigm", "tier": "paradigm", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "基础模型零样本驾驶智能体范式把通用预训练 VLM 或 LLM 当作驾驶决策核心,通过提示工程、少样本示例和工具调用解决长尾问题,押注于通用智能可以通过 scaling 与对齐迁移到驾驶。代表工作包括 DriveVLM、Agent-Driver、LINGO-2 以及 CF-VLA 的反思变体。它的主要局限是 VLM 的几何与控制精度不足、推断延迟高、对车载算力压力大、安全性难以严格验证,因此实际部署多采取与传统快规划器并行的双系统架构。", "building_blocks": ["paper:gpt3", "paper:llava", "paper:2402.12289", "paper:2311.10813", "concept:vla", "concept:tool_use", "move:dual_system_fast_slow", "insight:dual_system_handles_latency_quality_tradeoff", "insight:in_context_learning_emerges_at_scale"]}, + + {"id": "paradigm:brain_inspired_event_sparse_compute", "label_en": "Paradigm: Brain-Inspired Event-Sparse Compute", "label_zh": "范式:类脑事件稀疏计算", "kind": "paradigm", "tier": "paradigm", "topic": "brain_inspired", "phase": "frontier", "year": 2020, "summary_zh": "类脑事件稀疏计算范式把脉冲神经元、事件相机、神经形态硬件作为核心计算单元,押注于自然信号的稀疏性可以在能耗与延迟上换取数量级优势。代表工作包括 Loihi 系列芯片、Spike-driven Transformer、事件相机视觉栈以及汽车端的低功耗感知试验。它与 Sutton 苦涩教训形成鲜明对照,主要局限是训练算法不成熟、与主流密集模型生态脱节、在大规模数据上仍未达到与稠密 transformer 相当的精度,因此目前主要作为车载补充而非主路径。", "building_blocks": ["concept:spiking_nn", "paper:2307.01694", "move:spike_event_compute", "insight:event_sparse_compute_matches_energy_budget"]}, + + {"id": "paradigm:counterfactual_data_centric_safety", "label_en": "Paradigm: Counterfactual Data-Centric Safety", "label_zh": "范式:以反事实数据为中心的安全", "kind": "paradigm", "tier": "paradigm", "topic": "vlm_vla", "phase": "frontier", "year": 2024, "summary_zh": "反事实数据中心安全范式把驾驶安全问题主要视为数据问题而非模型问题,依靠合成反事实场景、扰动安全约束、对抗性场景生成来覆盖真实世界采样不到的长尾事件。代表工作包括 CF-VLA、Waymo CCD、CARLA leaderboard 中的对抗场景以及反事实重规划方法。它的立场是更多的真实数据存在边际递减,应该把投入转向高质量的反事实合成。主要局限是合成数据的分布偏置可能与真实事故分布不一致,需要严格的分布匹配工具与人在回路审核。", "building_blocks": ["paper:2512.24426", "paper:gaia1", "paper:drivedreamer", "concept:counterfactual", "concept:meta_action", "move:counterfactual_replan", "insight:long_tail_solved_by_synthesis_not_data_alone"]}, + + {"id": "paradigm:knowledge_driven_reflective_agent", "label_en": "Paradigm: Knowledge-Driven Reflective Agent", "label_zh": "范式:知识驱动反思智能体", "kind": "paradigm", "tier": "paradigm", "topic": "vlm_vla", "phase": "frontier", "year": 2023, "summary_zh": "知识驱动反思智能体范式把 LLM 视为一个可以读写自然语言记忆、做事后反思并把经验沉淀为可复用知识的认知核心,押注于符号反思可以在数据有限时弥补 scaling 的不足。代表工作包括 DiLu、Reflexion 启发的驾驶变体、以及把法律法规与道路手册显式作为提示注入的 Agent-Driver。它的主要立场是与 Sutton 苦涩教训形成有意识的对照,承认知识工程在小规模、数据稀缺、跨地域迁移时仍有杠杆。主要局限是反思与记忆的可靠性依赖 LLM 推理稳定性,并且记忆膨胀后检索的精度与延迟需要工程化解决。", "building_blocks": ["paper:gpt3", "paper:2309.16292", "paper:2311.10813", "concept:cot", "concept:tool_use", "essay:bitter_lesson", "insight:symbolic_intermediate_enables_interpretability_and_transfer"]}, + + {"id": "paradigm:scaling_data_with_self_supervision", "label_en": "Paradigm: Scaling Data with Self-Supervision", "label_zh": "范式:以自监督扩张数据规模", "kind": "paradigm", "tier": "paradigm", "topic": "ssl_vision", "phase": "core", "year": 2020, "summary_zh": "以自监督扩张数据范式承诺把所有可获得的无标签数据通过掩码预测、对比对齐、自蒸馏等方式变成训练信号,押注于这样得到的通用表征可以在下游驾驶任务上以少量标签微调而获得显著迁移。代表工作包括 BERT、MAE、DINOv2、DINOv3、SAM 以及驾驶领域的占据预训练与 GAIA-1 自监督世界模型。它的主要局限是预训练任务与下游驾驶任务并非完全对齐、对计算预算极敏感,且自监督本身并不能解决奖励、规划与因果反思这类问题。", "building_blocks": ["paper:dinov2", "paper:2508.10104", "paper:sam", "concept:ssl", "move:masking_for_pretext", "move:contrastive_alignment", "insight:masked_prediction_yields_self_supervised_signal", "insight:foundation_pretraining_decouples_data_from_task", "insight:scaling_laws_predict_capability_emergence"]} + ], + + "edges": [ + + {"source": "move:residual_connection", "target": "insight:residual_learning_unlocks_arbitrary_depth", "rel": "supports"}, + {"source": "paper:he2015_resnet", "target": "insight:residual_learning_unlocks_arbitrary_depth", "rel": "supports"}, + {"source": "move:masking_for_pretext", "target": "insight:masked_prediction_yields_self_supervised_signal", "rel": "supports"}, + {"source": "concept:ssl", "target": "insight:masked_prediction_yields_self_supervised_signal", "rel": "supports"}, + {"source": "move:cross_attention_query", "target": "insight:attention_is_typed_entity_communication", "rel": "supports"}, + {"source": "concept:self_attention", "target": "insight:attention_is_typed_entity_communication", "rel": "supports"}, + {"source": "concept:detr_query", "target": "insight:attention_is_typed_entity_communication", "rel": "supports"}, + {"source": "move:contrastive_alignment", "target": "insight:contrastive_alignment_creates_zero_shot_transfer", "rel": "supports"}, + {"source": "concept:vlm", "target": "insight:contrastive_alignment_creates_zero_shot_transfer", "rel": "supports"}, + {"source": "move:diffusion_denoise_sampling", "target": "insight:diffusion_unifies_generation_and_decision", "rel": "supports"}, + {"source": "paper:diffuser", "target": "insight:diffusion_unifies_generation_and_decision", "rel": "supports"}, + {"source": "paper:2212.10156", "target": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", "rel": "supports"}, + {"source": "essay:bitter_lesson", "target": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", "rel": "supports"}, + {"source": "move:dual_system_fast_slow", "target": "insight:dual_system_handles_latency_quality_tradeoff", "rel": "supports"}, + {"source": "paper:2402.12289", "target": "insight:dual_system_handles_latency_quality_tradeoff", "rel": "supports"}, + {"source": "concept:cot", "target": "insight:symbolic_intermediate_enables_interpretability_and_transfer", "rel": "supports"}, + {"source": "paper:2309.16292", "target": "insight:symbolic_intermediate_enables_interpretability_and_transfer", "rel": "supports"}, + {"source": "move:counterfactual_replan", "target": "insight:long_tail_solved_by_synthesis_not_data_alone", "rel": "supports"}, + {"source": "paper:2512.24426", "target": "insight:long_tail_solved_by_synthesis_not_data_alone", "rel": "supports"}, + {"source": "paper:drivedreamer", "target": "insight:long_tail_solved_by_synthesis_not_data_alone", "rel": "supports"}, + {"source": "essay:bitter_lesson", "target": "insight:scaling_laws_predict_capability_emergence", "rel": "supports"}, + {"source": "concept:scaling_vs_knowledge", "target": "insight:scaling_laws_predict_capability_emergence", "rel": "supports"}, + {"source": "paper:gpt3", "target": "insight:scaling_laws_predict_capability_emergence", "rel": "supports"}, + {"source": "paper:dinov2", "target": "insight:foundation_pretraining_decouples_data_from_task", "rel": "supports"}, + {"source": "paper:2508.10104", "target": "insight:foundation_pretraining_decouples_data_from_task", "rel": "supports"}, + {"source": "concept:ssl", "target": "insight:foundation_pretraining_decouples_data_from_task", "rel": "supports"}, + {"source": "move:self_play_with_search", "target": "insight:test_time_compute_substitutes_train_time_via_search", "rel": "supports"}, + {"source": "paper:silver2017_alphazero", "target": "insight:test_time_compute_substitutes_train_time_via_search", "rel": "supports"}, + {"source": "concept:imitation_learning", "target": "insight:imitation_data_compresses_unspecified_reward", "rel": "supports"}, + {"source": "paper:ross2011_dagger", "target": "insight:imitation_data_compresses_unspecified_reward", "rel": "supports"}, + {"source": "move:latent_imagination_rollout", "target": "insight:world_models_let_planning_be_done_in_imagination", "rel": "supports"}, + {"source": "paper:world_models", "target": "insight:world_models_let_planning_be_done_in_imagination", "rel": "supports"}, + {"source": "paper:gaia1", "target": "insight:world_models_let_planning_be_done_in_imagination", "rel": "supports"}, + {"source": "move:tokenize_modalities", "target": "insight:tokenization_collapses_modality_gap", "rel": "supports"}, + {"source": "move:patchify_tokenization", "target": "insight:tokenization_collapses_modality_gap", "rel": "supports"}, + {"source": "move:set_prediction_with_hungarian", "target": "insight:set_prediction_eliminates_postprocessing_heuristics", "rel": "supports"}, + {"source": "paper:carion2020", "target": "insight:set_prediction_eliminates_postprocessing_heuristics", "rel": "supports"}, + {"source": "paper:gpt3", "target": "insight:in_context_learning_emerges_at_scale", "rel": "supports"}, + {"source": "concept:cot", "target": "insight:in_context_learning_emerges_at_scale", "rel": "supports"}, + {"source": "concept:policy_gradient", "target": "insight:safety_constraints_via_lagrangian_dual", "rel": "supports"}, + {"source": "concept:actor_critic", "target": "insight:safety_constraints_via_lagrangian_dual", "rel": "supports"}, + {"source": "move:spike_event_compute", "target": "insight:event_sparse_compute_matches_energy_budget", "rel": "supports"}, + {"source": "concept:spiking_nn", "target": "insight:event_sparse_compute_matches_energy_budget", "rel": "supports"}, + {"source": "paper:2307.01694", "target": "insight:event_sparse_compute_matches_energy_budget", "rel": "supports"}, + + {"source": "paper:li2022bevformer", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "paper:carion2020", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "concept:bev", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "concept:detr_query", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "move:cross_attention_query", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "move:set_prediction_with_hungarian", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "concept:imitation_learning", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "insight:attention_is_typed_entity_communication", "target": "validation:trace_unified_planning_oriented_e2e_driving", "rel": "composes"}, + {"source": "validation:trace_unified_planning_oriented_e2e_driving", "target": "paper:2212.10156", "rel": "validates"}, + + {"source": "paper:vaswani2017", "target": "validation:trace_object_level_planning_transformer", "rel": "composes"}, + {"source": "paper:carion2020", "target": "validation:trace_object_level_planning_transformer", "rel": "composes"}, + {"source": "concept:transformer", "target": "validation:trace_object_level_planning_transformer", "rel": "composes"}, + {"source": "move:cross_attention_query", "target": "validation:trace_object_level_planning_transformer", "rel": "composes"}, + {"source": "move:set_prediction_with_hungarian", "target": "validation:trace_object_level_planning_transformer", "rel": "composes"}, + {"source": "concept:imitation_learning", "target": "validation:trace_object_level_planning_transformer", "rel": "composes"}, + {"source": "paper:ad_benchmarks", "target": "validation:trace_object_level_planning_transformer", "rel": "composes"}, + {"source": "paper:transfuser", "target": "validation:trace_object_level_planning_transformer", "rel": "composes"}, + {"source": "validation:trace_object_level_planning_transformer", "target": "paper:2210.14222", "rel": "validates"}, + + {"source": "paper:llava", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "concept:vlm", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "concept:vla", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "concept:cot", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "move:dual_system_fast_slow", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "move:cross_attention_query", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "insight:dual_system_handles_latency_quality_tradeoff", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "insight:symbolic_intermediate_enables_interpretability_and_transfer", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "paper:li2022bevformer", "target": "validation:trace_vision_language_action_dual_loop", "rel": "composes"}, + {"source": "validation:trace_vision_language_action_dual_loop", "target": "paper:2402.12289", "rel": "validates"}, + + {"source": "paper:gpt3", "target": "validation:trace_llm_decision_agent_for_driving", "rel": "composes"}, + {"source": "concept:vlm", "target": "validation:trace_llm_decision_agent_for_driving", "rel": "composes"}, + {"source": "concept:cot", "target": "validation:trace_llm_decision_agent_for_driving", "rel": "composes"}, + {"source": "concept:tool_use", "target": "validation:trace_llm_decision_agent_for_driving", "rel": "composes"}, + {"source": "move:tool_use_function_calling", "target": "validation:trace_llm_decision_agent_for_driving", "rel": "composes"}, + {"source": "paper:2210.14222", "target": "validation:trace_llm_decision_agent_for_driving", "rel": "composes"}, + {"source": "insight:in_context_learning_emerges_at_scale", "target": "validation:trace_llm_decision_agent_for_driving", "rel": "composes"}, + {"source": "insight:symbolic_intermediate_enables_interpretability_and_transfer", "target": "validation:trace_llm_decision_agent_for_driving", "rel": "composes"}, + {"source": "validation:trace_llm_decision_agent_for_driving", "target": "paper:2311.10813", "rel": "validates"}, + + {"source": "paper:gpt3", "target": "validation:trace_knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "concept:vlm", "target": "validation:trace_knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "concept:cot", "target": "validation:trace_knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "concept:tool_use", "target": "validation:trace_knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "essay:bitter_lesson", "target": "validation:trace_knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "paper:ad_benchmarks", "target": "validation:trace_knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "insight:symbolic_intermediate_enables_interpretability_and_transfer", "target": "validation:trace_knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "insight:in_context_learning_emerges_at_scale", "target": "validation:trace_knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "validation:trace_knowledge_driven_reflective_agent", "target": "paper:2309.16292", "rel": "validates"}, + + {"source": "concept:spiking_nn", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "paper:vaswani2017", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "paper:vit", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "paper:he2015_resnet", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "concept:self_attention", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "move:residual_connection", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "move:spike_event_compute", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "insight:event_sparse_compute_matches_energy_budget", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "essay:bitter_lesson", "target": "validation:trace_brain_inspired_spike_attention", "rel": "composes"}, + {"source": "validation:trace_brain_inspired_spike_attention", "target": "paper:2307.01694", "rel": "validates"}, + + {"source": "paper:dinov2", "target": "validation:trace_scalable_self_supervised_vision_backbone", "rel": "composes"}, + {"source": "paper:vit", "target": "validation:trace_scalable_self_supervised_vision_backbone", "rel": "composes"}, + {"source": "concept:ssl", "target": "validation:trace_scalable_self_supervised_vision_backbone", "rel": "composes"}, + {"source": "move:masking_for_pretext", "target": "validation:trace_scalable_self_supervised_vision_backbone", "rel": "composes"}, + {"source": "essay:bitter_lesson", "target": "validation:trace_scalable_self_supervised_vision_backbone", "rel": "composes"}, + {"source": "insight:foundation_pretraining_decouples_data_from_task", "target": "validation:trace_scalable_self_supervised_vision_backbone", "rel": "composes"}, + {"source": "insight:scaling_laws_predict_capability_emergence", "target": "validation:trace_scalable_self_supervised_vision_backbone", "rel": "composes"}, + {"source": "insight:masked_prediction_yields_self_supervised_signal", "target": "validation:trace_scalable_self_supervised_vision_backbone", "rel": "composes"}, + {"source": "validation:trace_scalable_self_supervised_vision_backbone", "target": "paper:2508.10104", "rel": "validates"}, + + {"source": "paper:2402.12289", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "paper:llava", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "paper:world_models", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "paper:gaia1", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "paper:drivedreamer", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "paper:rlhf_dpo", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "concept:vla", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "concept:counterfactual", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "concept:meta_action", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "move:counterfactual_replan", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "move:latent_imagination_rollout", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "insight:long_tail_solved_by_synthesis_not_data_alone", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "insight:world_models_let_planning_be_done_in_imagination", "target": "validation:trace_counterfactual_vla_replanner", "rel": "composes"}, + {"source": "validation:trace_counterfactual_vla_replanner", "target": "paper:2512.24426", "rel": "validates"}, + + {"source": "paper:vaswani2017", "target": "validation:trace_set_prediction_with_object_queries", "rel": "composes"}, + {"source": "concept:transformer", "target": "validation:trace_set_prediction_with_object_queries", "rel": "composes"}, + {"source": "concept:self_attention", "target": "validation:trace_set_prediction_with_object_queries", "rel": "composes"}, + {"source": "move:cross_attention_query", "target": "validation:trace_set_prediction_with_object_queries", "rel": "composes"}, + {"source": "move:set_prediction_with_hungarian", "target": "validation:trace_set_prediction_with_object_queries", "rel": "composes"}, + {"source": "insight:attention_is_typed_entity_communication", "target": "validation:trace_set_prediction_with_object_queries", "rel": "composes"}, + {"source": "insight:set_prediction_eliminates_postprocessing_heuristics", "target": "validation:trace_set_prediction_with_object_queries", "rel": "composes"}, + {"source": "validation:trace_set_prediction_with_object_queries", "target": "paper:carion2020", "rel": "validates"}, + + {"source": "paper:he2015_resnet", "target": "validation:trace_self_attention_replaces_recurrence", "rel": "composes"}, + {"source": "concept:self_attention", "target": "validation:trace_self_attention_replaces_recurrence", "rel": "composes"}, + {"source": "concept:transformer", "target": "validation:trace_self_attention_replaces_recurrence", "rel": "composes"}, + {"source": "move:residual_connection", "target": "validation:trace_self_attention_replaces_recurrence", "rel": "composes"}, + {"source": "move:cross_attention_query", "target": "validation:trace_self_attention_replaces_recurrence", "rel": "composes"}, + {"source": "insight:residual_learning_unlocks_arbitrary_depth", "target": "validation:trace_self_attention_replaces_recurrence", "rel": "composes"}, + {"source": "insight:attention_is_typed_entity_communication", "target": "validation:trace_self_attention_replaces_recurrence", "rel": "composes"}, + {"source": "validation:trace_self_attention_replaces_recurrence", "target": "paper:vaswani2017", "rel": "validates"}, + + {"source": "paper:vaswani2017", "target": "validation:trace_image_transformer_via_patch_tokenization", "rel": "composes"}, + {"source": "concept:transformer", "target": "validation:trace_image_transformer_via_patch_tokenization", "rel": "composes"}, + {"source": "concept:self_attention", "target": "validation:trace_image_transformer_via_patch_tokenization", "rel": "composes"}, + {"source": "move:patchify_tokenization", "target": "validation:trace_image_transformer_via_patch_tokenization", "rel": "composes"}, + {"source": "insight:tokenization_collapses_modality_gap", "target": "validation:trace_image_transformer_via_patch_tokenization", "rel": "composes"}, + {"source": "insight:scaling_laws_predict_capability_emergence", "target": "validation:trace_image_transformer_via_patch_tokenization", "rel": "composes"}, + {"source": "validation:trace_image_transformer_via_patch_tokenization", "target": "paper:vit", "rel": "validates"}, + + {"source": "paper:vaswani2017", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "paper:vit", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "paper:carion2020", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "concept:transformer", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "concept:detr_query", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "concept:bev", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "move:cross_attention_query", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "move:lift_2d_to_3d", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "move:patchify_tokenization", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "insight:attention_is_typed_entity_communication", "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "rel": "composes"}, + {"source": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", "target": "paper:li2022bevformer", "rel": "validates"}, + + {"source": "paper:vaswani2017", "target": "validation:trace_few_shot_in_context_learning_at_scale", "rel": "composes"}, + {"source": "concept:transformer", "target": "validation:trace_few_shot_in_context_learning_at_scale", "rel": "composes"}, + {"source": "essay:bitter_lesson", "target": "validation:trace_few_shot_in_context_learning_at_scale", "rel": "composes"}, + {"source": "concept:scaling_vs_knowledge", "target": "validation:trace_few_shot_in_context_learning_at_scale", "rel": "composes"}, + {"source": "insight:scaling_laws_predict_capability_emergence", "target": "validation:trace_few_shot_in_context_learning_at_scale", "rel": "composes"}, + {"source": "insight:in_context_learning_emerges_at_scale", "target": "validation:trace_few_shot_in_context_learning_at_scale", "rel": "composes"}, + {"source": "validation:trace_few_shot_in_context_learning_at_scale", "target": "paper:gpt3", "rel": "validates"}, + + {"source": "concept:policy_gradient", "target": "validation:trace_clipped_policy_gradient_surrogate", "rel": "composes"}, + {"source": "concept:actor_critic", "target": "validation:trace_clipped_policy_gradient_surrogate", "rel": "composes"}, + {"source": "concept:ppo", "target": "validation:trace_clipped_policy_gradient_surrogate", "rel": "composes"}, + {"source": "move:clipped_surrogate_objective", "target": "validation:trace_clipped_policy_gradient_surrogate", "rel": "composes"}, + {"source": "course:zhao_rl", "target": "validation:trace_clipped_policy_gradient_surrogate", "rel": "composes"}, + {"source": "paper:sutton_barto", "target": "validation:trace_clipped_policy_gradient_surrogate", "rel": "composes"}, + {"source": "validation:trace_clipped_policy_gradient_surrogate", "target": "paper:schulman2017_ppo", "rel": "validates"}, + + {"source": "concept:mdp", "target": "validation:trace_deep_q_network_with_replay_and_target", "rel": "composes"}, + {"source": "concept:bellman_eq", "target": "validation:trace_deep_q_network_with_replay_and_target", "rel": "composes"}, + {"source": "concept:td_learning", "target": "validation:trace_deep_q_network_with_replay_and_target", "rel": "composes"}, + {"source": "concept:dqn", "target": "validation:trace_deep_q_network_with_replay_and_target", "rel": "composes"}, + {"source": "concept:replay_buffer", "target": "validation:trace_deep_q_network_with_replay_and_target", "rel": "composes"}, + {"source": "move:replay_and_target_net", "target": "validation:trace_deep_q_network_with_replay_and_target", "rel": "composes"}, + {"source": "course:zhao_rl", "target": "validation:trace_deep_q_network_with_replay_and_target", "rel": "composes"}, + {"source": "paper:sutton_barto", "target": "validation:trace_deep_q_network_with_replay_and_target", "rel": "composes"}, + {"source": "validation:trace_deep_q_network_with_replay_and_target", "target": "paper:mnih2015_dqn", "rel": "validates"}, + + {"source": "concept:mdp", "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", "rel": "composes"}, + {"source": "concept:policy_gradient", "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", "rel": "composes"}, + {"source": "concept:actor_critic", "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", "rel": "composes"}, + {"source": "essay:bitter_lesson", "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", "rel": "composes"}, + {"source": "move:self_play_with_search", "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", "rel": "composes"}, + {"source": "insight:test_time_compute_substitutes_train_time_via_search", "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", "rel": "composes"}, + {"source": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", "target": "paper:silver2017_alphazero", "rel": "validates"}, + + {"source": "concept:imitation_learning", "target": "validation:trace_dataset_aggregation_for_imitation", "rel": "composes"}, + {"source": "concept:covariate_shift", "target": "validation:trace_dataset_aggregation_for_imitation", "rel": "composes"}, + {"source": "move:dataset_aggregation", "target": "validation:trace_dataset_aggregation_for_imitation", "rel": "composes"}, + {"source": "course:cs285", "target": "validation:trace_dataset_aggregation_for_imitation", "rel": "composes"}, + {"source": "insight:imitation_data_compresses_unspecified_reward", "target": "validation:trace_dataset_aggregation_for_imitation", "rel": "composes"}, + {"source": "validation:trace_dataset_aggregation_for_imitation", "target": "paper:ross2011_dagger", "rel": "validates"}, + + {"source": "concept:mdp", "target": "validation:trace_world_model_in_latent_imagination", "rel": "composes"}, + {"source": "concept:policy_gradient", "target": "validation:trace_world_model_in_latent_imagination", "rel": "composes"}, + {"source": "concept:actor_critic", "target": "validation:trace_world_model_in_latent_imagination", "rel": "composes"}, + {"source": "move:latent_imagination_rollout", "target": "validation:trace_world_model_in_latent_imagination", "rel": "composes"}, + {"source": "insight:world_models_let_planning_be_done_in_imagination", "target": "validation:trace_world_model_in_latent_imagination", "rel": "composes"}, + {"source": "paper:world_models", "target": "validation:trace_world_model_in_latent_imagination", "rel": "composes"}, + {"source": "validation:trace_world_model_in_latent_imagination", "target": "paper:world_models", "rel": "validates"}, + + {"source": "paper:vit", "target": "validation:trace_vision_language_pretrained_dual_encoder", "rel": "composes"}, + {"source": "paper:vaswani2017", "target": "validation:trace_vision_language_pretrained_dual_encoder", "rel": "composes"}, + {"source": "concept:vlm", "target": "validation:trace_vision_language_pretrained_dual_encoder", "rel": "composes"}, + {"source": "move:contrastive_alignment", "target": "validation:trace_vision_language_pretrained_dual_encoder", "rel": "composes"}, + {"source": "concept:ssl", "target": "validation:trace_vision_language_pretrained_dual_encoder", "rel": "composes"}, + {"source": "insight:contrastive_alignment_creates_zero_shot_transfer", "target": "validation:trace_vision_language_pretrained_dual_encoder", "rel": "composes"}, + {"source": "insight:foundation_pretraining_decouples_data_from_task", "target": "validation:trace_vision_language_pretrained_dual_encoder", "rel": "composes"}, + {"source": "validation:trace_vision_language_pretrained_dual_encoder", "target": "paper:llava", "rel": "validates"}, + + {"source": "concept:imitation_learning", "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", "rel": "composes"}, + {"source": "move:diffusion_denoise_sampling", "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", "rel": "composes"}, + {"source": "paper:diffuser", "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", "rel": "composes"}, + {"source": "insight:diffusion_unifies_generation_and_decision", "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", "rel": "composes"}, + {"source": "insight:imitation_data_compresses_unspecified_reward", "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", "rel": "composes"}, + {"source": "validation:trace_diffusion_policy_as_score_based_action_sampler", "target": "paper:diffuser", "rel": "validates"}, + + {"source": "paper:vit", "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", "rel": "composes"}, + {"source": "paper:carion2020", "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", "rel": "composes"}, + {"source": "paper:li2022bevformer", "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", "rel": "composes"}, + {"source": "concept:bev", "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", "rel": "composes"}, + {"source": "concept:detr_query", "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", "rel": "composes"}, + {"source": "move:lift_2d_to_3d", "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", "rel": "composes"}, + {"source": "move:cross_attention_query", "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", "rel": "composes"}, + {"source": "paper:tesla_ai_day", "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", "rel": "composes"}, + {"source": "validation:trace_modular_perception_pipeline_with_bev_fusion", "target": "paper:li2022bevformer", "rel": "validates"}, + + {"source": "paper:vit", "target": "validation:trace_neural_field_for_dynamic_driving_scene", "rel": "composes"}, + {"source": "paper:gaia1", "target": "validation:trace_neural_field_for_dynamic_driving_scene", "rel": "composes"}, + {"source": "paper:drivedreamer", "target": "validation:trace_neural_field_for_dynamic_driving_scene", "rel": "composes"}, + {"source": "paper:world_models", "target": "validation:trace_neural_field_for_dynamic_driving_scene", "rel": "composes"}, + {"source": "move:latent_imagination_rollout", "target": "validation:trace_neural_field_for_dynamic_driving_scene", "rel": "composes"}, + {"source": "insight:long_tail_solved_by_synthesis_not_data_alone", "target": "validation:trace_neural_field_for_dynamic_driving_scene", "rel": "composes"}, + {"source": "insight:world_models_let_planning_be_done_in_imagination", "target": "validation:trace_neural_field_for_dynamic_driving_scene", "rel": "composes"}, + {"source": "validation:trace_neural_field_for_dynamic_driving_scene", "target": "paper:gaia1", "rel": "validates"}, + + {"source": "paper:vaswani2017", "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "rel": "composes"}, + {"source": "paper:gpt3", "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "rel": "composes"}, + {"source": "concept:transformer", "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "rel": "composes"}, + {"source": "concept:imitation_learning", "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "rel": "composes"}, + {"source": "move:tokenize_modalities", "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "rel": "composes"}, + {"source": "insight:tokenization_collapses_modality_gap", "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "rel": "composes"}, + {"source": "insight:imitation_data_compresses_unspecified_reward", "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "rel": "composes"}, + {"source": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", "target": "paper:diffuser", "rel": "validates"}, + + {"source": "concept:mdp", "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", "rel": "composes"}, + {"source": "concept:policy_gradient", "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", "rel": "composes"}, + {"source": "concept:actor_critic", "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", "rel": "composes"}, + {"source": "concept:ppo", "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", "rel": "composes"}, + {"source": "move:clipped_surrogate_objective", "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", "rel": "composes"}, + {"source": "insight:safety_constraints_via_lagrangian_dual", "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", "rel": "composes"}, + {"source": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", "target": "paper:schulman2017_ppo", "rel": "validates"}, + + {"source": "paper:tesla_ai_day", "target": "paradigm:modular_perception_to_planning_pipeline", "rel": "instantiates"}, + {"source": "paper:li2022bevformer", "target": "paradigm:modular_perception_to_planning_pipeline", "rel": "instantiates"}, + {"source": "paper:transfuser", "target": "paradigm:modular_perception_to_planning_pipeline", "rel": "instantiates"}, + {"source": "concept:bev", "target": "paradigm:modular_perception_to_planning_pipeline", "rel": "composes"}, + {"source": "move:lift_2d_to_3d", "target": "paradigm:modular_perception_to_planning_pipeline", "rel": "composes"}, + {"source": "insight:set_prediction_eliminates_postprocessing_heuristics", "target": "paradigm:modular_perception_to_planning_pipeline", "rel": "composes"}, + + {"source": "paper:2212.10156", "target": "paradigm:differentiable_end_to_end_imitation", "rel": "instantiates"}, + {"source": "paper:2210.14222", "target": "paradigm:differentiable_end_to_end_imitation", "rel": "instantiates"}, + {"source": "paper:vadv2", "target": "paradigm:differentiable_end_to_end_imitation", "rel": "instantiates"}, + {"source": "paper:transfuser", "target": "paradigm:differentiable_end_to_end_imitation", "rel": "instantiates"}, + {"source": "concept:imitation_learning", "target": "paradigm:differentiable_end_to_end_imitation", "rel": "composes"}, + {"source": "move:cross_attention_query", "target": "paradigm:differentiable_end_to_end_imitation", "rel": "composes"}, + {"source": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", "target": "paradigm:differentiable_end_to_end_imitation", "rel": "composes"}, + + {"source": "paper:world_models", "target": "paradigm:model_based_world_imagination_planning", "rel": "instantiates"}, + {"source": "paper:gaia1", "target": "paradigm:model_based_world_imagination_planning", "rel": "instantiates"}, + {"source": "paper:drivedreamer", "target": "paradigm:model_based_world_imagination_planning", "rel": "instantiates"}, + {"source": "move:latent_imagination_rollout", "target": "paradigm:model_based_world_imagination_planning", "rel": "composes"}, + {"source": "move:diffusion_denoise_sampling", "target": "paradigm:model_based_world_imagination_planning", "rel": "composes"}, + {"source": "insight:world_models_let_planning_be_done_in_imagination", "target": "paradigm:model_based_world_imagination_planning", "rel": "composes"}, + {"source": "insight:long_tail_solved_by_synthesis_not_data_alone", "target": "paradigm:model_based_world_imagination_planning", "rel": "composes"}, + + {"source": "paper:gpt3", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "instantiates"}, + {"source": "paper:llava", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "instantiates"}, + {"source": "paper:2402.12289", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "instantiates"}, + {"source": "paper:2311.10813", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "instantiates"}, + {"source": "paper:lingo2", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "instantiates"}, + {"source": "concept:vla", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "composes"}, + {"source": "concept:tool_use", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "composes"}, + {"source": "move:dual_system_fast_slow", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "composes"}, + {"source": "insight:dual_system_handles_latency_quality_tradeoff", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "composes"}, + {"source": "insight:in_context_learning_emerges_at_scale", "target": "paradigm:foundation_model_zero_shot_driving_agent", "rel": "composes"}, + + {"source": "concept:spiking_nn", "target": "paradigm:brain_inspired_event_sparse_compute", "rel": "composes"}, + {"source": "paper:2307.01694", "target": "paradigm:brain_inspired_event_sparse_compute", "rel": "instantiates"}, + {"source": "move:spike_event_compute", "target": "paradigm:brain_inspired_event_sparse_compute", "rel": "composes"}, + {"source": "insight:event_sparse_compute_matches_energy_budget", "target": "paradigm:brain_inspired_event_sparse_compute", "rel": "composes"}, + + {"source": "paper:2512.24426", "target": "paradigm:counterfactual_data_centric_safety", "rel": "instantiates"}, + {"source": "paper:gaia1", "target": "paradigm:counterfactual_data_centric_safety", "rel": "instantiates"}, + {"source": "paper:drivedreamer", "target": "paradigm:counterfactual_data_centric_safety", "rel": "instantiates"}, + {"source": "concept:counterfactual", "target": "paradigm:counterfactual_data_centric_safety", "rel": "composes"}, + {"source": "concept:meta_action", "target": "paradigm:counterfactual_data_centric_safety", "rel": "composes"}, + {"source": "move:counterfactual_replan", "target": "paradigm:counterfactual_data_centric_safety", "rel": "composes"}, + {"source": "insight:long_tail_solved_by_synthesis_not_data_alone", "target": "paradigm:counterfactual_data_centric_safety", "rel": "composes"}, + + {"source": "paper:gpt3", "target": "paradigm:knowledge_driven_reflective_agent", "rel": "instantiates"}, + {"source": "paper:2309.16292", "target": "paradigm:knowledge_driven_reflective_agent", "rel": "instantiates"}, + {"source": "paper:2311.10813", "target": "paradigm:knowledge_driven_reflective_agent", "rel": "instantiates"}, + {"source": "concept:cot", "target": "paradigm:knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "concept:tool_use", "target": "paradigm:knowledge_driven_reflective_agent", "rel": "composes"}, + {"source": "essay:bitter_lesson", "target": "paradigm:knowledge_driven_reflective_agent", "rel": "contrasts"}, + {"source": "insight:symbolic_intermediate_enables_interpretability_and_transfer", "target": "paradigm:knowledge_driven_reflective_agent", "rel": "composes"}, + + {"source": "paper:dinov2", "target": "paradigm:scaling_data_with_self_supervision", "rel": "instantiates"}, + {"source": "paper:2508.10104", "target": "paradigm:scaling_data_with_self_supervision", "rel": "instantiates"}, + {"source": "paper:sam", "target": "paradigm:scaling_data_with_self_supervision", "rel": "instantiates"}, + {"source": "concept:ssl", "target": "paradigm:scaling_data_with_self_supervision", "rel": "composes"}, + {"source": "move:masking_for_pretext", "target": "paradigm:scaling_data_with_self_supervision", "rel": "composes"}, + {"source": "move:contrastive_alignment", "target": "paradigm:scaling_data_with_self_supervision", "rel": "composes"}, + {"source": "insight:masked_prediction_yields_self_supervised_signal", "target": "paradigm:scaling_data_with_self_supervision", "rel": "composes"}, + {"source": "insight:foundation_pretraining_decouples_data_from_task", "target": "paradigm:scaling_data_with_self_supervision", "rel": "composes"}, + {"source": "insight:scaling_laws_predict_capability_emergence", "target": "paradigm:scaling_data_with_self_supervision", "rel": "composes"} + ] +} diff --git a/docs/data/generated/methodology_axis.json b/docs/data/generated/methodology_axis.json new file mode 100644 index 0000000..874c4a3 --- /dev/null +++ b/docs/data/generated/methodology_axis.json @@ -0,0 +1,316 @@ +{ + "$comment": "Methodology / Evaluation / Safety / Data-engineering / Brain-inspired-efficient-computing axis. Append-only extension to docs/data/graph.json. Edges may reference existing IDs in graph.json.", + "axis": "methodology_evaluation_safety_data_brain_efficient", + "nodes": [ + {"id": "paper:nuplan", "label": "nuPlan", "label_zh": "nuPlan(首个大规模闭环规划基准)", "kind": "paper", "tier": "A", "topic": "evaluation_benchmark", "phase": "core", "year": 2021, "card": "paper_nuplan.md", "summary_zh": "nuPlan 是 Motional 提出的首个大规模闭环自动驾驶规划基准,提供 1500 小时人类驾驶数据并定义了基于交通规则、舒适度与进展度的混合评分体系,使规划器评估首次从开环位移误差升级为闭环行为指标。"}, + {"id": "paper:waymo_motion", "label": "Waymo Open Motion", "label_zh": "Waymo Open Motion 数据集", "kind": "paper", "tier": "A", "topic": "evaluation_benchmark", "phase": "core", "year": 2021, "card": "paper_waymo_motion.md", "summary_zh": "Waymo Open Motion Dataset 提供约十万段高质量交互场景,专门为行为预测与多智能体规划设计,配套的预测排行榜推动了 MultiPath、Wayformer 等结构的迭代。"}, + {"id": "paper:argoverse2", "label": "Argoverse 2", "label_zh": "Argoverse 2(运动预测与高精地图基准)", "kind": "paper", "tier": "A", "topic": "evaluation_benchmark", "phase": "core", "year": 2022, "card": "paper_argoverse2.md", "summary_zh": "Argoverse 2 由 Argo AI 发布,包含 250000 段六类城市预测场景、激光雷达数据集与高精地图,并提出长时序多智能体预测指标 brier-minFDE,是预测与拓扑感知研究的事实标准。"}, + {"id": "paper:navsim", "label": "NAVSIM", "label_zh": "NAVSIM(非反应式闭环代理评测)", "kind": "paper", "tier": "A", "topic": "evaluation_benchmark", "phase": "frontier", "year": 2024, "card": "paper_navsim.md", "summary_zh": "NAVSIM 通过在 nuPlan 与 OpenScene 之上构建非反应式短时滚动评估,提出 PDM Score 这一可与真实安全代理强相关的离线指标,弥合了完全开环与昂贵闭环之间的鸿沟。"}, + {"id": "paper:bench2drive", "label": "Bench2Drive", "label_zh": "Bench2Drive(端到端闭环 CARLA 基准)", "kind": "paper", "tier": "A", "topic": "evaluation_benchmark", "phase": "frontier", "year": 2024, "card": "paper_bench2drive.md", "summary_zh": "Bench2Drive 在 CARLA Leaderboard 2.0 之上提供 44 个能力分桶与统一训练协议,使 UniAD、VAD 等端到端模型可在同一闭环环境下被公平比较,揭示了离线 L2 与闭环成功率之间的弱相关性。"}, + {"id": "paper:carla_lb2", "label": "CARLA Leaderboard 2.0", "label_zh": "CARLA Leaderboard 2.0", "kind": "paper", "tier": "B", "topic": "evaluation_benchmark", "phase": "core", "year": 2023, "card": "paper_carla_lb2.md", "summary_zh": "CARLA Leaderboard 2.0 引入更长路线、更密集对抗事件与新评分公式,把驾驶成功率与违规分数解耦,迫使端到端模型在长尾事件上具备真正的恢复与避撞能力。"}, + {"id": "paper:highway_env", "label": "HighwayEnv", "label_zh": "HighwayEnv(轻量决策仿真环境)", "kind": "paper", "tier": "B", "topic": "simulator", "phase": "prereq", "year": 2018, "card": "paper_highway_env.md", "summary_zh": "HighwayEnv 是基于 OpenAI Gym 的轻量级高速公路决策仿真器,支持 IDM、MOBIL 与多智能体设置,是验证 DQN/PPO 等算法在变道与汇入任务上的事实平台。"}, + {"id": "paper:metadrive", "label": "MetaDrive", "label_zh": "MetaDrive(程序生成驾驶模拟器)", "kind": "paper", "tier": "B", "topic": "simulator", "phase": "core", "year": 2021, "card": "paper_metadrive.md", "summary_zh": "MetaDrive 通过程序化生成无穷场景与可重复随机种子,专门用于研究强化学习的泛化与安全性,并提供与 SUMO、Waymo Open Motion 重放兼容的驾驶接口。"}, + {"id": "paper:smarts", "label": "SMARTS", "label_zh": "SMARTS(多智能体驾驶仿真)", "kind": "paper", "tier": "B", "topic": "simulator", "phase": "core", "year": 2020, "card": "paper_smarts.md", "summary_zh": "华为诺亚提出的 SMARTS 平台聚焦多智能体交互式驾驶,支持灵活的社交车辆行为模型与课程学习接口,被广泛用于多智能体 RL 与博弈式规划研究。"}, + {"id": "paper:commonroad", "label": "CommonRoad", "label_zh": "CommonRoad(形式化运动规划基准)", "kind": "paper", "tier": "B", "topic": "simulator", "phase": "core", "year": 2017, "card": "paper_commonroad.md", "summary_zh": "CommonRoad 由 TU München 维护,提供可被形式化验证的运动规划场景与代价函数语言,是 MPC、采样规划与神经规划共同的基准。"}, + {"id": "paper:lyft_l5", "label": "Lyft L5 Prediction", "label_zh": "Lyft Level-5 预测数据集", "kind": "paper", "tier": "B", "topic": "dataset", "phase": "core", "year": 2020, "card": "paper_lyft_l5.md", "summary_zh": "Lyft Level-5 数据集包含 1000 小时车队真实驾驶日志与对应栅格化语义地图,是第一个把行为预测当作大规模监督学习问题来求解的工业数据集。"}, + {"id": "paper:pandaset", "label": "PandaSet", "label_zh": "PandaSet(多模态感知数据集)", "kind": "paper", "tier": "B", "topic": "dataset", "phase": "core", "year": 2020, "card": "paper_pandaset.md", "summary_zh": "Hesai 与 Scale AI 联合发布的 PandaSet 提供机械式与固态激光雷达双路数据及精细 3D 语义标签,是研究激光雷达硬件差异对感知影响的少见公开数据集。"}, + {"id": "paper:apolloscape", "label": "ApolloScape", "label_zh": "ApolloScape(百度多任务驾驶数据集)", "kind": "paper", "tier": "B", "topic": "dataset", "phase": "core", "year": 2018, "card": "paper_apolloscape.md", "summary_zh": "百度 ApolloScape 涵盖语义分割、车道线、轨迹与立体匹配等多任务大规模标注,是国内最早开放的工业级自动驾驶数据集之一。"}, + {"id": "paper:bdd100k", "label": "BDD100K", "label_zh": "BDD100K(伯克利多样化驾驶视频)", "kind": "paper", "tier": "B", "topic": "dataset", "phase": "core", "year": 2018, "card": "paper_bdd100k.md", "summary_zh": "BDD100K 提供十万段一分钟驾驶视频与十类多任务标签,覆盖夜间、雨雪、城市等多种工况,使研究者能在感知模型中明确量化领域偏移。"}, + {"id": "paper:womd_pred", "label": "WOMD prediction benchmark", "label_zh": "WOMD 预测基准", "kind": "paper", "tier": "B", "topic": "evaluation_benchmark", "phase": "core", "year": 2021, "card": "paper_womd_pred.md", "summary_zh": "Waymo Open Motion Dataset 的预测排行榜定义了 minADE、minFDE、Miss Rate 与 mAP 等核心指标,并对长尾交互场景采用难度加权评分。"}, + {"id": "paper:interaction_dataset", "label": "INTERACTION Dataset", "label_zh": "INTERACTION 数据集", "kind": "paper", "tier": "B", "topic": "evaluation_benchmark", "phase": "core", "year": 2019, "card": "paper_interaction_dataset.md", "summary_zh": "INTERACTION 数据集聚焦无信号路口、并道与环岛等高交互性场景,由全球多个研究组共同采集,是博弈论建模与社会力学研究的标准平台。"}, + {"id": "paper:shift_dataset", "label": "SHIFT", "label_zh": "SHIFT(连续域偏移合成数据集)", "kind": "paper", "tier": "B", "topic": "dataset", "phase": "frontier", "year": 2022, "card": "paper_shift_dataset.md", "summary_zh": "SHIFT 在 CARLA 中合成连续变化的天气、时间、密度等参数,使研究者能精确量化分布偏移对感知模型的影响,是研究持续学习与领域适配的纯净基准。"}, + {"id": "paper:v2x_sim", "label": "V2X-Sim", "label_zh": "V2X-Sim(车路云协同仿真数据集)", "kind": "paper", "tier": "B", "topic": "dataset", "phase": "frontier", "year": 2022, "card": "paper_v2x_sim.md", "summary_zh": "V2X-Sim 在 CARLA 中合成多车、路侧单元的协同感知数据,配套点云、图像与通信延迟模型,是协同感知与协同规划研究的早期标杆。"}, + {"id": "paper:flashattention", "label": "FlashAttention", "label_zh": "FlashAttention(IO 感知精确注意力)", "kind": "paper", "tier": "A", "topic": "efficient_computing", "phase": "core", "year": 2022, "card": "paper_flashattention.md", "summary_zh": "FlashAttention 通过把注意力切片为可放入 GPU SRAM 的 tile,并将 softmax 与矩阵乘融合,实现了精确注意力的 2-4 倍加速,是长序列 transformer 实用化的关键工程突破。"}, + {"id": "paper:performer", "label": "Performer", "label_zh": "Performer(核近似线性注意力)", "kind": "paper", "tier": "B", "topic": "efficient_computing", "phase": "core", "year": 2020, "card": "paper_performer.md", "summary_zh": "Performer 用 FAVOR+ 随机特征近似 softmax 核,把注意力复杂度降至 O(N),为长视频与高频感知序列提供了可扩展的替代方案。"}, + {"id": "paper:linear_attention", "label": "Linear Attention", "label_zh": "线性注意力(Katharopoulos 2020)", "kind": "paper", "tier": "B", "topic": "efficient_computing", "phase": "core", "year": 2020, "card": "paper_linear_attention.md", "summary_zh": "Katharopoulos 等用核函数把注意力重新表达为线性递归形式,使其在自回归推理时与 RNN 等价,是当前许多高效驾驶序列模型的数学起点。"}, + {"id": "paper:gptq", "label": "GPTQ", "label_zh": "GPTQ(后训练 4bit 量化)", "kind": "paper", "tier": "B", "topic": "efficient_computing", "phase": "frontier", "year": 2022, "card": "paper_gptq.md", "summary_zh": "GPTQ 利用近似二阶 Hessian 信息一次性完成大模型 4bit 量化,几乎不损失精度,使大型 VLM 能在车规算力下离线部署成为可能。"}, + {"id": "paper:awq", "label": "AWQ", "label_zh": "AWQ(激活感知权重量化)", "kind": "paper", "tier": "B", "topic": "efficient_computing", "phase": "frontier", "year": 2023, "card": "paper_awq.md", "summary_zh": "AWQ 根据通道激活分布对权重进行尺度变换后再量化,避免显著通道被压扁,是车端 LLM/VLM 推理常用的工业级量化方案。"}, + {"id": "paper:distill_vlm", "label": "DistillVLM", "label_zh": "DistillVLM(VLM 蒸馏综述)", "kind": "paper", "tier": "B", "topic": "efficient_computing", "phase": "frontier", "year": 2023, "card": "paper_distill_vlm.md", "summary_zh": "DistillVLM 系列工作把大尺寸视觉语言模型的推理能力蒸馏到 1-3B 量级专家模型,是车端实时驾驶解释与意图预测的主要落地路径。"}, + {"id": "paper:loihi2", "label": "Intel Loihi 2", "label_zh": "Intel Loihi 2 神经形态芯片", "kind": "paper", "tier": "A", "topic": "neuromorphic_hardware", "phase": "frontier", "year": 2021, "card": "paper_loihi2.md", "summary_zh": "Intel Loihi 2 是异步事件驱动神经形态芯片,单芯片支持百万级可编程脉冲神经元与三因子塑性,是 SNN 算法在端侧实测能耗的首选硬件。"}, + {"id": "paper:truenorth", "label": "IBM TrueNorth", "label_zh": "IBM TrueNorth 神经形态芯片", "kind": "paper", "tier": "B", "topic": "neuromorphic_hardware", "phase": "core", "year": 2014, "card": "paper_truenorth.md", "summary_zh": "TrueNorth 是 IBM 早期纯数字事件驱动神经形态芯片,单芯片 100 万神经元、仅 70 mW,证明了大规模 SNN 在嵌入式场景的能效优势。"}, + {"id": "paper:tianjic", "label": "Tianjic", "label_zh": "天机芯(清华混合范式芯片)", "kind": "paper", "tier": "B", "topic": "neuromorphic_hardware", "phase": "core", "year": 2019, "card": "paper_tianjic.md", "summary_zh": "清华施路平团队的天机芯把 ANN 与 SNN 在同一硅片上统一调度,曾驱动自动驾驶自行车展示,是混合范式神经形态计算的代表作。"}, + {"id": "paper:grai", "label": "GrAI Matter", "label_zh": "GrAI Matter VIP 系列芯片", "kind": "paper", "tier": "B", "topic": "neuromorphic_hardware", "phase": "frontier", "year": 2022, "card": "paper_grai.md", "summary_zh": "GrAI Matter 的 NeuronFlow VIP 系列把事件驱动稀疏计算与传统视觉流水线结合,主打超低延迟车载感知 ASIC。"}, + {"id": "paper:dvs_event_camera", "label": "DVS event camera", "label_zh": "DVS 事件相机", "kind": "paper", "tier": "A", "topic": "neuromorphic_hardware", "phase": "core", "year": 2008, "card": "paper_dvs_event_camera.md", "summary_zh": "DVS 事件相机每像素异步输出亮度变化事件,具有微秒延迟与 120dB 动态范围,是高速运动与极端光照下感知的天然传感器。"}, + {"id": "paper:gs_for_ad", "label": "3DGS for AD", "label_zh": "3D 高斯泼溅用于自动驾驶(StreetGaussians 等)", "kind": "paper", "tier": "A", "topic": "data_engine", "phase": "frontier", "year": 2024, "card": "paper_gs_for_ad.md", "summary_zh": "StreetGaussians、DrivingGaussian 等工作把 3D 高斯泼溅扩展到带运动物体的街景重建,使闭环仿真器能以照片级真实度回放并扰动真实日志。"}, + {"id": "paper:lidar_cam_calib", "label": "Continuous-time LiDAR-Camera calibration", "label_zh": "连续时间 LiDAR-相机时空同步标定", "kind": "paper", "tier": "B", "topic": "data_engine", "phase": "core", "year": 2020, "card": "paper_lidar_cam_calib.md", "summary_zh": "连续时间样条标定把 LiDAR、相机、IMU 的时空外参联合优化为时间函数,能在运行中校正同步误差与振动漂移,是真实车队的工程基线。"}, + {"id": "paper:iso26262", "label": "ISO 26262", "label_zh": "ISO 26262 道路车辆功能安全", "kind": "paper", "tier": "A", "topic": "safety_standard", "phase": "core", "year": 2011, "card": "paper_iso26262.md", "summary_zh": "ISO 26262 定义了汽车电子电气系统的功能安全生命周期与 ASIL 等级,是任何量产自动驾驶硬件与软件必须满足的功能安全基础。"}, + {"id": "paper:sotif_21448", "label": "ISO 21448 SOTIF", "label_zh": "ISO 21448 预期功能安全 SOTIF", "kind": "paper", "tier": "A", "topic": "safety_standard", "phase": "core", "year": 2019, "card": "paper_sotif_21448.md", "summary_zh": "ISO 21448 SOTIF 把传统功能安全无法覆盖的功能不足与可预见误用纳入风险接受准则,是为机器学习驱动的自动驾驶量身定制的国际标准。"}, + {"id": "paper:ul4600", "label": "ANSI/UL 4600", "label_zh": "UL 4600 自动驾驶安全论证标准", "kind": "paper", "tier": "B", "topic": "safety_standard", "phase": "frontier", "year": 2020, "card": "paper_ul4600.md", "summary_zh": "UL 4600 提出以结构化安全论证(safety case)作为自动驾驶安全证明的核心,要求基于声明—证据—假设的三元组进行可审计推理。"}, + {"id": "paper:tesla_autolabel", "label": "Tesla Auto-Labeling", "label_zh": "Tesla 自动标注与数据引擎", "kind": "paper", "tier": "A", "topic": "data_engine", "phase": "core", "year": 2022, "card": "paper_tesla_autolabel.md", "summary_zh": "Tesla AI Day 披露的离线 4D 重建自动标注流水线,把车队回传片段离线超算重投影为高精轨迹标签,是数据引擎闭环的代表性工业实现。"}, + {"id": "paper:waymo_scenario_mining", "label": "Waymo Scenario Mining", "label_zh": "Waymo 场景挖掘与长尾发现", "kind": "paper", "tier": "B", "topic": "data_engine", "phase": "core", "year": 2022, "card": "paper_waymo_scenario_mining.md", "summary_zh": "Waymo 通过结构化查询语言与轨迹嵌入检索从车队日志中主动挖掘罕见交互场景,被用于针对性回归测试与训练样本扩充。"}, + + {"id": "move:design_closed_loop_metric_correlated_with_real_world_safety", "label": "Design closed-loop metric correlated with real safety", "label_zh": "设计与真实世界安全率强相关的闭环指标", "kind": "move", "tier": "move", "topic": "evaluation_benchmark", "phase": "frontier", "year": 2024, "card": "move_design_closed_loop_metric.md", "summary_zh": "围绕真实车队事故率与脱离率反推闭环评测指标,把碰撞、近碰撞、舒适度、规则违规与进展度按事故贡献加权融合,使离线评测对决策有真正的判别力。"}, + {"id": "move:augment_dataset_via_offline_scenario_perturbation", "label": "Perturb logs offline to augment dataset", "label_zh": "用离线扰动驾驶日志扩充训练数据", "kind": "move", "tier": "move", "topic": "data_engine", "phase": "core", "year": 2023, "card": "move_perturb_logs.md", "summary_zh": "在真实日志重建的 3DGS 或 NeRF 场景中对自车与他车的轨迹、速度、外观进行可控扰动,用极小的标注成本生成具有反事实意义的训练样本。"}, + {"id": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", "label": "Active learning over hardest frames", "label_zh": "用主动学习挑选最难未标注帧", "kind": "move", "tier": "move", "topic": "data_engine", "phase": "core", "year": 2022, "card": "move_active_learning_hard_frames.md", "summary_zh": "在车队回传日志中按模型不确定性、罕见性嵌入与下游闭环失败贡献多准则打分,仅把得分最高的帧送人工标注或仿真扩增,使标注预算回报最大化。"}, + {"id": "move:distill_large_VLM_into_small_realtime_specialist", "label": "Distill large VLM into realtime specialist", "label_zh": "把大 VLM 蒸馏为车端实时小模型", "kind": "move", "tier": "move", "topic": "efficient_computing", "phase": "frontier", "year": 2024, "card": "move_distill_vlm_small.md", "summary_zh": "用大规模 VLM 离线生成轨迹解释、风险标签与高层动作,再以教师-学生范式蒸馏到 1-3B 参数的车端模型,让大模型能力以可承担成本上车。"}, + {"id": "move:quantize_attention_to_int8_with_calibration", "label": "INT8 attention quantization", "label_zh": "用校准把注意力量化到 INT8", "kind": "move", "tier": "move", "topic": "efficient_computing", "phase": "core", "year": 2023, "card": "move_int8_attention.md", "summary_zh": "针对注意力中 softmax 与残差敏感性,采用每通道激活校准与 KV cache 异步量化策略,把 transformer 推理压到 INT8 而不损失驾驶决策质量。"}, + {"id": "move:replace_dense_attention_with_sparse_event_driven_attention", "label": "Sparse event-driven attention", "label_zh": "用事件驱动稀疏注意力替换稠密注意力", "kind": "move", "tier": "move", "topic": "efficient_computing", "phase": "frontier", "year": 2024, "card": "move_sparse_event_attention.md", "summary_zh": "把稠密注意力替换为只在脉冲发生时启用 QK 计算的事件驱动注意力,使能耗与场景稀疏度成线性关系,是 SNN 与神经形态硬件协同的核心算子。"}, + {"id": "move:use_event_camera_microsecond_latency_for_emergency_braking", "label": "Event camera microsecond braking", "label_zh": "用事件相机微秒延迟触发紧急制动", "kind": "move", "tier": "move", "topic": "neuromorphic_hardware", "phase": "frontier", "year": 2023, "card": "move_event_camera_braking.md", "summary_zh": "在传统帧相机感知管线之外并联事件相机的低延迟运动检测模块,使切入与急刹场景的端到端反应时延从百毫秒级压缩到十毫秒级。"}, + {"id": "move:implement_spiking_neuron_with_surrogate_gradient_for_backprop", "label": "Surrogate gradient for SNN backprop", "label_zh": "用代理梯度实现脉冲神经元的反向传播", "kind": "move", "tier": "move", "topic": "neuromorphic_hardware", "phase": "core", "year": 2018, "card": "move_surrogate_gradient.md", "summary_zh": "用平滑的代理函数替代脉冲激活的不可微阶跃,使脉冲神经元能在标准 PyTorch 反向传播框架中训练,是 SNN 走出实验室的关键技巧。"}, + {"id": "move:co_design_silicon_with_algorithm_for_minimum_energy", "label": "Co-design silicon and algorithm", "label_zh": "把芯片与算法联合设计以最小化能耗", "kind": "move", "tier": "move", "topic": "neuromorphic_hardware", "phase": "frontier", "year": 2023, "card": "move_silicon_algo_codesign.md", "summary_zh": "把数据通路、片上存储与算法稀疏模式作为一个优化对象,让网络结构、量化精度、调度顺序与硬件 NoC 拓扑共同搜索能耗-延迟前沿。"}, + {"id": "move:cache_KV_state_across_frames_to_amortize_attention_cost", "label": "Cross-frame KV cache", "label_zh": "跨帧缓存 KV 以摊销注意力开销", "kind": "move", "tier": "move", "topic": "efficient_computing", "phase": "frontier", "year": 2024, "card": "move_cross_frame_kv_cache.md", "summary_zh": "把视觉 token 的 K/V 缓存按时间窗滚动复用并只增量更新发生显著变化的部分,使长时序 BEV transformer 推理代价从 O(T) 降至接近 O(1)。"}, + {"id": "move:tile_attention_to_fit_SRAM_for_speedup", "label": "Tile attention into SRAM (FlashAttention)", "label_zh": "把注意力切块以适配 SRAM 获得加速", "kind": "move", "tier": "move", "topic": "efficient_computing", "phase": "core", "year": 2022, "card": "move_tile_attention_sram.md", "summary_zh": "把 Q、K、V 沿序列维切成可放入 GPU SRAM 的小块,并通过 online softmax 在小块间累加结果,避免对 HBM 的反复读写,是精确注意力加速的核心技巧。"}, + {"id": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", "label": "Linear kernel attention for long sequence", "label_zh": "用线性核注意力处理长序列", "kind": "move", "tier": "move", "topic": "efficient_computing", "phase": "core", "year": 2022, "card": "move_linear_kernel_attention.md", "summary_zh": "用核函数把 softmax 注意力近似为可交换求和顺序的形式,从而把复杂度降至 O(N),使长视频与连续驾驶日志的训练可线性扩展。"}, + {"id": "move:share_LiDAR_camera_calibration_via_continuous_time_optimization", "label": "Continuous-time multi-sensor calibration", "label_zh": "用连续时间优化联合标定 LiDAR 与相机", "kind": "move", "tier": "move", "topic": "data_engine", "phase": "core", "year": 2021, "card": "move_continuous_time_calibration.md", "summary_zh": "把多传感器外参与时间偏移建模为 B 样条曲线并与 IMU 预积分共同优化,使在线运行中也能持续修正同步漂移与温度变形。"}, + {"id": "move:auto_label_with_offline_model_then_human_in_loop_validate", "label": "Auto-label + human-in-loop validate", "label_zh": "用离线大模型自动标注再人工抽样验证", "kind": "move", "tier": "move", "topic": "data_engine", "phase": "core", "year": 2022, "card": "move_auto_label_hitl.md", "summary_zh": "先用大算力离线模型生成 4D 轨迹与语义标签,再让标注员仅审核置信度低或下游影响大的样本,把人工成本从线性降到对数级。"}, + {"id": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", "label": "Safety constraints in signal temporal logic", "label_zh": "用信号时序逻辑表达并验证安全约束", "kind": "move", "tier": "move", "topic": "safety_standard", "phase": "frontier", "year": 2023, "card": "move_stl_safety_constraint.md", "summary_zh": "把保持车距、限速、信号灯遵守等行为约束写成可解释、可量化的信号时序逻辑(STL)公式,运行中既可作监控也可纳入轨迹优化的硬性约束。"}, + {"id": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", "label": "Add shielding layer over policy", "label_zh": "在策略输出端加屏蔽层以拒绝不安全动作", "kind": "move", "tier": "move", "topic": "safety_standard", "phase": "frontier", "year": 2023, "card": "move_shielding_layer.md", "summary_zh": "在神经规划器后串接一个由可达性分析或控制屏障函数定义的屏蔽层,对违反安全包络的动作进行投影或替换为后备策略,使学习模型可在安全约束下逐步上车。"}, + {"id": "move:treat_corner_case_as_OOD_detection_then_route_to_human", "label": "Treat corner case as OOD then escalate", "label_zh": "把长尾事件视为 OOD 检测并升级到人类", "kind": "move", "tier": "move", "topic": "safety_standard", "phase": "frontier", "year": 2024, "card": "move_corner_case_ood.md", "summary_zh": "用密度估计、能量函数或 VLM 描述匹配等手段检测 OOD 输入,触发后切换到保守策略、远程驾驶或安全停车,避免模型在认知盲区强行决策。"}, + {"id": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", "label": "Replay sim with perturbed conditions", "label_zh": "用扰动初始条件的回放仿真评估鲁棒性", "kind": "move", "tier": "move", "topic": "evaluation_benchmark", "phase": "core", "year": 2023, "card": "move_replay_perturbation.md", "summary_zh": "在真实日志重建场景中对初始位姿、速度、他车意图施加噪声扰动,统计性地评估规划器在邻域内的稳定性,是闭环回归测试的事实方法。"}, + {"id": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", "label": "Track offline-vs-closed-loop correlation", "label_zh": "持续追踪离线与闭环指标的相关性以筛选模型", "kind": "move", "tier": "move", "topic": "evaluation_benchmark", "phase": "frontier", "year": 2024, "card": "move_metric_correlation_tracking.md", "summary_zh": "定期把候选模型同时在离线集合与昂贵闭环仿真上评分,对比 Spearman 相关性以筛选出真正预测闭环性能的指标,避免开发团队被偏好性指标误导。"}, + {"id": "move:use_difficulty_aware_curriculum_to_accelerate_RL", "label": "Difficulty-aware curriculum for driving RL", "label_zh": "用难度感知课程加速驾驶强化学习", "kind": "move", "tier": "move", "topic": "evaluation_benchmark", "phase": "core", "year": 2022, "card": "move_difficulty_curriculum.md", "summary_zh": "按场景难度排序逐步引入更密集的交互与异常事件,避免策略在早期被淹没在长尾,使收敛速度与最终成功率显著优于均匀采样。"}, + {"id": "move:add_explanation_head_to_promote_interpretability", "label": "Add explanation head to planner", "label_zh": "为规划器增加解释头以提升可解释性", "kind": "move", "tier": "move", "topic": "safety_standard", "phase": "frontier", "year": 2024, "card": "move_explanation_head.md", "summary_zh": "在端到端模型的轨迹头之外额外训练自然语言或符号化的解释头,使每个决策都能追溯到关键场景特征,是面向监管与事故复盘的必备工程实践。"}, + {"id": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", "label": "Deep ensemble / evidential uncertainty", "label_zh": "用深度集成或证据层做不确定性量化", "kind": "move", "tier": "move", "topic": "safety_standard", "phase": "core", "year": 2020, "card": "move_uncertainty_quantification.md", "summary_zh": "用深度集成、MC-Dropout 或证据深度学习层估计预测分布的偏差与方差,将其作为安全屏蔽与人机交接的触发信号,是模型化不确定性向决策回路传播的标准做法。"}, + {"id": "move:perform_neural_architecture_search_with_latency_constraint", "label": "NAS with latency constraint", "label_zh": "在延迟约束下做神经架构搜索", "kind": "move", "tier": "move", "topic": "efficient_computing", "phase": "core", "year": 2020, "card": "move_nas_latency.md", "summary_zh": "把车端 ECU 上的真实延迟、能耗与可调度性作为硬约束加入 NAS 目标函数,搜索能在 30fps 实时预算下保持精度的 BEV/规划网络。"}, + {"id": "move:formalize_safety_case_with_claim_evidence_assumption", "label": "Safety case with claim-evidence-assumption", "label_zh": "用声明-证据-假设结构形式化安全论证", "kind": "move", "tier": "move", "topic": "safety_standard", "phase": "frontier", "year": 2022, "card": "move_safety_case.md", "summary_zh": "按 UL 4600 的安全论证语言把每个安全主张分解为子声明、证据与显式假设,使外部审计可以逐条核查证据链与残余风险,是可审计 AD 落地的核心工程范式。"}, + {"id": "move:replay_buffer_prioritize_safety_critical_transitions", "label": "Prioritize safety-critical transitions", "label_zh": "在回放缓冲中优先采样安全关键转移", "kind": "move", "tier": "move", "topic": "evaluation_benchmark", "phase": "core", "year": 2022, "card": "move_priority_safety_replay.md", "summary_zh": "对涉及碰撞、近碰撞与急刹的转移赋予更高优先级,使 RL 策略不被海量的平凡跟车样本稀释,在安全长尾上的学习效率成倍提升。"}, + {"id": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", "label": "Continual learning with rehearsal buffer", "label_zh": "用回演缓冲做持续学习对抗遗忘", "kind": "move", "tier": "move", "topic": "data_engine", "phase": "frontier", "year": 2023, "card": "move_continual_learning_rehearsal.md", "summary_zh": "在模型每次 OTA 更新前重新混入过去版本的代表性样本与硬例,防止新数据驱动的微调让旧场景能力遗忘,是车队规模化运营的必备机制。"}, + + {"id": "problem:offline_metric_does_not_predict_closed_loop_performance", "label": "Offline metric ≠ closed-loop perf", "label_zh": "离线指标不能预测闭环性能", "kind": "problem", "tier": "problem", "topic": "evaluation_benchmark", "phase": "frontier", "year": 2023, "card": "problem_offline_vs_closed_loop.md", "summary_zh": "实证表明位移误差等开环监督指标与闭环成功率相关性很弱,导致模型选择与论文结论可能与真实驾驶安全脱节。"}, + {"id": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented", "label": "Rare safety events dominate risk", "label_zh": "罕见安全事件主导真实风险却严重欠采样", "kind": "problem", "tier": "problem", "topic": "evaluation_benchmark", "phase": "core", "year": 2022, "card": "problem_rare_safety_events.md", "summary_zh": "真实事故集中在长尾交互与极端工况,但训练与评测集中绝大多数样本为低风险跟车,造成模型在最关键场景上的统计置信度不足。"}, + {"id": "problem:energy_budget_too_small_for_full_transformer_at_30fps", "label": "Energy budget too small for full transformer", "label_zh": "车端能耗预算无法支撑 30fps 全注意力 transformer", "kind": "problem", "tier": "problem", "topic": "efficient_computing", "phase": "frontier", "year": 2024, "card": "problem_energy_budget.md", "summary_zh": "量产车 ECU 通常只有 30-80W 可用于 AI 推理,而原始多模态 transformer 在 30fps 闭环下功耗远超此范围,必须依赖稀疏化、量化与缓存技巧。"}, + {"id": "problem:sensor_calibration_drift_over_vehicle_lifetime", "label": "Sensor calibration drift over lifetime", "label_zh": "传感器标定在整车寿命周期内漂移", "kind": "problem", "tier": "problem", "topic": "data_engine", "phase": "core", "year": 2021, "card": "problem_calibration_drift.md", "summary_zh": "振动、温度循环与碰撞会让 LiDAR-相机-IMU 的相对外参缓慢漂移,若不持续在线标定,感知与重建质量会在数月内显著退化。"}, + {"id": "problem:label_noise_for_3d_object_categories", "label": "Label noise in 3D object categories", "label_zh": "3D 目标类别标注噪声", "kind": "problem", "tier": "problem", "topic": "data_engine", "phase": "core", "year": 2020, "card": "problem_label_noise_3d.md", "summary_zh": "夜间、远距离与遮挡场景下人工标注的 3D 边界框与类别一致性不足 90%,直接影响检测器的可比较性与可验证性。"}, + {"id": "problem:verification_of_neural_network_safety_properties_at_scale", "label": "Scalable NN safety verification", "label_zh": "大规模神经网络安全属性的形式化验证", "kind": "problem", "tier": "problem", "topic": "safety_standard", "phase": "frontier", "year": 2022, "card": "problem_nn_verification.md", "summary_zh": "在百兆参数级网络上完成对全部安全属性的形式化证明仍超出现有 SMT 求解器与符号传播工具的可扩展范围,是 AD 形式化安全的关键瓶颈。"}, + {"id": "problem:realistic_other_agent_behavior_in_simulator", "label": "Realistic other-agent behavior in sim", "label_zh": "仿真器中其他智能体的真实行为", "kind": "problem", "tier": "problem", "topic": "simulator", "phase": "core", "year": 2021, "card": "problem_agent_realism.md", "summary_zh": "现有仿真器多采用 IDM 等手工模型驱动他车,难以再现真实人类驾驶员的礼让、博弈与误判,使闭环训练存在难以察觉的 sim-to-real 偏差。"}, + {"id": "problem:catastrophic_forgetting_under_continual_learning", "label": "Catastrophic forgetting under continual update", "label_zh": "持续学习下的灾难性遗忘", "kind": "problem", "tier": "problem", "topic": "data_engine", "phase": "frontier", "year": 2022, "card": "problem_catastrophic_forgetting.md", "summary_zh": "在 OTA 不断引入新数据微调模型时,老旧场景与少数群体场景的能力会被覆盖性遗忘,造成已通过验收的功能在新版本中悄悄回退。"}, + {"id": "problem:auditability_of_decisions_for_regulatory_compliance", "label": "Decision auditability for regulators", "label_zh": "决策对监管机构的可审计性", "kind": "problem", "tier": "problem", "topic": "safety_standard", "phase": "frontier", "year": 2023, "card": "problem_auditability.md", "summary_zh": "黑盒端到端模型在事故复盘与监管问询中难以提供逐步可追溯的因果链,使制造商面临严苛的合规与法律风险。"}, + {"id": "problem:simulator_visual_gap_breaks_perception_models", "label": "Simulator visual gap breaks perception", "label_zh": "仿真视觉差异破坏感知模型迁移", "kind": "problem", "tier": "problem", "topic": "simulator", "phase": "core", "year": 2022, "card": "problem_sim_visual_gap.md", "summary_zh": "传统游戏引擎渲染与真实相机噪声、镜头光晕与全局光照差距过大,使在仿真上预训练的感知模型在真车上性能显著下降。"}, + + {"id": "insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners", "label": "Closed-loop is the only ground truth", "label_zh": "闭环评测是规划器唯一的真值", "kind": "insight", "tier": "insight", "topic": "evaluation_benchmark", "phase": "frontier", "year": 2024, "card": "insight_closed_loop_ground_truth.md", "summary_zh": "由于规划器的每一步输出都会改变后续观测,离线监督指标无法捕捉滚动决策的反馈结构,只有闭环(或非反应式滚动)评测才能反映其真实驾驶能力。"}, + {"id": "insight:safety_emerges_from_layered_constraints_not_single_objective", "label": "Safety as layered constraints", "label_zh": "安全来自分层约束而非单一目标", "kind": "insight", "tier": "insight", "topic": "safety_standard", "phase": "frontier", "year": 2023, "card": "insight_layered_safety.md", "summary_zh": "鲁棒的自动驾驶安全不可能通过单一损失函数达成,而是由神经规划、屏蔽层、应急 MPC、监管软件与电子电气冗余共同构成多层约束的涌现性质。"}, + {"id": "insight:event_driven_computation_matches_natural_sparsity_of_driving_scene", "label": "Event computation matches scene sparsity", "label_zh": "事件驱动计算契合驾驶场景的天然稀疏性", "kind": "insight", "tier": "insight", "topic": "neuromorphic_hardware", "phase": "frontier", "year": 2023, "card": "insight_event_driven_sparsity.md", "summary_zh": "驾驶场景中大多数像素与时间步信息冗余度极高,事件驱动 SNN 与神经形态硬件以稀疏脉冲表征自然匹配这种统计结构,从而以更低能耗换取相同决策质量。"}, + {"id": "insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency", "label": "HW-SW co-design unlocks 10x efficiency", "label_zh": "软硬协同设计释放数量级能效", "kind": "insight", "tier": "insight", "topic": "neuromorphic_hardware", "phase": "frontier", "year": 2023, "card": "insight_hw_sw_codesign.md", "summary_zh": "单独优化算法或单独优化硬件难以突破当前能效瓶颈,只有把架构搜索、稀疏调度与片上存储拓扑当作联合搜索目标,才能在车规算力下实现 10 倍以上的能效。"}, + {"id": "insight:data_engine_loop_is_more_valuable_than_static_dataset", "label": "Data engine > static dataset", "label_zh": "数据引擎闭环比静态数据集更有价值", "kind": "insight", "tier": "insight", "topic": "data_engine", "phase": "core", "year": 2022, "card": "insight_data_engine.md", "summary_zh": "真正决定模型上限的是车队-标注-训练-评测的闭环速度,而不是某个固定数据集的规模,谁能让缺陷在数天内回到训练样本谁就拥有持续领先。"}, + {"id": "insight:simulator_realism_is_lower_bound_on_training_value", "label": "Simulator realism lower-bounds training value", "label_zh": "仿真真实度是其训练价值的下界", "kind": "insight", "tier": "insight", "topic": "simulator", "phase": "frontier", "year": 2023, "card": "insight_sim_realism.md", "summary_zh": "无论行为模型还是视觉渲染,仿真器的真实度直接决定了在其上学得策略的下游可迁移性,提高真实度的边际投入往往比扩大数据规模带来更大的闭环收益。"}, + {"id": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation", "label": "Uncertainty calibration enables safe delegation", "label_zh": "不确定性校准是安全委派的前提", "kind": "insight", "tier": "insight", "topic": "safety_standard", "phase": "frontier", "year": 2024, "card": "insight_uncertainty_calibration.md", "summary_zh": "只有当模型输出的概率与实际错误率高度一致,才能可靠地决定何时由 AI 决策、何时回退到 MPC 或人类,校准误差因此是任何分层安全架构的隐性瓶颈。"}, + {"id": "insight:offline_metrics_co_evolve_with_methods_so_must_be_re_audited", "label": "Offline metrics co-evolve with methods", "label_zh": "离线指标与方法共同演化故须周期性重审", "kind": "insight", "tier": "insight", "topic": "evaluation_benchmark", "phase": "frontier", "year": 2024, "card": "insight_metric_reaudit.md", "summary_zh": "随着模型能力提升,旧指标可能逐步被过拟合并失去判别力,团队必须像维护代码一样周期性重审与再设计指标,使其继续与真实驾驶安全保持一致。"}, + + {"id": "paradigm:closed_loop_data_engine_centric_development", "label": "Closed-loop data engine centric", "label_zh": "以闭环数据引擎为中心的开发范式", "kind": "paradigm", "tier": "paradigm", "topic": "data_engine", "phase": "frontier", "year": 2022, "card": "paradigm_data_engine.md", "summary_zh": "把组织能力组织在车队回传、自动标注、定向训练与闭环回归四个环节的迭代速度上,使每条新发现的失败模式都在数天内变成训练信号与回归用例。"}, + {"id": "paradigm:safety_by_constraint_layered_architecture", "label": "Layered safety-by-constraint", "label_zh": "以分层约束实现安全的范式", "kind": "paradigm", "tier": "paradigm", "topic": "safety_standard", "phase": "frontier", "year": 2023, "card": "paradigm_layered_safety.md", "summary_zh": "把神经规划、屏蔽层、应急控制、监管软件与电子电气冗余视为同一安全论证下的协同层次,每层都用可独立验证的约束承担一部分残余风险。"}, + {"id": "paradigm:brain_inspired_neuromorphic_co_design", "label": "Brain-inspired neuromorphic co-design", "label_zh": "类脑神经形态软硬协同的范式", "kind": "paradigm", "tier": "paradigm", "topic": "neuromorphic_hardware", "phase": "frontier", "year": 2024, "card": "paradigm_brain_inspired.md", "summary_zh": "以事件驱动稀疏计算、脉冲表征与软硬协同搜索为主轴,把感知-决策管线从基于稠密 GPU 的范式重写为可承载未来 L4/L5 算力预算的新范式。"}, + {"id": "paradigm:simulator_first_synthetic_data_centric", "label": "Simulator-first synthetic-data centric", "label_zh": "以仿真与合成数据为先的范式", "kind": "paradigm", "tier": "paradigm", "topic": "simulator", "phase": "frontier", "year": 2024, "card": "paradigm_sim_first.md", "summary_zh": "把可重建、可扰动、可控参数的合成场景作为训练与评测的一等公民,使长尾问题可以在不依赖事故数据采集的前提下被系统性研究与回归。"} + ], + + "edges": [ + {"source": "paper:ad_benchmarks", "target": "paper:nuplan", "rel": "covers"}, + {"source": "paper:ad_benchmarks", "target": "paper:waymo_motion", "rel": "covers"}, + {"source": "paper:ad_benchmarks", "target": "paper:argoverse2", "rel": "covers"}, + {"source": "paper:ad_benchmarks", "target": "paper:navsim", "rel": "covers"}, + {"source": "paper:ad_benchmarks", "target": "paper:bench2drive", "rel": "covers"}, + {"source": "paper:ad_benchmarks", "target": "paper:carla_lb2", "rel": "covers"}, + {"source": "paper:ad_benchmarks", "target": "paper:womd_pred", "rel": "covers"}, + {"source": "paper:ad_benchmarks", "target": "paper:interaction_dataset", "rel": "covers"}, + + {"source": "paper:nuplan", "target": "paper:navsim", "rel": "feeds"}, + {"source": "paper:nuplan", "target": "paper:bench2drive", "rel": "parallel"}, + {"source": "paper:navsim", "target": "paper:bench2drive", "rel": "parallel"}, + {"source": "paper:carla_lb2", "target": "paper:bench2drive", "rel": "feeds"}, + {"source": "paper:waymo_motion", "target": "paper:womd_pred", "rel": "feeds"}, + {"source": "paper:argoverse2", "target": "paper:womd_pred", "rel": "parallel"}, + {"source": "paper:interaction_dataset", "target": "paper:waymo_motion", "rel": "parallel"}, + {"source": "paper:bdd100k", "target": "paper:argoverse2", "rel": "parallel"}, + {"source": "paper:lyft_l5", "target": "paper:waymo_motion", "rel": "parallel"}, + {"source": "paper:pandaset", "target": "paper:argoverse2", "rel": "parallel"}, + {"source": "paper:apolloscape", "target": "paper:bdd100k", "rel": "parallel"}, + + {"source": "paper:highway_env", "target": "paper:metadrive", "rel": "prereq"}, + {"source": "paper:metadrive", "target": "paper:smarts", "rel": "parallel"}, + {"source": "paper:smarts", "target": "paper:commonroad", "rel": "parallel"}, + {"source": "paper:commonroad", "target": "paper:carla_lb2", "rel": "parallel"}, + {"source": "paper:highway_env", "target": "paper:smarts", "rel": "prereq"}, + + {"source": "paper:tesla_ai_day", "target": "paper:tesla_autolabel", "rel": "covers"}, + {"source": "paper:tesla_autolabel", "target": "paper:waymo_scenario_mining", "rel": "parallel"}, + {"source": "paper:tesla_autolabel", "target": "paper:gs_for_ad", "rel": "feeds"}, + {"source": "paper:gs_for_ad", "target": "paper:shift_dataset", "rel": "parallel"}, + {"source": "paper:shift_dataset", "target": "paper:v2x_sim", "rel": "parallel"}, + + {"source": "paper:vaswani2017", "target": "paper:flashattention", "rel": "prereq"}, + {"source": "paper:vaswani2017", "target": "paper:performer", "rel": "prereq"}, + {"source": "paper:vaswani2017", "target": "paper:linear_attention", "rel": "prereq"}, + {"source": "paper:performer", "target": "paper:linear_attention", "rel": "parallel"}, + {"source": "paper:flashattention", "target": "paper:performer", "rel": "contrasts"}, + {"source": "paper:flashattention", "target": "paper:mamba", "rel": "contrasts"}, + {"source": "paper:linear_attention", "target": "paper:mamba", "rel": "prereq"}, + + {"source": "paper:llava", "target": "paper:distill_vlm", "rel": "feeds"}, + {"source": "paper:gptq", "target": "paper:awq", "rel": "parallel"}, + {"source": "paper:gpt3", "target": "paper:gptq", "rel": "feeds"}, + {"source": "paper:llava", "target": "paper:gptq", "rel": "feeds"}, + {"source": "paper:llava", "target": "paper:awq", "rel": "feeds"}, + {"source": "paper:distill_vlm", "target": "paper:2402.12289", "rel": "feeds"}, + + {"source": "paper:2307.01694", "target": "paper:loihi2", "rel": "feeds"}, + {"source": "paper:loihi2", "target": "paper:truenorth", "rel": "extends"}, + {"source": "paper:truenorth", "target": "paper:tianjic", "rel": "parallel"}, + {"source": "paper:loihi2", "target": "paper:tianjic", "rel": "parallel"}, + {"source": "paper:loihi2", "target": "paper:grai", "rel": "parallel"}, + {"source": "paper:dvs_event_camera", "target": "paper:loihi2", "rel": "feeds"}, + {"source": "paper:dvs_event_camera", "target": "paper:2307.01694", "rel": "feeds"}, + + {"source": "paper:iso26262", "target": "paper:sotif_21448", "rel": "prereq"}, + {"source": "paper:sotif_21448", "target": "paper:ul4600", "rel": "extends"}, + {"source": "paper:iso26262", "target": "paper:ul4600", "rel": "prereq"}, + + {"source": "paper:nuplan", "target": "move:design_closed_loop_metric_correlated_with_real_world_safety", "rel": "feeds"}, + {"source": "paper:navsim", "target": "move:design_closed_loop_metric_correlated_with_real_world_safety", "rel": "feeds"}, + {"source": "paper:bench2drive", "target": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", "rel": "feeds"}, + {"source": "paper:navsim", "target": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", "rel": "feeds"}, + {"source": "paper:nuplan", "target": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", "rel": "feeds"}, + {"source": "paper:waymo_scenario_mining", "target": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", "rel": "feeds"}, + {"source": "paper:tesla_autolabel", "target": "move:auto_label_with_offline_model_then_human_in_loop_validate", "rel": "feeds"}, + {"source": "paper:tesla_autolabel", "target": "move:augment_dataset_via_offline_scenario_perturbation", "rel": "feeds"}, + {"source": "paper:gs_for_ad", "target": "move:augment_dataset_via_offline_scenario_perturbation", "rel": "feeds"}, + {"source": "paper:gs_for_ad", "target": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", "rel": "feeds"}, + {"source": "paper:lidar_cam_calib", "target": "move:share_LiDAR_camera_calibration_via_continuous_time_optimization", "rel": "feeds"}, + + {"source": "paper:flashattention", "target": "move:tile_attention_to_fit_SRAM_for_speedup", "rel": "covers"}, + {"source": "paper:performer", "target": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", "rel": "covers"}, + {"source": "paper:linear_attention", "target": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", "rel": "covers"}, + {"source": "paper:gptq", "target": "move:quantize_attention_to_int8_with_calibration", "rel": "feeds"}, + {"source": "paper:awq", "target": "move:quantize_attention_to_int8_with_calibration", "rel": "feeds"}, + {"source": "paper:distill_vlm", "target": "move:distill_large_VLM_into_small_realtime_specialist", "rel": "covers"}, + {"source": "paper:flashattention", "target": "move:cache_KV_state_across_frames_to_amortize_attention_cost", "rel": "feeds"}, + {"source": "paper:mamba", "target": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", "rel": "parallel"}, + + {"source": "paper:2307.01694", "target": "move:replace_dense_attention_with_sparse_event_driven_attention", "rel": "covers"}, + {"source": "paper:2307.01694", "target": "move:implement_spiking_neuron_with_surrogate_gradient_for_backprop", "rel": "feeds"}, + {"source": "paper:loihi2", "target": "move:co_design_silicon_with_algorithm_for_minimum_energy", "rel": "feeds"}, + {"source": "paper:tianjic", "target": "move:co_design_silicon_with_algorithm_for_minimum_energy", "rel": "feeds"}, + {"source": "paper:grai", "target": "move:co_design_silicon_with_algorithm_for_minimum_energy", "rel": "feeds"}, + {"source": "paper:dvs_event_camera", "target": "move:use_event_camera_microsecond_latency_for_emergency_braking", "rel": "feeds"}, + {"source": "paper:dvs_event_camera", "target": "move:replace_dense_attention_with_sparse_event_driven_attention", "rel": "feeds"}, + + {"source": "paper:iso26262", "target": "move:formalize_safety_case_with_claim_evidence_assumption", "rel": "feeds"}, + {"source": "paper:sotif_21448", "target": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", "rel": "feeds"}, + {"source": "paper:sotif_21448", "target": "move:treat_corner_case_as_OOD_detection_then_route_to_human", "rel": "feeds"}, + {"source": "paper:ul4600", "target": "move:formalize_safety_case_with_claim_evidence_assumption", "rel": "covers"}, + {"source": "paper:sotif_21448", "target": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", "rel": "feeds"}, + {"source": "paper:sotif_21448", "target": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", "rel": "feeds"}, + {"source": "paper:ul4600", "target": "move:add_explanation_head_to_promote_interpretability", "rel": "feeds"}, + + {"source": "paper:metadrive", "target": "move:use_difficulty_aware_curriculum_to_accelerate_RL", "rel": "feeds"}, + {"source": "paper:smarts", "target": "move:use_difficulty_aware_curriculum_to_accelerate_RL", "rel": "feeds"}, + {"source": "paper:metadrive", "target": "move:replay_buffer_prioritize_safety_critical_transitions", "rel": "feeds"}, + {"source": "paper:highway_env", "target": "move:replay_buffer_prioritize_safety_critical_transitions", "rel": "feeds"}, + {"source": "paper:commonroad", "target": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", "rel": "feeds"}, + {"source": "paper:waymo_scenario_mining", "target": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", "rel": "feeds"}, + {"source": "paper:shift_dataset", "target": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", "rel": "feeds"}, + + {"source": "move:design_closed_loop_metric_correlated_with_real_world_safety", "rel": "addresses", "target": "problem:offline_metric_does_not_predict_closed_loop_performance"}, + {"source": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", "rel": "addresses", "target": "problem:offline_metric_does_not_predict_closed_loop_performance"}, + {"source": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", "rel": "addresses", "target": "problem:offline_metric_does_not_predict_closed_loop_performance"}, + {"source": "move:augment_dataset_via_offline_scenario_perturbation", "rel": "addresses", "target": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented"}, + {"source": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", "rel": "addresses", "target": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented"}, + {"source": "move:replay_buffer_prioritize_safety_critical_transitions", "rel": "addresses", "target": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented"}, + {"source": "move:distill_large_VLM_into_small_realtime_specialist", "rel": "addresses", "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps"}, + {"source": "move:quantize_attention_to_int8_with_calibration", "rel": "addresses", "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps"}, + {"source": "move:tile_attention_to_fit_SRAM_for_speedup", "rel": "addresses", "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps"}, + {"source": "move:cache_KV_state_across_frames_to_amortize_attention_cost", "rel": "addresses", "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps"}, + {"source": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", "rel": "addresses", "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps"}, + {"source": "move:replace_dense_attention_with_sparse_event_driven_attention", "rel": "addresses", "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps"}, + {"source": "move:perform_neural_architecture_search_with_latency_constraint", "rel": "addresses", "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps"}, + {"source": "move:co_design_silicon_with_algorithm_for_minimum_energy", "rel": "addresses", "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps"}, + {"source": "move:share_LiDAR_camera_calibration_via_continuous_time_optimization", "rel": "addresses", "target": "problem:sensor_calibration_drift_over_vehicle_lifetime"}, + {"source": "move:auto_label_with_offline_model_then_human_in_loop_validate", "rel": "addresses", "target": "problem:label_noise_for_3d_object_categories"}, + {"source": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", "rel": "addresses", "target": "problem:label_noise_for_3d_object_categories"}, + {"source": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", "rel": "addresses", "target": "problem:verification_of_neural_network_safety_properties_at_scale"}, + {"source": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", "rel": "addresses", "target": "problem:verification_of_neural_network_safety_properties_at_scale"}, + {"source": "move:augment_dataset_via_offline_scenario_perturbation", "rel": "addresses", "target": "problem:realistic_other_agent_behavior_in_simulator"}, + {"source": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", "rel": "addresses", "target": "problem:catastrophic_forgetting_under_continual_learning"}, + {"source": "move:add_explanation_head_to_promote_interpretability", "rel": "addresses", "target": "problem:auditability_of_decisions_for_regulatory_compliance"}, + {"source": "move:formalize_safety_case_with_claim_evidence_assumption", "rel": "addresses", "target": "problem:auditability_of_decisions_for_regulatory_compliance"}, + {"source": "move:treat_corner_case_as_OOD_detection_then_route_to_human", "rel": "addresses", "target": "problem:auditability_of_decisions_for_regulatory_compliance"}, + {"source": "paper:gs_for_ad", "target": "problem:simulator_visual_gap_breaks_perception_models", "rel": "addresses"}, + {"source": "paper:shift_dataset", "target": "problem:simulator_visual_gap_breaks_perception_models", "rel": "addresses"}, + {"source": "paper:v2x_sim", "target": "problem:realistic_other_agent_behavior_in_simulator", "rel": "addresses"}, + + {"source": "problem:offline_metric_does_not_predict_closed_loop_performance", "rel": "supports", "target": "insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners"}, + {"source": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented", "rel": "supports", "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset"}, + {"source": "problem:energy_budget_too_small_for_full_transformer_at_30fps", "rel": "supports", "target": "insight:event_driven_computation_matches_natural_sparsity_of_driving_scene"}, + {"source": "problem:energy_budget_too_small_for_full_transformer_at_30fps", "rel": "supports", "target": "insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency"}, + {"source": "problem:verification_of_neural_network_safety_properties_at_scale", "rel": "supports", "target": "insight:safety_emerges_from_layered_constraints_not_single_objective"}, + {"source": "problem:auditability_of_decisions_for_regulatory_compliance", "rel": "supports", "target": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation"}, + {"source": "problem:realistic_other_agent_behavior_in_simulator", "rel": "supports", "target": "insight:simulator_realism_is_lower_bound_on_training_value"}, + {"source": "problem:simulator_visual_gap_breaks_perception_models", "rel": "supports", "target": "insight:simulator_realism_is_lower_bound_on_training_value"}, + {"source": "problem:catastrophic_forgetting_under_continual_learning", "rel": "supports", "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset"}, + {"source": "problem:sensor_calibration_drift_over_vehicle_lifetime", "rel": "supports", "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset"}, + + {"source": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", "rel": "supports", "target": "insight:offline_metrics_co_evolve_with_methods_so_must_be_re_audited"}, + {"source": "move:design_closed_loop_metric_correlated_with_real_world_safety", "rel": "supports", "target": "insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners"}, + {"source": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", "rel": "supports", "target": "insight:safety_emerges_from_layered_constraints_not_single_objective"}, + {"source": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", "rel": "supports", "target": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation"}, + {"source": "move:treat_corner_case_as_OOD_detection_then_route_to_human", "rel": "supports", "target": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation"}, + {"source": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", "rel": "supports", "target": "insight:safety_emerges_from_layered_constraints_not_single_objective"}, + {"source": "move:co_design_silicon_with_algorithm_for_minimum_energy", "rel": "supports", "target": "insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency"}, + {"source": "move:replace_dense_attention_with_sparse_event_driven_attention", "rel": "supports", "target": "insight:event_driven_computation_matches_natural_sparsity_of_driving_scene"}, + {"source": "move:auto_label_with_offline_model_then_human_in_loop_validate", "rel": "supports", "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset"}, + {"source": "move:augment_dataset_via_offline_scenario_perturbation", "rel": "supports", "target": "insight:simulator_realism_is_lower_bound_on_training_value"}, + + {"source": "insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners", "rel": "supports", "target": "paradigm:closed_loop_data_engine_centric_development"}, + {"source": "insight:data_engine_loop_is_more_valuable_than_static_dataset", "rel": "supports", "target": "paradigm:closed_loop_data_engine_centric_development"}, + {"source": "insight:offline_metrics_co_evolve_with_methods_so_must_be_re_audited", "rel": "supports", "target": "paradigm:closed_loop_data_engine_centric_development"}, + {"source": "insight:safety_emerges_from_layered_constraints_not_single_objective", "rel": "supports", "target": "paradigm:safety_by_constraint_layered_architecture"}, + {"source": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation", "rel": "supports", "target": "paradigm:safety_by_constraint_layered_architecture"}, + {"source": "insight:event_driven_computation_matches_natural_sparsity_of_driving_scene", "rel": "supports", "target": "paradigm:brain_inspired_neuromorphic_co_design"}, + {"source": "insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency", "rel": "supports", "target": "paradigm:brain_inspired_neuromorphic_co_design"}, + {"source": "insight:simulator_realism_is_lower_bound_on_training_value", "rel": "supports", "target": "paradigm:simulator_first_synthetic_data_centric"}, + + {"source": "paradigm:closed_loop_data_engine_centric_development", "rel": "covers", "target": "paper:tesla_autolabel"}, + {"source": "paradigm:closed_loop_data_engine_centric_development", "rel": "covers", "target": "paper:waymo_scenario_mining"}, + {"source": "paradigm:closed_loop_data_engine_centric_development", "rel": "covers", "target": "paper:nuplan"}, + {"source": "paradigm:closed_loop_data_engine_centric_development", "rel": "covers", "target": "paper:bench2drive"}, + {"source": "paradigm:safety_by_constraint_layered_architecture", "rel": "covers", "target": "paper:iso26262"}, + {"source": "paradigm:safety_by_constraint_layered_architecture", "rel": "covers", "target": "paper:sotif_21448"}, + {"source": "paradigm:safety_by_constraint_layered_architecture", "rel": "covers", "target": "paper:ul4600"}, + {"source": "paradigm:brain_inspired_neuromorphic_co_design", "rel": "covers", "target": "paper:2307.01694"}, + {"source": "paradigm:brain_inspired_neuromorphic_co_design", "rel": "covers", "target": "paper:loihi2"}, + {"source": "paradigm:brain_inspired_neuromorphic_co_design", "rel": "covers", "target": "paper:tianjic"}, + {"source": "paradigm:brain_inspired_neuromorphic_co_design", "rel": "covers", "target": "paper:dvs_event_camera"}, + {"source": "paradigm:simulator_first_synthetic_data_centric", "rel": "covers", "target": "paper:carla_lb2"}, + {"source": "paradigm:simulator_first_synthetic_data_centric", "rel": "covers", "target": "paper:shift_dataset"}, + {"source": "paradigm:simulator_first_synthetic_data_centric", "rel": "covers", "target": "paper:v2x_sim"}, + {"source": "paradigm:simulator_first_synthetic_data_centric", "rel": "covers", "target": "paper:gs_for_ad"}, + {"source": "paradigm:simulator_first_synthetic_data_centric", "rel": "covers", "target": "paper:metadrive"}, + + {"source": "essay:bitter_lesson", "target": "paradigm:closed_loop_data_engine_centric_development", "rel": "parallel"}, + {"source": "essay:bitter_lesson", "target": "paradigm:simulator_first_synthetic_data_centric", "rel": "parallel"}, + {"source": "essay:bitter_lesson", "target": "paradigm:brain_inspired_neuromorphic_co_design", "rel": "contrasts"}, + {"source": "essay:bitter_lesson", "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset", "rel": "parallel"}, + + {"source": "paper:2212.10156", "target": "paper:bench2drive", "rel": "covers"}, + {"source": "paper:vadv2", "target": "paper:nuplan", "rel": "covers"}, + {"source": "paper:transfuser", "target": "paper:carla_lb2", "rel": "covers"}, + {"source": "paper:2402.12289", "target": "paper:distill_vlm", "rel": "feeds"}, + {"source": "paper:diffuser", "target": "paper:nuplan", "rel": "parallel"}, + + {"source": "concept:spiking_nn", "target": "paper:loihi2", "rel": "covers"}, + {"source": "concept:spiking_nn", "target": "paper:tianjic", "rel": "covers"}, + {"source": "concept:spiking_nn", "target": "paper:truenorth", "rel": "covers"}, + {"source": "concept:spiking_nn", "target": "move:implement_spiking_neuron_with_surrogate_gradient_for_backprop", "rel": "covers"}, + {"source": "concept:self_attention", "target": "paper:flashattention", "rel": "covers"}, + {"source": "concept:self_attention", "target": "paper:performer", "rel": "covers"}, + {"source": "concept:transformer", "target": "paper:flashattention", "rel": "covers"}, + + {"source": "lab:lab06", "target": "move:implement_spiking_neuron_with_surrogate_gradient_for_backprop", "rel": "implements"}, + {"source": "lab:lab06", "target": "move:replace_dense_attention_with_sparse_event_driven_attention", "rel": "implements"}, + {"source": "lab:lab02", "target": "move:use_difficulty_aware_curriculum_to_accelerate_RL", "rel": "parallel"}, + {"source": "lab:lab03", "target": "move:cache_KV_state_across_frames_to_amortize_attention_cost", "rel": "parallel"}, + {"source": "lab:lab04", "target": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", "rel": "parallel"}, + + {"source": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", "target": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", "rel": "feeds"}, + {"source": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", "target": "move:treat_corner_case_as_OOD_detection_then_route_to_human", "rel": "parallel"}, + {"source": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", "target": "move:treat_corner_case_as_OOD_detection_then_route_to_human", "rel": "feeds"}, + {"source": "move:tile_attention_to_fit_SRAM_for_speedup", "target": "move:cache_KV_state_across_frames_to_amortize_attention_cost", "rel": "parallel"}, + {"source": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", "target": "move:replace_dense_attention_with_sparse_event_driven_attention", "rel": "parallel"}, + {"source": "move:quantize_attention_to_int8_with_calibration", "target": "move:perform_neural_architecture_search_with_latency_constraint", "rel": "parallel"}, + {"source": "move:distill_large_VLM_into_small_realtime_specialist", "target": "move:quantize_attention_to_int8_with_calibration", "rel": "feeds"}, + {"source": "move:augment_dataset_via_offline_scenario_perturbation", "target": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", "rel": "feeds"}, + {"source": "move:auto_label_with_offline_model_then_human_in_loop_validate", "target": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", "rel": "feeds"}, + {"source": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", "target": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", "rel": "feeds"}, + {"source": "move:share_LiDAR_camera_calibration_via_continuous_time_optimization", "target": "move:auto_label_with_offline_model_then_human_in_loop_validate", "rel": "prereq"}, + {"source": "move:design_closed_loop_metric_correlated_with_real_world_safety", "target": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", "rel": "feeds"}, + {"source": "move:formalize_safety_case_with_claim_evidence_assumption", "target": "move:add_explanation_head_to_promote_interpretability", "rel": "feeds"} + ] +} diff --git a/docs/data/generated/perception_axis.json b/docs/data/generated/perception_axis.json new file mode 100644 index 0000000..c1fc30b --- /dev/null +++ b/docs/data/generated/perception_axis.json @@ -0,0 +1,374 @@ +{ + "$comment": "Perception, vision-backbone and scene-representation axis of the atlas. Decomposes seminal works into reusable methodological 'moves', anchors them to open problems, and exposes the cross-domain insight templates that recur across image, point-cloud, occupancy and neural-rendering representations.", + + "nodes": [ + {"id": "paper:lift_splat_shoot", "label_en": "Lift-Splat-Shoot", "label_zh": "Lift-Splat-Shoot(LSS:相机到鸟瞰图的可微提升)", "kind": "paper", "tier": "S", "topic": "geometry_3d", "phase": "core", "year": 2020, "summary_zh": "LSS 提出了一种把多路相机图像投射到鸟瞰图坐标系的可微方法:对每个像素同时预测一个语义特征向量和一个深度概率分布,然后用相机内外参把像素特征按深度概率撒到三维体素里,再压扁成鸟瞰特征图。它第一次把基于相机的鸟瞰感知做成端到端可学习的统一管线,成为后续几乎所有基于相机的鸟瞰感知方法的几何先验。"}, + + {"id": "paper:detr3d", "label_en": "DETR3D", "label_zh": "DETR3D(基于稀疏查询的多视角三维检测)", "kind": "paper", "tier": "S", "topic": "geometry_3d", "phase": "core", "year": 2021, "summary_zh": "DETR3D 把二维检测中的 DETR 范式直接搬到多视角三维检测:用一组可学习的三维参考点作为目标查询,把每个三维查询点反投影到所有相机平面上去采样图像特征,再用 Transformer 解码器迭代更新目标位置。它绕开了显式的鸟瞰特征构建,证明了稀疏查询本身就是一种隐式的三维到二维对齐机制。"}, + + {"id": "paper:petr", "label_en": "PETR", "label_zh": "PETR(位置编码即三维感知)", "kind": "paper", "tier": "S", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "PETR 提出在二维图像特征上叠加一个由相机射线生成的三维位置编码,让每个二维像素特征隐式携带它所属相机射线的几何信息。这样目标查询只需做一次普通的注意力就能完成跨视角的三维定位,把多视角三维检测的几何建模问题转换成了纯粹的位置编码工程。"}, + + {"id": "paper:petrv2", "label_en": "PETRv2", "label_zh": "PETRv2(带时序位置编码的多任务三维感知)", "kind": "paper", "tier": "A", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "PETRv2 在 PETR 的三维位置编码上增加了时间维度,把前一帧的相机射线按自车运动对齐到当前坐标系,使得跨时刻的图像特征共享一致的几何参考。它还首次在同一套查询机制上同时承载三维检测和鸟瞰分割两类任务,奠定了多任务共享查询的设计模板。"}, + + {"id": "paper:bevdet", "label_en": "BEVDet", "label_zh": "BEVDet(鸟瞰图检测的标准管线)", "kind": "paper", "tier": "A", "topic": "geometry_3d", "phase": "core", "year": 2021, "summary_zh": "BEVDet 把 Lift-Splat-Shoot 的视角变换、鸟瞰特征编码与基于密集卷积的检测头组装成一条工程化的标准管线,并系统地研究了图像分辨率、视角变换分辨率、鸟瞰分辨率、数据增广之间的耦合。它使得基于相机的鸟瞰检测从研究原型变成可大规模复现、可工程优化的基线。"}, + + {"id": "paper:bevdet4d", "label_en": "BEVDet4D", "label_zh": "BEVDet4D(时序鸟瞰特征融合)", "kind": "paper", "tier": "A", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "BEVDet4D 在 BEVDet 之上引入跨帧鸟瞰特征对齐,把上一帧鸟瞰特征按自车运动平移到当前帧的参考系,然后简单拼接给后续头部。它把一直困扰相机方案的速度估计问题从依赖图像光流的复杂方案简化为鸟瞰特征上的时间差分,极大改善了与点云方案的速度差距。"}, + + {"id": "paper:bevfusion", "label_en": "BEVFusion", "label_zh": "BEVFusion(鸟瞰特征空间的多模态融合)", "kind": "paper", "tier": "S", "topic": "sensor_fusion", "phase": "core", "year": 2022, "summary_zh": "BEVFusion 把相机分支的鸟瞰特征和点云分支的鸟瞰特征在同一个鸟瞰栅格上对齐后再融合,而不是在感知头处晚融合或在图像-点云层面早融合。这种统一的中间表示使得任一模态失效时另一模态仍可独立工作,并且为后续在鸟瞰空间做规划、预测打开了模块解耦的接口。"}, + + {"id": "paper:bevformer_v2", "label_en": "BEVFormer v2", "label_zh": "BEVFormer v2(带透视监督的鸟瞰图 Transformer)", "kind": "paper", "tier": "A", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "BEVFormer v2 给 BEVFormer 增加了一个额外的透视视角检测头作为辅助监督,强迫鸟瞰特征同时携带与原始图像可对齐的语义。这种\"双视角监督\"思路有效缓解了鸟瞰特征空洞、二维表观信息流失的问题,也为引入图像域预训练大模型铺平了道路。"}, + + {"id": "paper:occupancy_networks_tesla", "label_en": "Tesla Occupancy Networks", "label_zh": "Tesla 占用网络(占用栅格替代检测框)", "kind": "paper", "tier": "S", "topic": "scene_understanding", "phase": "frontier", "year": 2022, "summary_zh": "Tesla 在 AI Day 公开的占用网络把场景表示从\"对每个已知类别画三维框\"换成\"对空间每一个体素预测是否被占用以及流速\",从而天然地处理未知类别物体和不规则形状。这一表示让感知不再依赖固定的类别集合,对应了开放世界自动驾驶的核心需求。"}, + + {"id": "paper:surroundocc", "label_en": "SurroundOcc", "label_zh": "SurroundOcc(多相机三维占用预测)", "kind": "paper", "tier": "A", "topic": "scene_understanding", "phase": "core", "year": 2023, "summary_zh": "SurroundOcc 把多相机图像融合到一个完整的三维体素空间,预测每个体素的占用与语义类别,并通过多尺度交叉注意力让低分辨率的占用预测引导高分辨率的细节。它是把 Tesla 占用网络思路开源化、学术基准化的代表性工作,让占用预测进入公开的研究比赛。"}, + + {"id": "paper:occ3d", "label_en": "Occ3D / Occ3D-nuScenes", "label_zh": "Occ3D(三维占用基准)", "kind": "paper", "tier": "A", "topic": "scene_understanding", "phase": "core", "year": 2023, "summary_zh": "Occ3D 在 nuScenes 等数据集上构建了一致的三维占用真值生成管线,把累积多帧点云、可见性掩码、类别标签统一成体素化标准。它的真正贡献在于让\"占用预测\"作为一个独立任务有了可比较的基准,研究者从此能在同一坐标系下评估不同表示方案的优劣。"}, + + {"id": "paper:simplebev", "label_en": "SimpleBEV", "label_zh": "SimpleBEV(去掉深度估计的鸟瞰感知基线)", "kind": "paper", "tier": "B", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "SimpleBEV 用最朴素的\"对每条相机射线均匀采样三维点然后聚合特征\"代替显式的深度概率分布,得到的鸟瞰感知性能竟然能逼近复杂的 LSS 体系。它充当了一个去伪存真的对照实验:揭示了真正起作用的是体素聚合策略而不是深度网络本身。"}, + + {"id": "paper:streampetr", "label_en": "StreamPETR", "label_zh": "StreamPETR(流式查询的长时序三维感知)", "kind": "paper", "tier": "A", "topic": "geometry_3d", "phase": "core", "year": 2023, "summary_zh": "StreamPETR 把 PETR 的目标查询做成跨帧持续传播的隐状态:每个查询不仅做当前帧的检测,还把更新后的状态传到下一帧继续推理。它把时序融合从\"对齐特征图\"上升到\"对齐对象级抽象\",让相机方案在长时间窗口下的检测稳定性接近基于点云的方案。"}, + + {"id": "paper:mae", "label_en": "MAE", "label_zh": "MAE(掩码图像建模)", "kind": "paper", "tier": "S", "topic": "ssl_vision", "phase": "prereq", "year": 2021, "summary_zh": "MAE 把 BERT 的掩码语言建模迁移到图像上,遮掉百分之七十五的图像块,让编码器只看可见块,再用一个轻量解码器重建被遮像素。它最重要的发现是\"高比例遮挡是必要的\",这使得自监督预训练在视觉上第一次能像在文本上那样稳定地扩展到大模型与大数据。"}, + + {"id": "paper:beit", "label_en": "BEiT", "label_zh": "BEiT(图像版 BERT 预训练)", "kind": "paper", "tier": "A", "topic": "ssl_vision", "phase": "prereq", "year": 2021, "summary_zh": "BEiT 先用一个离散视觉分词器把图像块映射成视觉词元,再用掩码视觉词元预测来训练编码器。它把语言模型的\"离散 token + 掩码预测\"范式整体复制到图像上,为后续多模态统一架构提供了\"视觉也可以离散化\"的基础工具。"}, + + {"id": "paper:clip", "label_en": "CLIP", "label_zh": "CLIP(图文对比预训练)", "kind": "paper", "tier": "S", "topic": "ssl_vision", "phase": "prereq", "year": 2021, "summary_zh": "CLIP 用四亿对网络爬取的图文对做对比学习,让图像编码器和文本编码器把语义对齐到同一空间。它最深远的影响不是分类精度,而是证明了\"用自然语言作为零样本类别接口\"这个范式可以替代固定类别表,后续所有开放词表感知和视觉-语言驱动方法都建立在这一对齐之上。"}, + + {"id": "paper:blip2", "label_en": "BLIP-2", "label_zh": "BLIP-2(冻结视觉与语言基座的轻量桥接)", "kind": "paper", "tier": "A", "topic": "ssl_vision", "phase": "prereq", "year": 2023, "summary_zh": "BLIP-2 提出 Q-Former 这个小型可训练桥接模块,让冻结的视觉编码器与冻结的大语言模型之间只需训练极少参数即可对齐。它确立了\"把昂贵的预训练基座当成不可改的事实,只在中间加可学习适配器\"这一极具工程价值的多模态对接范式。"}, + + {"id": "paper:vilt", "label_en": "ViLT", "label_zh": "ViLT(无卷积的极简视觉语言 Transformer)", "kind": "paper", "tier": "B", "topic": "ssl_vision", "phase": "prereq", "year": 2021, "summary_zh": "ViLT 去掉了图像目标检测器和卷积主干,直接把图像块和文本词元拼接送进一个 Transformer 学习视觉-语言对齐。它的极简主义证明了模态融合本身比模态特有的归纳偏置更关键,是后来多模态大模型简化结构的早期先声。"}, + + {"id": "paper:nerf", "label_en": "NeRF", "label_zh": "NeRF(神经辐射场)", "kind": "paper", "tier": "S", "topic": "scene_understanding", "phase": "prereq", "year": 2020, "summary_zh": "NeRF 用一个多层感知机把三维空间坐标和观察方向映射为颜色与体密度,再用体渲染积分合成新视角图像。它把整个静态场景压缩进神经网络权重,开创了\"用隐式函数代替显式三维表示\"的全新流派,并直接催生了驾驶场景重建与仿真的神经渲染分支。"}, + + {"id": "paper:3dgs", "label_en": "3D Gaussian Splatting", "label_zh": "三维高斯泼溅(3D Gaussian Splatting)", "kind": "paper", "tier": "S", "topic": "scene_understanding", "phase": "core", "year": 2023, "summary_zh": "三维高斯泼溅用一组带位置、协方差、颜色与不透明度的各向异性高斯点来显式表示场景,再用可微的栅格化器投射到屏幕。它在保持神经辐射场可微优化能力的同时,把渲染速度提升了一两个数量级,迅速取代纯 MLP 表示成为驾驶场景重建的新基础设施。"}, + + {"id": "paper:emernerf", "label_en": "EmerNeRF", "label_zh": "EmerNeRF(驾驶场景的自监督动静解耦神经辐射场)", "kind": "paper", "tier": "A", "topic": "scene_understanding", "phase": "frontier", "year": 2023, "summary_zh": "EmerNeRF 通过把场景分解为静态背景流和时变动态流两支辐射场,并加入光流自监督,让驾驶场景的重建可以在没有人工标注动态物体的情况下自动分离动静。它把神经辐射场从\"重建一段静态片段\"推进到\"理解一段含动态对象的驾驶序列\"。"}, + + {"id": "paper:drivinggaussian", "label_en": "DrivingGaussian", "label_zh": "DrivingGaussian(动态驾驶场景的高斯重建)", "kind": "paper", "tier": "A", "topic": "scene_understanding", "phase": "frontier", "year": 2024, "summary_zh": "DrivingGaussian 把三维高斯泼溅扩展到带有大量运动车辆与行人的城市驾驶场景:用静态高斯捕获背景、用以物体为中心的动态高斯捕获每个移动目标,再统一渲染。它让大规模、可编辑、可仿真的驾驶场景数字孪生成为现实。"}, + + {"id": "paper:dinov1", "label_en": "DINO (self-distillation)", "label_zh": "DINO(自蒸馏视觉自监督)", "kind": "paper", "tier": "A", "topic": "ssl_vision", "phase": "prereq", "year": 2021, "summary_zh": "DINO 让一个学生网络匹配教师网络对同一图像不同视图的输出分布,教师权重是学生的指数滑动平均,无需负样本即可避免坍缩。它意外地展示了 ViT 在纯自监督下会自动浮现出物体分割的注意力图,启示了\"无标签即可学到结构\"这一持续推动 SSL 路线的核心现象。"}, + + {"id": "paper:simclr_mocov3", "label_en": "SimCLR / MoCo v3", "label_zh": "SimCLR / MoCo v3(对比学习视觉预训练)", "kind": "paper", "tier": "B", "topic": "ssl_vision", "phase": "prereq", "year": 2020, "summary_zh": "SimCLR 与后续的 MoCo v3 用同一张图的不同数据增广作为正对、其他图作为负对做对比学习,把表示空间塑造为语义近则近、语义远则远。它们确立了对比学习作为视觉自监督的主流方案,也暴露了对负样本数与批大小的依赖,为后续无负样本的方法(如 DINO)提供了反面参照。"}, + + {"id": "paper:depth_anything", "label_en": "Depth Anything", "label_zh": "Depth Anything(通用单目深度大模型)", "kind": "paper", "tier": "A", "topic": "geometry_3d", "phase": "frontier", "year": 2024, "summary_zh": "Depth Anything 用约六千万张未标注图像做半监督蒸馏,让一个 ViT 主干学会几乎在任意场景给出鲁棒的单目相对深度。它使\"深度\"这一原本需要昂贵设备或多视角几何的几何量,变成了一个像图像分类一样可现成调用的通用先验,显著降低了下游三维感知的入门门槛。"}, + + {"id": "paper:vggt", "label_en": "VGGT", "label_zh": "VGGT(前馈三维几何 Transformer)", "kind": "paper", "tier": "B", "topic": "geometry_3d", "phase": "frontier", "year": 2025, "summary_zh": "VGGT 用一个 Transformer 在一次前向中同时输出相机内外参、深度图、点云和稠密对应,完全绕开传统的逐对匹配与捆绑调整。它把\"结构从运动\"这一经典计算机视觉问题彻底重写为大模型预训练问题,为基于视频的快速三维重建提供了新的工程上限。"}, + + {"id": "paper:openocc_unic", "label_en": "OpenOccupancy / UniOcc", "label_zh": "OpenOccupancy 与 UniOcc(开放占用与统一占用基准)", "kind": "paper", "tier": "B", "topic": "scene_understanding", "phase": "core", "year": 2023, "summary_zh": "OpenOccupancy 与 UniOcc 联合提供了大规模、长尾、含未知类别的占用基准,强调评估在\"语义未知但空间存在\"情境下的鲁棒性。它们把占用预测的研究焦点从\"对已知类的分类精度\"推向\"对世界结构本身的覆盖率\",反向影响了后续模型架构的设计取向。"}, + + {"id": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "label_en": "Lift 2D features to 3D via learned depth distribution", "label_zh": "把二维特征按学习到的深度分布提升到三维", "kind": "move", "tier": "move", "topic": "geometry_3d", "phase": "core", "year": 2020, "summary_zh": "不再尝试为每个像素回归单一深度,而是预测一个离散深度分布,把图像特征按概率\"撒\"到对应的三维体素位置。这样视角变换变得可微分,模型可以在端到端训练中自己决定特征该聚到哪个深度,绕过了深度估计不可靠时整条管线崩溃的问题。其代表性应用是 LSS、BEVDet 系列,并被 BEVFusion 借去做相机分支。"}, + + {"id": "move:treat_detection_as_set_prediction_with_learnable_queries", "label_en": "Treat detection as set prediction with learnable queries", "label_zh": "把检测任务转化为学习查询集合的预测问题", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "core", "year": 2020, "summary_zh": "用一组固定数量的可学习查询向量代替密集锚框,每个查询通过注意力机制竞争性地\"认领\"一个目标,最后由匈牙利匹配与真值对齐。这一移动消除了非极大值抑制等手工后处理,使得检测器变成纯可微管线,并把每个查询天然变成下游任务可继承的对象级抽象。其最早形态是 DETR,被 DETR3D 推向三维,被 UniAD 进一步当成统一规划的接口。"}, + + {"id": "move:reproject_3d_query_to_2d_for_feature_sampling", "label_en": "Reproject 3D query to 2D for feature sampling", "label_zh": "把三维查询点反投影到二维图像采样特征", "kind": "move", "tier": "move", "topic": "geometry_3d", "phase": "core", "year": 2021, "summary_zh": "不构建显式的鸟瞰特征图,而是让每个三维查询点利用相机内外参反投影到各路相机上,直接采样原图特征。这把\"特征对齐\"从工程化的视角变换转化为简单的几何投影,省掉了对体素分辨率的依赖,也让多视角融合天然处理不同相机的几何关系。代表为 DETR3D、StreamPETR 系列。"}, + + {"id": "move:embed_camera_geometry_into_positional_encoding", "label_en": "Embed camera geometry into positional encoding", "label_zh": "把相机几何信息直接编码进位置编码", "kind": "move", "tier": "move", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "把每个像素所属的相机射线起点和方向参数化后加到二维特征的位置编码中,让每个二维特征自带三维上下文。注意力机制就无需显式投影即可学到正确的几何对齐。这一移动彻底简化了多视角三维检测的结构,在 PETR 上首次工作,后被几乎所有\"几何不显式\"的方案采用。"}, + + {"id": "move:replace_explicit_module_with_implicit_function", "label_en": "Replace explicit module with implicit function", "label_zh": "用隐式函数替换显式模块", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "core", "year": 2020, "summary_zh": "把场景、形状、占用等原本以显式离散结构存储的量,替换为以连续坐标为输入的神经函数。这样表示自然连续、可微、可任意精细化采样,并能由可微渲染监督训练。这一移动的源头是 NeRF,扩散到 SDF 表面表示、占用场表示、神经隐式定位等,几乎重塑了几何视觉的下游栈。"}, + + {"id": "move:swap_implicit_for_explicit_primitives_when_compute_allows", "label_en": "Swap implicit for explicit primitives when compute allows", "label_zh": "当算力允许时,用显式基元换回隐式表示", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "frontier", "year": 2023, "summary_zh": "在保持渲染过程可微的前提下,把神经网络隐式表示换成大量显式基元(高斯点、点云、网格片)。这种反向移动牺牲一些泛化性,换来数量级的渲染加速与可编辑性。三维高斯泼溅是范式样本,提醒研究者\"隐式\"和\"显式\"是连续光谱上的两个极端而非二元对立。"}, + + {"id": "move:add_auxiliary_perspective_supervision_to_bev", "label_en": "Add auxiliary perspective supervision to BEV", "label_zh": "为鸟瞰特征增加透视视角的辅助监督", "kind": "move", "tier": "move", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "在鸟瞰特征上额外挂一个透视视角的检测/分割头作为副任务,迫使鸟瞰特征不丢失原始图像里能直接观察到的语义。它解决了纯鸟瞰监督下特征容易\"塌缩\"为只关心俯视框的问题,也是接入大规模图像预训练模型的天然入口。BEVFormer v2 是典范应用。"}, + + {"id": "move:carry_object_query_across_time_as_recurrent_state", "label_en": "Carry object query across time as recurrent state", "label_zh": "把对象查询作为循环状态跨帧传递", "kind": "move", "tier": "move", "topic": "geometry_3d", "phase": "core", "year": 2023, "summary_zh": "不再在每一帧重新生成查询,而是把上一帧更新后的查询连同其隐藏状态传到当前帧作为初始状态。这种循环式查询机制把时序融合从\"对齐特征图\"升级为\"对齐对象级抽象\",天然带来 ID 一致性和长时序稳定性,典型如 StreamPETR、Sparse4D 系列。"}, + + {"id": "move:fuse_modalities_in_shared_intermediate_space", "label_en": "Fuse modalities in shared intermediate space", "label_zh": "在共享中间表示空间中融合多种模态", "kind": "move", "tier": "move", "topic": "sensor_fusion", "phase": "core", "year": 2022, "summary_zh": "为相机、点云、雷达等设计各自的编码器,把它们各自的输出都投影到同一中间表示(如鸟瞰栅格、占用体素)后再融合。相较于早融合或晚融合,中间融合既保留模态特有的归纳偏置,又使单模态失效时其他模态可独立工作,显著提升鲁棒性。BEVFusion 是范例。"}, + + {"id": "move:replace_class_specific_box_with_class_agnostic_occupancy", "label_en": "Replace class-specific box with class-agnostic occupancy", "label_zh": "用与类别无关的占用代替按类别画框", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "frontier", "year": 2022, "summary_zh": "放弃\"先认出类别再画框\"的范式,改为对空间每个体素预测它是否被占用以及流速。这样未知类别、不规则形状、累积的小障碍物都能被统一表示。这一移动是 Tesla 占用网络的核心思想,也被随后所有\"通用障碍物检测\"研究采用。"}, + + {"id": "move:scale_pretraining_then_fine_tune_with_minimal_labels", "label_en": "Scale self-supervised pretraining, then fine-tune with minimal labels", "label_zh": "扩规模做自监督预训练,再用少量标签微调", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "core", "year": 2021, "summary_zh": "把大量未标注数据投喂给一个自监督目标,先得到一个通用视觉表征,再用少量任务相关标签做轻量下游微调。这一移动把\"数据标注\"从瓶颈变为可选项,使得自动驾驶研究可以受益于互联网级数据。MAE、DINOv2、DINOv3 等都是这一移动在不同自监督信号下的实例。"}, + + {"id": "move:freeze_giant_backbone_train_small_adapter", "label_en": "Freeze giant backbone, train small adapter", "label_zh": "冻结大型主干,只训练小型适配器", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "core", "year": 2022, "summary_zh": "把昂贵的预训练大模型权重冻结,只在中间插入轻量级可训练桥接(如 Q-Former、LoRA 适配器、线性投影)。这样既保留基座知识,又把训练成本压到能在小集群上完成,使学术界也能复现工业级多模态系统。BLIP-2 是该移动的典型实现。"}, + + {"id": "move:tokenize_continuous_signal_to_use_transformer", "label_en": "Tokenize continuous signal to use a Transformer", "label_zh": "把连续信号离散化以套用 Transformer", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "prereq", "year": 2020, "summary_zh": "把图像切块、把点云分体素、把动作分位段——总之先把连续模态人为切成有限符号集,然后整个 Transformer 训练栈(掩码预测、对比学习、自回归)都能直接复用。这一移动是 ViT、VQ-VAE、BEiT、各类世界模型共同的入场卷。"}, + + {"id": "move:use_geometry_as_self_supervision", "label_en": "Use geometric consistency as a free self-supervision signal", "label_zh": "用几何一致性作为免费的自监督信号", "kind": "move", "tier": "move", "topic": "geometry_3d", "phase": "core", "year": 2017, "summary_zh": "多视角一致、时序一致、立体一致、光度一致——这些几何约束在采集数据时自动成立,无需人工标注就可以作为损失项。可用于自监督深度、流、位姿、辐射场。NeRF 的体渲染监督、单目深度的光度损失、EmerNeRF 的光流监督都建立在这一移动上。"}, + + {"id": "move:make_pipeline_differentiable_via_shared_latent", "label_en": "Make pipeline differentiable via shared latent representation", "label_zh": "通过共享隐表示使整条管线可微", "kind": "move", "tier": "move", "topic": "e2e_ad", "phase": "core", "year": 2022, "summary_zh": "把\"感知-预测-规划\"等本来用规则连接的模块,改用共享的隐表示(查询、鸟瞰特征、占用体素)做接口,让梯度从最后的任务损失一直反传到原始图像。这是 UniAD、VAD 等端到端方案的方法论根基,也使得\"上游模块为下游目标服务\"这件事第一次能被训练数据自动达成。"}, + + {"id": "move:rasterize_differentiable_renderer_for_inverse_problem", "label_en": "Wrap a differentiable renderer to invert image formation", "label_zh": "用可微渲染器反演成像过程", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "core", "year": 2020, "summary_zh": "把渲染过程写成对场景参数可微的算子,然后把\"已知图像、求场景\"的反问题变成普通梯度下降。NeRF 的体积分、三维高斯泼溅的栅格化都是这一移动的实例,使得任意场景表示形式只要配上可微渲染器就能从图像直接监督。"}, + + {"id": "move:distill_internet_data_into_small_specialist", "label_en": "Distill from web-scale data into a specialist model", "label_zh": "把网络规模数据蒸馏进领域专家模型", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "frontier", "year": 2024, "summary_zh": "先用海量未标注互联网数据训练一个能力宽泛的教师模型,再用伪标签把其能力蒸馏到一个针对具体任务/数据分布的较小学生。Depth Anything、SAM 的训练管线都体现了这一移动:把互联网视觉数据\"打包\"成可被下游直接使用的归纳偏置。"}, + + {"id": "move:make_camera_only_temporal_match_lidar", "label_en": "Make camera-only with temporal aggregation match LiDAR", "label_zh": "用时序聚合让纯相机方案逼近激光雷达", "kind": "move", "tier": "move", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "把多个时刻的相机鸟瞰特征/查询对齐叠加,等价地累积了视差与运动信息,从而在静态深度和速度估计上逼近激光雷达。这一移动让\"去激光雷达\"从口号变成可量化的工程目标,代表为 BEVDet4D、SOLOFusion、StreamPETR。"}, + + {"id": "move:open_vocabulary_via_text_alignment", "label_en": "Open vocabulary via image-text alignment", "label_zh": "通过图文对齐实现开放词表识别", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "core", "year": 2021, "summary_zh": "把分类头换成\"和文本嵌入做内积\",于是任何能写成自然语言的概念都能即时变成新类别。这把感知的类别集从\"固定 80 类\"变成\"语言空间任意子集\",对自动驾驶里频发的长尾物体尤其重要。CLIP 是该移动的奠基者,所有开放词表检测/分割都建立在其上。"}, + + {"id": "move:emergent_segmentation_from_self_distillation", "label_en": "Let segmentation emerge from self-distillation", "label_zh": "让分割能力在自蒸馏中自然涌现", "kind": "move", "tier": "move", "topic": "ssl_vision", "phase": "core", "year": 2021, "summary_zh": "不显式训练任何分割损失,只让一个 ViT 学生模仿教师在不同视图下的输出分布,结果注意力图自动浮现出物体级别的分割。这一移动揭示\"结构归纳偏置 + 强自监督\"足以让显式监督本不该出现的能力涌现,改变了\"先标注后训练\"的研究次序。DINO 是源头。"}, + + {"id": "move:replace_handcrafted_sfm_with_feedforward_transformer", "label_en": "Replace handcrafted SfM with a feed-forward Transformer", "label_zh": "用前馈 Transformer 取代手工的多视图几何流水线", "kind": "move", "tier": "move", "topic": "geometry_3d", "phase": "frontier", "year": 2025, "summary_zh": "把\"特征匹配-本质矩阵-捆绑调整\"这套四十年累积的几何工程,整体替换为一个直接吃多张图像、输出相机参数和稠密三维结构的 Transformer。这一移动延续了\"苦涩教训\"的判断:大模型加规模终将赢过精雕细琢的几何方法。VGGT 是其代表。"}, + + {"id": "move:decompose_scene_into_static_and_dynamic_streams", "label_en": "Decompose scene into static and dynamic streams", "label_zh": "把场景显式分解为静态流与动态流", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "core", "year": 2023, "summary_zh": "用两套独立但联合渲染的表示——一套表示固定背景,一套表示时变前景——再让它们相互约束。这一移动让无需人工标注就能自动分离动态对象,是把神经辐射场扩展到真实驾驶场景的必要步骤。EmerNeRF、DrivingGaussian 都使用了它的变体。"}, + + {"id": "move:bridge_sim_and_real_via_neural_reconstruction", "label_en": "Bridge sim and real via neural reconstruction of real logs", "label_zh": "用对真实日志的神经重建桥接仿真与现实", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "frontier", "year": 2023, "summary_zh": "不再从零搭仿真世界,而是把真实路采视频重建成可重新渲染、可摆放对象、可改变天气的数字孪生,再在其中插入新场景做策略训练。这一移动用神经重建打通了\"采集-标注-训练-评估\"闭环,是当前最被工业界看重的仿真新路线。DrivingGaussian 等是入口。"}, + + {"id": "move:augment_via_counterfactual_object_insertion", "label_en": "Augment data via counterfactual object insertion", "label_zh": "通过反事实物体插入扩充数据", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "frontier", "year": 2024, "summary_zh": "在已有真实序列里插入并不存在的物体(被神经渲染或图像生成模型逼真地合成),得到大量\"几乎真实但永远不会被采集到\"的边缘案例。这一移动让长尾问题第一次有了规模化的解,是当前神经重建管线最具产品价值的副产物。"}, + + {"id": "move:share_queries_across_multiple_tasks", "label_en": "Share queries across multiple downstream tasks", "label_zh": "在多个下游任务之间共享同一组查询", "kind": "move", "tier": "move", "topic": "e2e_ad", "phase": "core", "year": 2022, "summary_zh": "让检测、跟踪、地图、占用、规划等任务复用同一组对象级查询,把多任务联合训练变成天然的隐表示对齐器。这一移动是 UniAD\"以规划为目的统一感知\"的方法论核心,也是 PETRv2 等多任务方案的设计范式。"}, + + {"id": "move:learn_motion_in_latent_space_then_decode", "label_en": "Learn motion in latent space then decode to pixels", "label_zh": "在隐空间预测运动后再解码到像素", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "frontier", "year": 2023, "summary_zh": "不直接在像素级预测下一帧,而是先把图像压成隐令牌,然后在令牌空间预测未来,最后只在需要可视化时才解码回像素。这把高维像素预测的难题转化为低维语义预测,是世界模型方案在驾驶场景可工作的关键工程基石。"}, + + {"id": "move:use_visibility_mask_to_filter_supervision", "label_en": "Use visibility masks to filter supervision in occupancy learning", "label_zh": "用可见性掩码过滤占用学习的监督信号", "kind": "move", "tier": "move", "topic": "scene_understanding", "phase": "core", "year": 2023, "summary_zh": "在体素化激光雷达累积的占用真值上叠加可见性掩码,只在确实被传感器观测到的体素上计算损失,避免模型被\"遮挡背面\"的伪真值污染。这一看似工程的细节其实决定了占用学习能否泛化,是 Occ3D 等基准能稳定收敛的关键。"}, + + {"id": "problem:long_tail_object_categories_in_open_world", "label_en": "Long-tail object categories in open-world driving", "label_zh": "开放世界中长尾物体类别问题", "kind": "problem", "tier": "problem", "topic": "scene_understanding", "phase": "frontier", "year": 2022, "summary_zh": "真实道路上不断出现训练集没见过的物体:翻倒的椅子、散落的轮胎、奇怪形状的工程车。任何依赖固定类别表的检测器都会对其失效。开放词表识别、占用预测、世界模型蒸馏都是对该问题的不同尝试,但至今没有一个方案能在所有维度都做得好。"}, + + {"id": "problem:sim_to_real_gap_in_camera_only_perception", "label_en": "Sim-to-real gap in camera-only perception", "label_zh": "纯相机感知的仿真到现实差距", "kind": "problem", "tier": "problem", "topic": "geometry_3d", "phase": "core", "year": 2018, "summary_zh": "由 CARLA 等仿真器渲染出的图像与真实相机图像在光照、噪声、镜头畸变、运动模糊上的分布差异,使得在仿真里训得好的纯相机感知模型一到真实路况就大幅退化。神经重建仿真、域随机化、对抗增广都是减弱这一差距的方向,但仍未根除。"}, + + {"id": "problem:temporal_consistency_in_bev_segmentation", "label_en": "Temporal consistency in BEV segmentation", "label_zh": "鸟瞰图分割的时序一致性问题", "kind": "problem", "tier": "problem", "topic": "geometry_3d", "phase": "core", "year": 2021, "summary_zh": "逐帧独立的鸟瞰分割常常出现物体边界跳动、车道线闪烁,严重影响下游规划。如何在保持低延迟的同时让鸟瞰输出在时间上平滑、ID 稳定,是一个工程上反复出现却很难有原则性解的痛点,目前主要靠循环查询、ID 追踪损失等弱方案。"}, + + {"id": "problem:occlusion_reasoning_without_dense_lidar", "label_en": "Occlusion reasoning without dense LiDAR", "label_zh": "没有稠密激光雷达时的遮挡推理", "kind": "problem", "tier": "problem", "topic": "scene_understanding", "phase": "core", "year": 2020, "summary_zh": "纯相机方案无法直接观察被遮挡区域,但安全驾驶又必须对\"可能藏着行人的盲区\"做出合理假设。占用预测、生成式世界模型、几何先验都试图回答这一点,但缺乏一种既准确又可校准不确定性的方法。"}, + + {"id": "problem:label_efficiency_for_3d_annotation", "label_en": "Label efficiency for 3D annotation", "label_zh": "三维标注的标签效率问题", "kind": "problem", "tier": "problem", "topic": "scene_understanding", "phase": "core", "year": 2019, "summary_zh": "标注三维框、占用、轨迹的人工成本远高于二维。自监督预训练、自动标注管线、神经重建辅助标注都尝试缩减成本,但目前业界仍依赖大量人工质检,这是开放数据集规模扩展和长尾覆盖的最大障碍之一。"}, + + {"id": "problem:unknown_geometry_in_distant_or_dark_regions", "label_en": "Unknown geometry in distant or dark regions", "label_zh": "远距离或低光区域的几何未知问题", "kind": "problem", "tier": "problem", "topic": "geometry_3d", "phase": "frontier", "year": 2021, "summary_zh": "图像在远距离失去分辨率、在低光下失去对比度,传统几何方法直接失效。学习方法虽能\"猜测\"几何,但难以量化置信度。这是夜间、隧道、雨雾等场景中所有纯视觉方案共同的根本困难。"}, + + {"id": "problem:multi_modal_calibration_drift", "label_en": "Multi-modal sensor calibration drift", "label_zh": "多模态传感器外参漂移问题", "kind": "problem", "tier": "problem", "topic": "sensor_fusion", "phase": "core", "year": 2018, "summary_zh": "相机、激光雷达、毫米波雷达之间的外参在车辆使用过程中会缓慢漂移,使中间表示空间里的特征对不齐,融合性能急剧下降。在线自标定、可微外参学习都是回应,但鲁棒性仍未达到工业级要求。"}, + + {"id": "problem:rendering_speed_vs_quality_tradeoff", "label_en": "Rendering speed versus quality trade-off in neural reconstruction", "label_zh": "神经重建中渲染速度与质量的权衡", "kind": "problem", "tier": "problem", "topic": "scene_understanding", "phase": "frontier", "year": 2022, "summary_zh": "纯神经辐射场质量高但渲染慢,三维高斯泼溅渲染快但难以表达半透明、反射等复杂材质,且在动态场景中颗粒感明显。如何在保持训练-推理时间合理的前提下覆盖驾驶场景的所有材质与天气,仍是开放问题。"}, + + {"id": "problem:catastrophic_failure_on_rare_weather", "label_en": "Catastrophic failure on rare weather and lighting", "label_zh": "罕见天气与光照下的灾难性失效", "kind": "problem", "tier": "problem", "topic": "geometry_3d", "phase": "core", "year": 2019, "summary_zh": "雨、雪、大雾、夕阳逆光在自然数据集中比例极低,使得感知模型在这些条件下的表现几乎不可预测。合成数据增广、生成式数据扩充、专项数据集都被尝试,但没有任何方法已经把这一长尾压到产品可接受的水平。"}, + + {"id": "problem:annotation_inconsistency_across_datasets", "label_en": "Annotation inconsistency across driving datasets", "label_zh": "跨驾驶数据集的标注不一致问题", "kind": "problem", "tier": "problem", "topic": "scene_understanding", "phase": "core", "year": 2020, "summary_zh": "nuScenes、Waymo Open、Argoverse、KITTI 各自的类别表、坐标系、可见性定义都不一致,使得\"联合训练\"变得困难,模型必须在不同语义之间做痛苦的折中。统一占用类、跨数据集联合学习是部分解,但根本协议仍未统一。"}, + + {"id": "problem:depth_ambiguity_in_low_parallax", "label_en": "Depth ambiguity in low-parallax monocular views", "label_zh": "低视差单目视图的深度歧义问题", "kind": "problem", "tier": "problem", "topic": "geometry_3d", "phase": "core", "year": 2017, "summary_zh": "前向单目相机的视差几乎为零,导致深度尺度无法从几何唯一确定,任何深度预测都依赖学习到的先验。这一根本歧义解释了为什么\"完全去掉激光雷达\"在前向场景中始终困难,促使了多相机环视、毫米波辅助、时间累积等多种间接解。"}, + + {"id": "insight:multi_view_geometry_as_free_supervision", "label_en": "Multi-view geometry is a free supervision signal", "label_zh": "多视图几何即是一种免费的监督信号", "kind": "insight", "tier": "insight", "topic": "geometry_3d", "phase": "core", "year": 2017, "summary_zh": "只要采集时具备多视角或多时刻,几何一致性(光度、视差、流、深度)就免费提供大量自监督信号,无需任何人工标注。这一观察跨越深度估计、辐射场、自监督表征,是把\"采集数据\"和\"标注数据\"解耦的根本支点。"}, + + {"id": "insight:foundation_features_transfer_without_finetune", "label_en": "Foundation model features often transfer without finetuning", "label_zh": "基础模型的特征通常无需微调即可迁移", "kind": "insight", "tier": "insight", "topic": "ssl_vision", "phase": "core", "year": 2023, "summary_zh": "大规模自监督预训练得到的特征常常在下游任务上线性分类即可达到强基线,甚至冻结后即可直接使用。这一现象使\"训练通用基座 + 冻结特征 + 轻量适配\"成为成本-性能最佳折中,是当前感知系统设计的主导原则之一。"}, + + {"id": "insight:occupancy_unifies_static_and_dynamic_scene", "label_en": "Occupancy fields unify static and dynamic scene representation", "label_zh": "占用场可以统一表示静态与动态场景", "kind": "insight", "tier": "insight", "topic": "scene_understanding", "phase": "frontier", "year": 2022, "summary_zh": "把场景表示成\"每个体素的占用 + 流速\"自然涵盖了刚体、可变形体、未知类别,既能描述静态街景又能刻画动态参与者。这一统一性使得感知输出可以直接服务于规划、预测、神经渲染,是占用范式相对检测框范式的最深结构性优势。"}, + + {"id": "insight:open_vocabulary_via_language_anchoring", "label_en": "Open vocabulary perception via language anchoring", "label_zh": "通过语言锚定实现开放词表感知", "kind": "insight", "tier": "insight", "topic": "ssl_vision", "phase": "core", "year": 2021, "summary_zh": "把\"类别\"这个固定离散符号替换为\"自然语言描述\",感知模型的可识别概念集合就可以随时随地扩展。这一思路统一了零样本检测、分割、属性识别,并把感知接口和大语言模型天然连通。"}, + + {"id": "insight:implicit_vs_explicit_is_a_continuum", "label_en": "Implicit versus explicit representation is a continuum", "label_zh": "隐式与显式表示是一个连续光谱", "kind": "insight", "tier": "insight", "topic": "scene_understanding", "phase": "frontier", "year": 2023, "summary_zh": "纯隐式(MLP)和纯显式(点、网格)只是连续光谱的两端,中间有体素哈希、各向异性高斯、张量分解等无穷多种混合形式。研究者应根据具体任务对可微性、可编辑性、渲染速度、表达力的需求,在该光谱上选择合适的折中,而不是把两者当作二元选择。"}, + + {"id": "insight:bev_is_planning_friendly_intermediate", "label_en": "BEV is the planning-friendly intermediate representation", "label_zh": "鸟瞰图是对规划最友好的中间表示", "kind": "insight", "tier": "insight", "topic": "e2e_ad", "phase": "core", "year": 2022, "summary_zh": "鸟瞰图保留了道路布局、可通行区域、对象位置的真实尺度,同时去掉了透视下的纵深扭曲,使得后续路径采样、代价场计算都可以在欧氏空间中直接进行。它之所以在自动驾驶中流行不是因为感知最准,而是因为它是接口最适配规划的中间表示。"}, + + {"id": "insight:temporal_aggregation_buys_what_depth_sensor_buys", "label_en": "Temporal aggregation buys what a depth sensor would have bought", "label_zh": "时序聚合能换取深度传感器所能换取的", "kind": "insight", "tier": "insight", "topic": "geometry_3d", "phase": "core", "year": 2022, "summary_zh": "运动带来的视差等价于一个虚拟的立体相机基线,因此\"看更多帧\"在很大程度上替代了\"装一个激光雷达\"。这一等价性是纯相机方案能与激光雷达方案在同一基准上比较的根本前提。"}, + + {"id": "insight:differentiable_rendering_is_universal_inverse_solver", "label_en": "Differentiable rendering is a universal inverse-problem solver", "label_zh": "可微渲染是一种通用的反问题求解器", "kind": "insight", "tier": "insight", "topic": "scene_understanding", "phase": "core", "year": 2020, "summary_zh": "凡是能写成\"场景 → 图像\"前向过程的问题,只要把这个过程做成可微,就可以通过反向传播从图像反推场景参数。这一观点把传统计算机视觉里许多孤立的反问题(深度、形状、光照、相机)统一进同一个框架,极大地拓展了可学习视觉的边界。"}, + + {"id": "paradigm:camera_first_autonomy", "label_en": "Camera-first autonomy paradigm", "label_zh": "相机优先的自动驾驶范式", "kind": "paradigm", "tier": "paradigm", "topic": "geometry_3d", "phase": "core", "year": 2021, "summary_zh": "相机优先范式认为:得益于深度学习,纯相机方案在感知能力上正在逼近激光雷达,且成本和可扩展性占优。它驱动了 BEV 系列、占用预测、神经重建等一连串研究浪潮,与\"必须保留激光雷达\"的传统稳健派形成长期张力。"}, + + {"id": "paradigm:neural_scene_reconstruction_as_engine", "label_en": "Neural scene reconstruction as the simulation engine", "label_zh": "把神经场景重建当作仿真引擎", "kind": "paradigm", "tier": "paradigm", "topic": "scene_understanding", "phase": "frontier", "year": 2023, "summary_zh": "这一范式主张:与其手工建模仿真世界,不如把真实驾驶日志神经重建成可重新渲染、可编辑的数字孪生,然后在其中扩增、对抗、闭环。它把\"采集-训练-评估\"三件事统一进同一套表示,正成为头部公司投入最大的方向之一。"} + ], + + "edges": [ + {"source": "paper:lift_splat_shoot", "target": "paper:bevdet", "rel": "prereq"}, + {"source": "paper:bevdet", "target": "paper:bevdet4d", "rel": "extends"}, + {"source": "paper:lift_splat_shoot", "target": "paper:li2022bevformer", "rel": "parallel"}, + {"source": "paper:li2022bevformer", "target": "paper:bevformer_v2", "rel": "extends"}, + {"source": "paper:carion2020", "target": "paper:detr3d", "rel": "prereq"}, + {"source": "paper:detr3d", "target": "paper:petr", "rel": "parallel"}, + {"source": "paper:petr", "target": "paper:petrv2", "rel": "extends"}, + {"source": "paper:petr", "target": "paper:streampetr", "rel": "extends"}, + {"source": "paper:detr3d", "target": "paper:streampetr", "rel": "prereq"}, + {"source": "paper:bevdet", "target": "paper:bevfusion", "rel": "feeds"}, + {"source": "paper:lift_splat_shoot", "target": "paper:bevfusion", "rel": "prereq"}, + {"source": "paper:lift_splat_shoot", "target": "paper:simplebev", "rel": "contrasts"}, + {"source": "paper:simplebev", "target": "paper:bevdet", "rel": "contrasts"}, + + {"source": "paper:occupancy_networks_tesla", "target": "paper:surroundocc", "rel": "feeds"}, + {"source": "paper:tesla_ai_day", "target": "paper:occupancy_networks_tesla", "rel": "covers"}, + {"source": "paper:surroundocc", "target": "paper:occ3d", "rel": "parallel"}, + {"source": "paper:occ3d", "target": "paper:openocc_unic", "rel": "parallel"}, + {"source": "paper:li2022bevformer", "target": "paper:surroundocc", "rel": "prereq"}, + + {"source": "paper:vit", "target": "paper:mae", "rel": "prereq"}, + {"source": "paper:vit", "target": "paper:beit", "rel": "prereq"}, + {"source": "paper:mae", "target": "paper:dinov2", "rel": "parallel"}, + {"source": "paper:beit", "target": "paper:mae", "rel": "parallel"}, + {"source": "paper:vit", "target": "paper:clip", "rel": "prereq"}, + {"source": "paper:clip", "target": "paper:blip2", "rel": "prereq"}, + {"source": "paper:clip", "target": "paper:vilt", "rel": "parallel"}, + {"source": "paper:blip2", "target": "paper:llava", "rel": "prereq"}, + {"source": "paper:vit", "target": "paper:dinov1", "rel": "prereq"}, + {"source": "paper:dinov1", "target": "paper:dinov2", "rel": "prereq"}, + {"source": "paper:simclr_mocov3", "target": "paper:dinov1", "rel": "contrasts"}, + {"source": "paper:simclr_mocov3", "target": "paper:mae", "rel": "contrasts"}, + + {"source": "paper:nerf", "target": "paper:3dgs", "rel": "contrasts"}, + {"source": "paper:nerf", "target": "paper:emernerf", "rel": "prereq"}, + {"source": "paper:3dgs", "target": "paper:drivinggaussian", "rel": "prereq"}, + {"source": "paper:emernerf", "target": "paper:drivinggaussian", "rel": "parallel"}, + + {"source": "paper:dinov2", "target": "paper:depth_anything", "rel": "prereq"}, + {"source": "paper:depth_anything", "target": "paper:lift_splat_shoot", "rel": "feeds"}, + {"source": "paper:depth_anything", "target": "paper:bevdet", "rel": "feeds"}, + + {"source": "paper:vit", "target": "paper:vggt", "rel": "prereq"}, + {"source": "paper:vggt", "target": "paper:3dgs", "rel": "feeds"}, + + {"source": "paper:li2022bevformer", "target": "paper:2212.10156", "rel": "feeds"}, + {"source": "paper:bevfusion", "target": "paper:2212.10156", "rel": "feeds"}, + {"source": "paper:streampetr", "target": "paper:2212.10156", "rel": "feeds"}, + {"source": "paper:occupancy_networks_tesla", "target": "paper:2212.10156", "rel": "feeds"}, + {"source": "paper:occupancy_networks_tesla", "target": "paper:vadv2", "rel": "feeds"}, + + {"source": "paper:dinov2", "target": "paper:surroundocc", "rel": "feeds"}, + {"source": "paper:2508.10104", "target": "paper:surroundocc", "rel": "feeds"}, + {"source": "paper:sam", "target": "paper:depth_anything", "rel": "parallel"}, + {"source": "paper:clip", "target": "paper:llava", "rel": "prereq"}, + + {"source": "paper:lift_splat_shoot", "target": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "rel": "covers"}, + {"source": "paper:bevdet", "target": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "rel": "covers"}, + {"source": "paper:bevfusion", "target": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "rel": "covers"}, + {"source": "paper:carion2020", "target": "move:treat_detection_as_set_prediction_with_learnable_queries", "rel": "covers"}, + {"source": "paper:detr3d", "target": "move:treat_detection_as_set_prediction_with_learnable_queries", "rel": "covers"}, + {"source": "paper:2212.10156", "target": "move:treat_detection_as_set_prediction_with_learnable_queries", "rel": "covers"}, + {"source": "paper:detr3d", "target": "move:reproject_3d_query_to_2d_for_feature_sampling", "rel": "covers"}, + {"source": "paper:streampetr", "target": "move:reproject_3d_query_to_2d_for_feature_sampling", "rel": "covers"}, + {"source": "paper:petr", "target": "move:embed_camera_geometry_into_positional_encoding", "rel": "covers"}, + {"source": "paper:petrv2", "target": "move:embed_camera_geometry_into_positional_encoding", "rel": "covers"}, + {"source": "paper:nerf", "target": "move:replace_explicit_module_with_implicit_function", "rel": "covers"}, + {"source": "paper:3dgs", "target": "move:swap_implicit_for_explicit_primitives_when_compute_allows", "rel": "covers"}, + {"source": "paper:bevformer_v2", "target": "move:add_auxiliary_perspective_supervision_to_bev", "rel": "covers"}, + {"source": "paper:streampetr", "target": "move:carry_object_query_across_time_as_recurrent_state", "rel": "covers"}, + {"source": "paper:bevdet4d", "target": "move:carry_object_query_across_time_as_recurrent_state", "rel": "covers"}, + {"source": "paper:bevfusion", "target": "move:fuse_modalities_in_shared_intermediate_space", "rel": "covers"}, + {"source": "paper:occupancy_networks_tesla", "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", "rel": "covers"}, + {"source": "paper:surroundocc", "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", "rel": "covers"}, + {"source": "paper:mae", "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", "rel": "covers"}, + {"source": "paper:dinov2", "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", "rel": "covers"}, + {"source": "paper:2508.10104", "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", "rel": "covers"}, + {"source": "paper:blip2", "target": "move:freeze_giant_backbone_train_small_adapter", "rel": "covers"}, + {"source": "paper:llava", "target": "move:freeze_giant_backbone_train_small_adapter", "rel": "covers"}, + {"source": "paper:vit", "target": "move:tokenize_continuous_signal_to_use_transformer", "rel": "covers"}, + {"source": "paper:beit", "target": "move:tokenize_continuous_signal_to_use_transformer", "rel": "covers"}, + {"source": "paper:nerf", "target": "move:use_geometry_as_self_supervision", "rel": "covers"}, + {"source": "paper:emernerf", "target": "move:use_geometry_as_self_supervision", "rel": "covers"}, + {"source": "paper:2212.10156", "target": "move:make_pipeline_differentiable_via_shared_latent", "rel": "covers"}, + {"source": "paper:vadv2", "target": "move:make_pipeline_differentiable_via_shared_latent", "rel": "covers"}, + {"source": "paper:nerf", "target": "move:rasterize_differentiable_renderer_for_inverse_problem", "rel": "covers"}, + {"source": "paper:3dgs", "target": "move:rasterize_differentiable_renderer_for_inverse_problem", "rel": "covers"}, + {"source": "paper:depth_anything", "target": "move:distill_internet_data_into_small_specialist", "rel": "covers"}, + {"source": "paper:sam", "target": "move:distill_internet_data_into_small_specialist", "rel": "covers"}, + {"source": "paper:bevdet4d", "target": "move:make_camera_only_temporal_match_lidar", "rel": "covers"}, + {"source": "paper:streampetr", "target": "move:make_camera_only_temporal_match_lidar", "rel": "covers"}, + {"source": "paper:clip", "target": "move:open_vocabulary_via_text_alignment", "rel": "covers"}, + {"source": "paper:dinov1", "target": "move:emergent_segmentation_from_self_distillation", "rel": "covers"}, + {"source": "paper:dinov2", "target": "move:emergent_segmentation_from_self_distillation", "rel": "covers"}, + {"source": "paper:vggt", "target": "move:replace_handcrafted_sfm_with_feedforward_transformer", "rel": "covers"}, + {"source": "paper:emernerf", "target": "move:decompose_scene_into_static_and_dynamic_streams", "rel": "covers"}, + {"source": "paper:drivinggaussian", "target": "move:decompose_scene_into_static_and_dynamic_streams", "rel": "covers"}, + {"source": "paper:drivinggaussian", "target": "move:bridge_sim_and_real_via_neural_reconstruction", "rel": "covers"}, + {"source": "paper:emernerf", "target": "move:bridge_sim_and_real_via_neural_reconstruction", "rel": "covers"}, + {"source": "paper:drivinggaussian", "target": "move:augment_via_counterfactual_object_insertion", "rel": "covers"}, + {"source": "paper:2212.10156", "target": "move:share_queries_across_multiple_tasks", "rel": "covers"}, + {"source": "paper:petrv2", "target": "move:share_queries_across_multiple_tasks", "rel": "covers"}, + {"source": "paper:gaia1", "target": "move:learn_motion_in_latent_space_then_decode", "rel": "covers"}, + {"source": "paper:drivedreamer", "target": "move:learn_motion_in_latent_space_then_decode", "rel": "covers"}, + {"source": "paper:occ3d", "target": "move:use_visibility_mask_to_filter_supervision", "rel": "covers"}, + + {"source": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "target": "paper:lift_splat_shoot", "rel": "manifests"}, + {"source": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "target": "paper:bevdet", "rel": "manifests"}, + {"source": "move:embed_camera_geometry_into_positional_encoding", "target": "paper:petr", "rel": "manifests"}, + {"source": "move:replace_class_specific_box_with_class_agnostic_occupancy", "target": "paper:occupancy_networks_tesla", "rel": "manifests"}, + {"source": "move:carry_object_query_across_time_as_recurrent_state", "target": "paper:streampetr", "rel": "manifests"}, + {"source": "move:share_queries_across_multiple_tasks", "target": "paper:2212.10156", "rel": "manifests"}, + + {"source": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "target": "paper:bevfusion", "rel": "composes"}, + {"source": "move:fuse_modalities_in_shared_intermediate_space", "target": "paper:bevfusion", "rel": "composes"}, + {"source": "move:treat_detection_as_set_prediction_with_learnable_queries", "target": "paper:detr3d", "rel": "composes"}, + {"source": "move:reproject_3d_query_to_2d_for_feature_sampling", "target": "paper:detr3d", "rel": "composes"}, + {"source": "move:treat_detection_as_set_prediction_with_learnable_queries", "target": "paper:petr", "rel": "composes"}, + {"source": "move:embed_camera_geometry_into_positional_encoding", "target": "paper:petr", "rel": "composes"}, + {"source": "move:treat_detection_as_set_prediction_with_learnable_queries", "target": "paper:2212.10156", "rel": "composes"}, + {"source": "move:share_queries_across_multiple_tasks", "target": "paper:2212.10156", "rel": "composes"}, + {"source": "move:make_pipeline_differentiable_via_shared_latent", "target": "paper:2212.10156", "rel": "composes"}, + {"source": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "target": "paper:li2022bevformer", "rel": "composes"}, + {"source": "move:treat_detection_as_set_prediction_with_learnable_queries", "target": "paper:li2022bevformer", "rel": "composes"}, + {"source": "move:add_auxiliary_perspective_supervision_to_bev", "target": "paper:bevformer_v2", "rel": "composes"}, + {"source": "move:replace_explicit_module_with_implicit_function", "target": "paper:nerf", "rel": "composes"}, + {"source": "move:rasterize_differentiable_renderer_for_inverse_problem", "target": "paper:nerf", "rel": "composes"}, + {"source": "move:swap_implicit_for_explicit_primitives_when_compute_allows", "target": "paper:3dgs", "rel": "composes"}, + {"source": "move:decompose_scene_into_static_and_dynamic_streams", "target": "paper:emernerf", "rel": "composes"}, + {"source": "move:bridge_sim_and_real_via_neural_reconstruction", "target": "paper:drivinggaussian", "rel": "composes"}, + {"source": "move:carry_object_query_across_time_as_recurrent_state", "target": "paper:streampetr", "rel": "composes"}, + {"source": "move:make_camera_only_temporal_match_lidar", "target": "paper:bevdet4d", "rel": "composes"}, + + {"source": "problem:long_tail_object_categories_in_open_world", "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", "rel": "motivates"}, + {"source": "problem:long_tail_object_categories_in_open_world", "target": "move:open_vocabulary_via_text_alignment", "rel": "motivates"}, + {"source": "problem:long_tail_object_categories_in_open_world", "target": "move:augment_via_counterfactual_object_insertion", "rel": "motivates"}, + {"source": "problem:sim_to_real_gap_in_camera_only_perception", "target": "move:bridge_sim_and_real_via_neural_reconstruction", "rel": "motivates"}, + {"source": "problem:temporal_consistency_in_bev_segmentation", "target": "move:carry_object_query_across_time_as_recurrent_state", "rel": "motivates"}, + {"source": "problem:occlusion_reasoning_without_dense_lidar", "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", "rel": "motivates"}, + {"source": "problem:occlusion_reasoning_without_dense_lidar", "target": "move:learn_motion_in_latent_space_then_decode", "rel": "motivates"}, + {"source": "problem:label_efficiency_for_3d_annotation", "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", "rel": "motivates"}, + {"source": "problem:label_efficiency_for_3d_annotation", "target": "move:use_geometry_as_self_supervision", "rel": "motivates"}, + {"source": "problem:label_efficiency_for_3d_annotation", "target": "move:distill_internet_data_into_small_specialist", "rel": "motivates"}, + {"source": "problem:unknown_geometry_in_distant_or_dark_regions", "target": "move:make_camera_only_temporal_match_lidar", "rel": "motivates"}, + {"source": "problem:multi_modal_calibration_drift", "target": "move:fuse_modalities_in_shared_intermediate_space", "rel": "motivates"}, + {"source": "problem:rendering_speed_vs_quality_tradeoff", "target": "move:swap_implicit_for_explicit_primitives_when_compute_allows", "rel": "motivates"}, + {"source": "problem:catastrophic_failure_on_rare_weather", "target": "move:augment_via_counterfactual_object_insertion", "rel": "motivates"}, + {"source": "problem:annotation_inconsistency_across_datasets", "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", "rel": "motivates"}, + {"source": "problem:depth_ambiguity_in_low_parallax", "target": "move:make_camera_only_temporal_match_lidar", "rel": "motivates"}, + + {"source": "problem:long_tail_object_categories_in_open_world", "target": "paper:detr3d", "rel": "unsolved_by"}, + {"source": "problem:long_tail_object_categories_in_open_world", "target": "paper:bevdet", "rel": "unsolved_by"}, + {"source": "problem:occlusion_reasoning_without_dense_lidar", "target": "paper:detr3d", "rel": "unsolved_by"}, + {"source": "problem:catastrophic_failure_on_rare_weather", "target": "paper:bevfusion", "rel": "unsolved_by"}, + {"source": "problem:depth_ambiguity_in_low_parallax", "target": "paper:lift_splat_shoot", "rel": "unsolved_by"}, + {"source": "problem:rendering_speed_vs_quality_tradeoff", "target": "paper:nerf", "rel": "unsolved_by"}, + {"source": "problem:annotation_inconsistency_across_datasets", "target": "paper:occ3d", "rel": "unsolved_by"}, + + {"source": "insight:multi_view_geometry_as_free_supervision", "target": "paper:nerf", "rel": "manifests"}, + {"source": "insight:multi_view_geometry_as_free_supervision", "target": "paper:emernerf", "rel": "manifests"}, + {"source": "insight:multi_view_geometry_as_free_supervision", "target": "paper:depth_anything", "rel": "manifests"}, + {"source": "insight:foundation_features_transfer_without_finetune", "target": "paper:dinov2", "rel": "manifests"}, + {"source": "insight:foundation_features_transfer_without_finetune", "target": "paper:2508.10104", "rel": "manifests"}, + {"source": "insight:foundation_features_transfer_without_finetune", "target": "paper:blip2", "rel": "manifests"}, + {"source": "insight:occupancy_unifies_static_and_dynamic_scene", "target": "paper:occupancy_networks_tesla", "rel": "manifests"}, + {"source": "insight:occupancy_unifies_static_and_dynamic_scene", "target": "paper:surroundocc", "rel": "manifests"}, + {"source": "insight:open_vocabulary_via_language_anchoring", "target": "paper:clip", "rel": "manifests"}, + {"source": "insight:open_vocabulary_via_language_anchoring", "target": "paper:llava", "rel": "manifests"}, + {"source": "insight:implicit_vs_explicit_is_a_continuum", "target": "paper:nerf", "rel": "manifests"}, + {"source": "insight:implicit_vs_explicit_is_a_continuum", "target": "paper:3dgs", "rel": "manifests"}, + {"source": "insight:bev_is_planning_friendly_intermediate", "target": "paper:li2022bevformer", "rel": "manifests"}, + {"source": "insight:bev_is_planning_friendly_intermediate", "target": "paper:bevfusion", "rel": "manifests"}, + {"source": "insight:bev_is_planning_friendly_intermediate", "target": "paper:2212.10156", "rel": "manifests"}, + {"source": "insight:temporal_aggregation_buys_what_depth_sensor_buys", "target": "paper:bevdet4d", "rel": "manifests"}, + {"source": "insight:temporal_aggregation_buys_what_depth_sensor_buys", "target": "paper:streampetr", "rel": "manifests"}, + {"source": "insight:differentiable_rendering_is_universal_inverse_solver", "target": "paper:nerf", "rel": "manifests"}, + {"source": "insight:differentiable_rendering_is_universal_inverse_solver", "target": "paper:3dgs", "rel": "manifests"}, + {"source": "insight:differentiable_rendering_is_universal_inverse_solver", "target": "paper:drivinggaussian", "rel": "manifests"}, + + {"source": "insight:multi_view_geometry_as_free_supervision", "target": "move:use_geometry_as_self_supervision", "rel": "enables"}, + {"source": "insight:foundation_features_transfer_without_finetune", "target": "move:freeze_giant_backbone_train_small_adapter", "rel": "enables"}, + {"source": "insight:foundation_features_transfer_without_finetune", "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", "rel": "enables"}, + {"source": "insight:occupancy_unifies_static_and_dynamic_scene", "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", "rel": "enables"}, + {"source": "insight:open_vocabulary_via_language_anchoring", "target": "move:open_vocabulary_via_text_alignment", "rel": "enables"}, + {"source": "insight:implicit_vs_explicit_is_a_continuum", "target": "move:swap_implicit_for_explicit_primitives_when_compute_allows", "rel": "enables"}, + {"source": "insight:bev_is_planning_friendly_intermediate", "target": "move:lift_2d_features_to_3d_via_learned_depth_distribution", "rel": "enables"}, + {"source": "insight:bev_is_planning_friendly_intermediate", "target": "move:fuse_modalities_in_shared_intermediate_space", "rel": "enables"}, + {"source": "insight:temporal_aggregation_buys_what_depth_sensor_buys", "target": "move:make_camera_only_temporal_match_lidar", "rel": "enables"}, + {"source": "insight:temporal_aggregation_buys_what_depth_sensor_buys", "target": "move:carry_object_query_across_time_as_recurrent_state", "rel": "enables"}, + {"source": "insight:differentiable_rendering_is_universal_inverse_solver", "target": "move:rasterize_differentiable_renderer_for_inverse_problem", "rel": "enables"}, + {"source": "insight:differentiable_rendering_is_universal_inverse_solver", "target": "move:replace_explicit_module_with_implicit_function", "rel": "enables"}, + + {"source": "paradigm:camera_first_autonomy", "target": "insight:temporal_aggregation_buys_what_depth_sensor_buys", "rel": "composes"}, + {"source": "paradigm:camera_first_autonomy", "target": "insight:bev_is_planning_friendly_intermediate", "rel": "composes"}, + {"source": "paradigm:camera_first_autonomy", "target": "paper:lift_splat_shoot", "rel": "manifests"}, + {"source": "paradigm:camera_first_autonomy", "target": "paper:occupancy_networks_tesla", "rel": "manifests"}, + {"source": "paradigm:camera_first_autonomy", "target": "paper:tesla_ai_day", "rel": "manifests"}, + {"source": "paradigm:neural_scene_reconstruction_as_engine", "target": "insight:differentiable_rendering_is_universal_inverse_solver", "rel": "composes"}, + {"source": "paradigm:neural_scene_reconstruction_as_engine", "target": "move:bridge_sim_and_real_via_neural_reconstruction", "rel": "composes"}, + {"source": "paradigm:neural_scene_reconstruction_as_engine", "target": "paper:drivinggaussian", "rel": "manifests"}, + {"source": "paradigm:neural_scene_reconstruction_as_engine", "target": "paper:emernerf", "rel": "manifests"}, + {"source": "paradigm:neural_scene_reconstruction_as_engine", "target": "paper:gaia1", "rel": "parallel"}, + + {"source": "essay:bitter_lesson", "target": "move:replace_handcrafted_sfm_with_feedforward_transformer", "rel": "covers"}, + {"source": "essay:bitter_lesson", "target": "move:distill_internet_data_into_small_specialist", "rel": "covers"}, + {"source": "essay:bitter_lesson", "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", "rel": "covers"}, + + {"source": "concept:bev", "target": "paper:lift_splat_shoot", "rel": "covers"}, + {"source": "concept:bev", "target": "paper:bevfusion", "rel": "covers"}, + {"source": "concept:bev", "target": "insight:bev_is_planning_friendly_intermediate", "rel": "covers"}, + {"source": "concept:detr_query", "target": "paper:detr3d", "rel": "covers"}, + {"source": "concept:detr_query", "target": "move:treat_detection_as_set_prediction_with_learnable_queries", "rel": "covers"}, + {"source": "concept:ssl", "target": "paper:mae", "rel": "covers"}, + {"source": "concept:ssl", "target": "paper:dinov1", "rel": "covers"}, + {"source": "concept:ssl", "target": "paper:simclr_mocov3", "rel": "covers"}, + {"source": "concept:ssl", "target": "paper:clip", "rel": "covers"} + ] +} diff --git a/docs/data/graph_extended.json b/docs/data/graph_extended.json new file mode 100644 index 0000000..ea3890b --- /dev/null +++ b/docs/data/graph_extended.json @@ -0,0 +1,12874 @@ +{ + "$comment": "Merged research-insight atlas. Source: docs/data/graph.json + docs/data/generated/*.json. Regenerate with tools/merge_graph.py.", + "nodes": [ + { + "id": "paper:2212.10156", + "label": "UniAD", + "label_zh": "UniAD(统一规划导向 AD)", + "kind": "paper", + "tier": "spine", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "card": "paper_2212.10156_uniad.md", + "labs": [ + "lab:lab03" + ], + "degree": 25 + }, + { + "id": "paper:2210.14222", + "label": "PlanT", + "label_zh": "PlanT(对象级规划 transformer)", + "kind": "paper", + "tier": "spine", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "card": "paper_2210.14222_plant.md", + "labs": [ + "lab:lab04" + ], + "degree": 11 + }, + { + "id": "paper:2402.12289", + "label": "DriveVLM", + "label_zh": "DriveVLM / DriveVLM-Dual", + "kind": "paper", + "tier": "spine", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "card": "paper_2402.12289_drivevlm.md", + "labs": [ + "lab:lab09" + ], + "degree": 25 + }, + { + "id": "paper:2311.10813", + "label": "Agent-Driver", + "label_zh": "Agent-Driver(LLM 认知 agent)", + "kind": "paper", + "tier": "spine", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "card": "paper_2311.10813_agent_driver.md", + "labs": [ + "lab:lab08" + ], + "degree": 21 + }, + { + "id": "paper:2309.16292", + "label": "DiLu", + "label_zh": "DiLu(知识驱动 + 反思)", + "kind": "paper", + "tier": "spine", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "card": "paper_2309.16292_dilu.md", + "labs": [ + "lab:lab07" + ], + "degree": 15 + }, + { + "id": "paper:2307.01694", + "label": "Spike-driven Transformer", + "label_zh": "Spike-driven Transformer", + "kind": "paper", + "tier": "spine", + "topic": "brain_inspired", + "phase": "frontier", + "year": 2023, + "card": "paper_2307.01694_spike_driven_transformer.md", + "labs": [ + "lab:lab06" + ], + "degree": 15 + }, + { + "id": "paper:2508.10104", + "label": "DINOv3", + "label_zh": "DINOv3", + "kind": "paper", + "tier": "spine", + "topic": "ssl_vision", + "phase": "frontier", + "year": 2025, + "card": "paper_2508.10104_dinov3.md", + "labs": [ + "lab:lab05" + ], + "degree": 13 + }, + { + "id": "paper:2512.24426", + "label": "CF-VLA", + "label_zh": "CF-VLA(反事实 VLA)", + "kind": "paper", + "tier": "spine", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2025, + "card": "paper_2512.24426_cfvla.md", + "labs": [ + "lab:lab10" + ], + "degree": 23 + }, + { + "id": "channel:3blue1brown", + "label": "3Blue1Brown", + "label_zh": "3Blue1Brown(可视化数学)", + "kind": "channel", + "tier": "spine", + "topic": "math_foundations", + "phase": "prereq", + "year": 2015, + "card": "channel_3blue1brown.md", + "degree": 3 + }, + { + "id": "channel:mu_li_bilibili", + "label": "跟李沐学AI", + "label_zh": "跟李沐学AI(B 站论文精读)", + "kind": "channel", + "tier": "spine", + "topic": "companion_media", + "phase": "prereq", + "year": 2020, + "card": "channel_mu_li_bilibili.md", + "degree": 6 + }, + { + "id": "channel:ez_encoder_academy", + "label": "EZ.Encoder", + "label_zh": "EZ.Encoder Academy", + "kind": "channel", + "tier": "spine", + "topic": "companion_media", + "phase": "prereq", + "year": 2023, + "card": "channel_ez_encoder_academy.md", + "degree": 2 + }, + { + "id": "course:zhao_rl", + "label": "Zhao Shiyu RL", + "label_zh": "赵世钰 强化学习的数学原理", + "kind": "course", + "tier": "spine", + "topic": "rl_foundations", + "phase": "prereq", + "year": 2024, + "card": "course_zhao_shiyu_rl.md", + "labs": [ + "lab:lab01" + ], + "degree": 13 + }, + { + "id": "course:cs285", + "label": "CS285 Deep RL", + "label_zh": "Berkeley CS285 深度强化学习", + "kind": "course", + "tier": "spine", + "topic": "deep_rl", + "phase": "core", + "year": 2023, + "card": "course_cs285_levine.md", + "labs": [ + "lab:lab02" + ], + "degree": 11 + }, + { + "id": "essay:bitter_lesson", + "label": "Bitter Lesson", + "label_zh": "苦涩的教训(Sutton)", + "kind": "essay", + "tier": "spine", + "topic": "meta_philosophy", + "phase": "prereq", + "year": 2019, + "card": "essay_bitter_lesson.md", + "degree": 28 + }, + { + "id": "paper:vaswani2017", + "label": "Transformer", + "label_zh": "Attention Is All You Need", + "kind": "paper", + "tier": "S", + "topic": "math_foundations", + "phase": "prereq", + "year": 2017, + "card": "paper_vaswani2017.md", + "degree": 25 + }, + { + "id": "paper:vit", + "label": "ViT", + "label_zh": "ViT(图像 transformer)", + "kind": "paper", + "tier": "S", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2020, + "card": "paper_vit.md", + "degree": 21 + }, + { + "id": "paper:carion2020", + "label": "DETR", + "label_zh": "DETR(端到端检测)", + "kind": "paper", + "tier": "S", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2020, + "card": "paper_carion2020.md", + "degree": 14 + }, + { + "id": "paper:he2015_resnet", + "label": "ResNet", + "label_zh": "ResNet(残差网络)", + "kind": "paper", + "tier": "S", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2015, + "card": "paper_he2015_resnet.md", + "degree": 6 + }, + { + "id": "paper:gpt3", + "label": "GPT-3", + "label_zh": "GPT-3(few-shot 大语言模型)", + "kind": "paper", + "tier": "S", + "topic": "vlm_vla", + "phase": "prereq", + "year": 2020, + "card": "paper_gpt3.md", + "degree": 26 + }, + { + "id": "paper:schulman2017_ppo", + "label": "PPO", + "label_zh": "PPO(近端策略优化)", + "kind": "paper", + "tier": "S", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "card": "paper_schulman2017_ppo.md", + "degree": 12 + }, + { + "id": "paper:mnih2015_dqn", + "label": "DQN", + "label_zh": "DQN(深度 Q 网络)", + "kind": "paper", + "tier": "S", + "topic": "deep_rl", + "phase": "prereq", + "year": 2015, + "card": "paper_mnih2015_dqn.md", + "degree": 11 + }, + { + "id": "paper:ross2011_dagger", + "label": "DAgger", + "label_zh": "DAgger(解决协变量偏移)", + "kind": "paper", + "tier": "S", + "topic": "deep_rl", + "phase": "core", + "year": 2011, + "card": "paper_ross2011_dagger.md", + "labs": [ + "lab:lab02" + ], + "degree": 9 + }, + { + "id": "paper:silver2017_alphazero", + "label": "AlphaGo Zero", + "label_zh": "AlphaGo Zero / AlphaZero", + "kind": "paper", + "tier": "S", + "topic": "deep_rl", + "phase": "prereq", + "year": 2017, + "card": "paper_silver2017_alphazero.md", + "degree": 8 + }, + { + "id": "paper:li2022bevformer", + "label": "BEVFormer", + "label_zh": "BEVFormer(时空 BEV transformer)", + "kind": "paper", + "tier": "S", + "topic": "e2e_ad", + "phase": "prereq", + "year": 2022, + "card": "paper_li2022bevformer.md", + "degree": 17 + }, + { + "id": "paper:vadv2", + "label": "VADv2", + "label_zh": "VADv2(向量化端到端 + 概率规划)", + "kind": "paper", + "tier": "A", + "topic": "e2e_ad", + "phase": "frontier", + "year": 2024, + "card": "paper_vadv2.md", + "degree": 10 + }, + { + "id": "paper:transfuser", + "label": "TransFuser", + "label_zh": "TransFuser(多模态融合 BC)", + "kind": "paper", + "tier": "A", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "card": "paper_transfuser.md", + "degree": 11 + }, + { + "id": "paper:llava", + "label": "LLaVA / Qwen-VL", + "label_zh": "LLaVA / Qwen-VL(开源 VLM)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "prereq", + "year": 2023, + "card": "paper_llava.md", + "degree": 26 + }, + { + "id": "paper:gaia1", + "label": "GAIA-1", + "label_zh": "GAIA-1(生成式驾驶世界模型)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "card": "paper_gaia1.md", + "degree": 20 + }, + { + "id": "paper:drivedreamer", + "label": "DriveDreamer", + "label_zh": "DriveDreamer(视频扩散世界模型)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "card": "paper_drivedreamer.md", + "degree": 14 + }, + { + "id": "paper:dinov2", + "label": "DINOv2", + "label_zh": "DINOv2", + "kind": "paper", + "tier": "A", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2023, + "card": "paper_dinov2.md", + "degree": 15 + }, + { + "id": "paper:sam", + "label": "SAM", + "label_zh": "SAM / SAM 2(分割万物)", + "kind": "paper", + "tier": "A", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2023, + "card": "paper_sam.md", + "degree": 5 + }, + { + "id": "paper:sutton_barto", + "label": "Sutton & Barto RL", + "label_zh": "Sutton & Barto《强化学习导论》", + "kind": "paper", + "tier": "A", + "topic": "rl_foundations", + "phase": "prereq", + "year": 2018, + "card": "paper_sutton_barto.md", + "degree": 4 + }, + { + "id": "paper:rlhf_dpo", + "label": "RLHF / DPO", + "label_zh": "RLHF / DPO(人类偏好对齐)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "core", + "year": 2023, + "card": "paper_rlhf_dpo.md", + "degree": 14 + }, + { + "id": "paper:world_models", + "label": "World Models", + "label_zh": "World Models(Ha & Schmidhuber)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "core", + "year": 2018, + "card": "paper_world_models.md", + "degree": 19 + }, + { + "id": "paper:mamba", + "label": "Mamba", + "label_zh": "Mamba(状态空间模型)", + "kind": "paper", + "tier": "B", + "topic": "math_foundations", + "phase": "frontier", + "year": 2023, + "card": "paper_mamba.md", + "degree": 4 + }, + { + "id": "paper:diffuser", + "label": "Diffuser", + "label_zh": "Diffuser / Decision Diffuser", + "kind": "paper", + "tier": "B", + "topic": "deep_rl", + "phase": "frontier", + "year": 2022, + "card": "paper_diffuser.md", + "degree": 13 + }, + { + "id": "paper:ad_benchmarks", + "label": "AD benchmarks", + "label_zh": "CARLA / nuScenes / NAVSIM / Bench2Drive", + "kind": "paper", + "tier": "B", + "topic": "e2e_ad", + "phase": "prereq", + "year": 2017, + "card": "benchmarks_ad.md", + "degree": 18 + }, + { + "id": "paper:lingo2", + "label": "Wayve LINGO-2", + "label_zh": "Wayve LINGO-2", + "kind": "paper", + "tier": "B", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "card": "paper_lingo2.md", + "degree": 5 + }, + { + "id": "paper:tesla_ai_day", + "label": "Tesla AI Day", + "label_zh": "Tesla AI Day", + "kind": "paper", + "tier": "B", + "topic": "e2e_ad", + "phase": "prereq", + "year": 2022, + "card": "paper_tesla_ai_day.md", + "degree": 6 + }, + { + "id": "concept:mdp", + "label": "MDP", + "label_zh": "马尔可夫决策过程 MDP", + "kind": "concept", + "tier": "concept", + "topic": "rl_foundations", + "phase": "prereq", + "year": 1957, + "card": "../../../concepts.md#mdp", + "degree": 6 + }, + { + "id": "concept:bellman_eq", + "label": "Bellman 方程", + "label_zh": "Bellman 方程 / 最优方程", + "kind": "concept", + "tier": "concept", + "topic": "rl_foundations", + "phase": "prereq", + "year": 1957, + "card": "../../../concepts.md#bellman方程", + "degree": 5 + }, + { + "id": "concept:value_iteration", + "label": "Value Iteration", + "label_zh": "值迭代 / 策略迭代", + "kind": "concept", + "tier": "concept", + "topic": "rl_foundations", + "phase": "prereq", + "year": 1957, + "card": "../../../concepts.md#值迭代--策略迭代", + "degree": 2 + }, + { + "id": "concept:td_learning", + "label": "TD Learning", + "label_zh": "TD 学习 / Q-learning / SARSA", + "kind": "concept", + "tier": "concept", + "topic": "rl_foundations", + "phase": "prereq", + "year": 1988, + "card": "../../../concepts.md#td学习--q-learning--sarsa", + "degree": 2 + }, + { + "id": "concept:policy_gradient", + "label": "Policy Gradient", + "label_zh": "策略梯度 / REINFORCE", + "kind": "concept", + "tier": "concept", + "topic": "deep_rl", + "phase": "core", + "year": 1992, + "card": "../../../concepts.md#policy-gradient--reinforce", + "degree": 6 + }, + { + "id": "concept:actor_critic", + "label": "Actor-Critic", + "label_zh": "Actor-Critic", + "kind": "concept", + "tier": "concept", + "topic": "deep_rl", + "phase": "core", + "year": 2000, + "card": "../../../concepts.md#actor-critic", + "degree": 11 + }, + { + "id": "concept:ppo", + "label": "PPO 概念", + "label_zh": "PPO 替代目标", + "kind": "concept", + "tier": "concept", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "card": "../../../concepts.md#ppo", + "degree": 4 + }, + { + "id": "concept:dqn", + "label": "DQN 概念", + "label_zh": "DQN / replay buffer / target net", + "kind": "concept", + "tier": "concept", + "topic": "deep_rl", + "phase": "core", + "year": 2015, + "card": "../../../concepts.md#dqn", + "degree": 3 + }, + { + "id": "concept:replay_buffer", + "label": "Replay buffer", + "label_zh": "经验回放", + "kind": "concept", + "tier": "concept", + "topic": "deep_rl", + "phase": "core", + "year": 1992, + "card": "../../../concepts.md#replay-buffer", + "degree": 4 + }, + { + "id": "concept:imitation_learning", + "label": "Imitation Learning", + "label_zh": "模仿学习 / BC", + "kind": "concept", + "tier": "concept", + "topic": "deep_rl", + "phase": "core", + "year": 1989, + "card": "../../../concepts.md#模仿学习--bc", + "degree": 10 + }, + { + "id": "concept:covariate_shift", + "label": "Covariate shift", + "label_zh": "协变量偏移", + "kind": "concept", + "tier": "concept", + "topic": "deep_rl", + "phase": "core", + "year": 2010, + "card": "../../../concepts.md#协变量偏移", + "degree": 3 + }, + { + "id": "concept:transformer", + "label": "Transformer", + "label_zh": "Transformer 架构", + "kind": "concept", + "tier": "concept", + "topic": "math_foundations", + "phase": "prereq", + "year": 2017, + "card": "../../../concepts.md#transformer", + "degree": 10 + }, + { + "id": "concept:self_attention", + "label": "Self-Attention", + "label_zh": "Self-Attention", + "kind": "concept", + "tier": "concept", + "topic": "math_foundations", + "phase": "prereq", + "year": 2017, + "card": "../../../concepts.md#self-attention", + "degree": 9 + }, + { + "id": "concept:detr_query", + "label": "DETR query", + "label_zh": "DETR object query", + "kind": "concept", + "tier": "concept", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2020, + "card": "../../../concepts.md#detr-query", + "degree": 7 + }, + { + "id": "concept:bev", + "label": "BEV", + "label_zh": "BEV 感知", + "kind": "concept", + "tier": "concept", + "topic": "e2e_ad", + "phase": "prereq", + "year": 2020, + "card": "../../../concepts.md#bev感知", + "degree": 8 + }, + { + "id": "concept:vlm", + "label": "VLM", + "label_zh": "Vision-Language Model", + "kind": "concept", + "tier": "concept", + "topic": "vlm_vla", + "phase": "core", + "year": 2021, + "card": "../../../concepts.md#vlm", + "degree": 6 + }, + { + "id": "concept:vla", + "label": "VLA", + "label_zh": "Vision-Language-Action", + "kind": "concept", + "tier": "concept", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "card": "../../../concepts.md#vla", + "degree": 4 + }, + { + "id": "concept:cot", + "label": "Chain-of-Thought", + "label_zh": "Chain-of-Thought 推理", + "kind": "concept", + "tier": "concept", + "topic": "vlm_vla", + "phase": "core", + "year": 2022, + "card": "../../../concepts.md#chain-of-thought", + "degree": 8 + }, + { + "id": "concept:tool_use", + "label": "Tool use", + "label_zh": "Tool use / function calling", + "kind": "concept", + "tier": "concept", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "card": "../../../concepts.md#tool-use", + "degree": 5 + }, + { + "id": "concept:counterfactual", + "label": "Counterfactual reasoning", + "label_zh": "反事实推理", + "kind": "concept", + "tier": "concept", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2025, + "card": "../../../concepts.md#反事实推理", + "degree": 3 + }, + { + "id": "concept:meta_action", + "label": "Meta-action", + "label_zh": "Meta-action / 高层语义动作", + "kind": "concept", + "tier": "concept", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "card": "../../../concepts.md#meta-action", + "degree": 3 + }, + { + "id": "concept:ssl", + "label": "SSL", + "label_zh": "自监督学习", + "kind": "concept", + "tier": "concept", + "topic": "ssl_vision", + "phase": "core", + "year": 2018, + "card": "../../../concepts.md#ssl", + "degree": 11 + }, + { + "id": "concept:spiking_nn", + "label": "Spiking NN", + "label_zh": "脉冲神经网络", + "kind": "concept", + "tier": "concept", + "topic": "brain_inspired", + "phase": "frontier", + "year": 1997, + "card": "../../../concepts.md#脉冲神经网络", + "degree": 8 + }, + { + "id": "concept:rlhf", + "label": "RLHF", + "label_zh": "RLHF / DPO 概念", + "kind": "concept", + "tier": "concept", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "card": "../../../concepts.md#rlhf", + "degree": 1 + }, + { + "id": "concept:scaling_vs_knowledge", + "label": "Scaling vs Knowledge", + "label_zh": "Scaling vs 人工知识", + "kind": "concept", + "tier": "concept", + "topic": "meta_philosophy", + "phase": "prereq", + "year": 2019, + "card": "../../../concepts.md#scaling-vs-人工知识", + "degree": 3 + }, + { + "id": "lab:lab00", + "label": "lab00 sanity", + "label_zh": "lab00 环境检查", + "kind": "lab", + "tier": "lab", + "topic": "math_foundations", + "phase": "prereq", + "year": 2026, + "card": "../../../labs/lab00_environment_check.ipynb", + "degree": 0 + }, + { + "id": "lab:lab01", + "label": "lab01 value iter", + "label_zh": "lab01 值迭代 gridworld", + "kind": "lab", + "tier": "lab", + "topic": "rl_foundations", + "phase": "prereq", + "year": 2026, + "card": "../../../labs/lab01_zhao_value_iteration_gridworld.ipynb", + "degree": 2 + }, + { + "id": "lab:lab02", + "label": "lab02 BC vs DAgger", + "label_zh": "lab02 BC vs DAgger", + "kind": "lab", + "tier": "lab", + "topic": "deep_rl", + "phase": "core", + "year": 2026, + "card": "../../../labs/lab02_cs285_bc_vs_dagger_minicar.ipynb", + "degree": 3 + }, + { + "id": "lab:lab03", + "label": "lab03 UniAD query", + "label_zh": "lab03 UniAD query intuition", + "kind": "lab", + "tier": "lab", + "topic": "e2e_ad", + "phase": "core", + "year": 2026, + "card": "../../../labs/lab03_uniad_query_intuition.ipynb", + "degree": 2 + }, + { + "id": "lab:lab04", + "label": "lab04 PlanT", + "label_zh": "lab04 PlanT object planner", + "kind": "lab", + "tier": "lab", + "topic": "e2e_ad", + "phase": "core", + "year": 2026, + "card": "../../../labs/lab04_plant_object_level_planner.ipynb", + "degree": 2 + }, + { + "id": "lab:lab05", + "label": "lab05 DINOv3", + "label_zh": "lab05 DINOv3 features", + "kind": "lab", + "tier": "lab", + "topic": "ssl_vision", + "phase": "frontier", + "year": 2026, + "card": "../../../labs/lab05_dinov3_features_minidata.ipynb", + "degree": 1 + }, + { + "id": "lab:lab06", + "label": "lab06 Spike attn", + "label_zh": "lab06 Spike-driven attention", + "kind": "lab", + "tier": "lab", + "topic": "brain_inspired", + "phase": "frontier", + "year": 2026, + "card": "../../../labs/lab06_spike_driven_attention_mnist.ipynb", + "degree": 3 + }, + { + "id": "lab:lab07", + "label": "lab07 DiLu loop", + "label_zh": "lab07 DiLu LLM 决策循环", + "kind": "lab", + "tier": "lab", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2026, + "card": "../../../labs/lab07_dilu_llm_decision_loop.ipynb", + "degree": 1 + }, + { + "id": "lab:lab08", + "label": "lab08 Agent-Driver tools", + "label_zh": "lab08 Agent-Driver tool calling", + "kind": "lab", + "tier": "lab", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2026, + "card": "../../../labs/lab08_agent_driver_tool_calling.ipynb", + "degree": 1 + }, + { + "id": "lab:lab09", + "label": "lab09 DriveVLM-Dual", + "label_zh": "lab09 DriveVLM-Dual pipeline", + "kind": "lab", + "tier": "lab", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2026, + "card": "../../../labs/lab09_drivevlm_dual_pipeline.ipynb", + "degree": 1 + }, + { + "id": "lab:lab10", + "label": "lab10 CF-VLA", + "label_zh": "lab10 CF-VLA replanner", + "kind": "lab", + "tier": "lab", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2026, + "card": "../../../labs/lab10_cfvla_counterfactual_replanner.ipynb", + "degree": 1 + }, + { + "id": "paper:muzero", + "label_zh": "MuZero(学得隐式动力学模型 + 蒙特卡洛树搜索)", + "kind": "paper", + "tier": "S", + "topic": "world_models", + "phase": "core", + "year": 2019, + "summary_zh": "MuZero 把基于模型的强化学习推到了不需要事先知道环境规则的程度。它在抽象隐空间里同时学习一个表示网络、一个动力学转移网络和一个预测网络,然后在这个隐空间里跑蒙特卡洛树搜索做规划,从而在围棋、国际象棋、将棋以及雅达利游戏上同时取得当时最强的成绩。", + "label": "MuZero", + "degree": 8 + }, + { + "id": "paper:dreamer_v2", + "label_zh": "DreamerV2(离散隐变量世界模型)", + "kind": "paper", + "tier": "A", + "topic": "world_models", + "phase": "core", + "year": 2020, + "summary_zh": "DreamerV2 在 PlaNet 和 DreamerV1 的基础上把世界模型的隐状态改成离散随机变量,并配合 KL 平衡和直通梯度估计,使得在雅达利套件上仅凭想象中的回放就能训练出与无模型强者相当的策略。它第一次证明纯粹在世界模型内部的想象中训练就可以超过同等数据预算的无模型方法。", + "label": "DreamerV2", + "degree": 8 + }, + { + "id": "paper:dreamer_v3", + "label_zh": "DreamerV3(统一超参的通用世界模型)", + "kind": "paper", + "tier": "S", + "topic": "world_models", + "phase": "frontier", + "year": 2023, + "summary_zh": "DreamerV3 通过对回报、价值和奖励做对称对数变换以及一系列规范化技巧,让同一套超参数无需调参就能跨越雅达利、ProcGen、DMLab、Minecraft 等数十个不同动力学的任务取得领先成绩。它把世界模型方法从精细调参的研究原型变成了一个真正可以照搬使用的通用基线。", + "label": "DreamerV3", + "degree": 7 + }, + { + "id": "paper:iris_world_model", + "label_zh": "IRIS(用离散自编码 + transformer 当世界模型)", + "kind": "paper", + "tier": "A", + "topic": "world_models", + "phase": "frontier", + "year": 2022, + "summary_zh": "IRIS 把世界模型重构成两个模块:一个把图像帧压成离散视觉 token 的 VQ-VAE,以及一个像语言模型一样在 token 序列上自回归预测下一帧和奖励的 transformer。这样想象出来的轨迹细节远好于循环型世界模型,使得仅用 100k 步真实数据就能在雅达利上和人类水平相当。", + "label": "IRIS", + "degree": 5 + }, + { + "id": "paper:sac", + "label_zh": "SAC(最大熵柔性 Actor-Critic)", + "kind": "paper", + "tier": "S", + "topic": "deep_rl", + "phase": "core", + "year": 2018, + "summary_zh": "SAC 把最大熵强化学习推广到连续动作空间,让策略在最大化期望回报的同时也最大化策略的熵。它自动调节温度系数来控制探索强度,配合双 Q 评论员减小过估计偏差,成为连续控制领域最稳定也最常用的基线算法之一。", + "label": "Soft Actor-Critic", + "degree": 10 + }, + { + "id": "paper:td3", + "label_zh": "TD3(双延迟深度确定性策略梯度)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "core", + "year": 2018, + "summary_zh": "TD3 针对 DDPG 在连续控制上不稳定的问题,引入了取两个目标 Q 网络较小值以抑制过估计、延迟更新策略网络以让评论员先收敛、以及在目标动作上加平滑噪声以避免被尖峰估值欺骗这三项关键修正。它把确定性策略梯度方法的样本效率和稳定性同时提升到了实用水平。", + "label": "TD3", + "degree": 5 + }, + { + "id": "paper:a3c_a2c", + "label_zh": "A3C / A2C(异步与同步优势 Actor-Critic)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "prereq", + "year": 2016, + "summary_zh": "A3C 让多个并行的工人各自和环境交互,把梯度异步推送到一个共享参数服务器,从而在不依赖回放缓冲的情况下打破样本相关性。后来研究者发现把异步改成同步、批量平均梯度就能得到同样甚至更好的效果,于是出现了更简单的 A2C 变体,并成为很多策略梯度方法的入门骨架。", + "label": "A3C / A2C", + "degree": 7 + }, + { + "id": "paper:impala", + "label_zh": "IMPALA(V-trace 重要性修正分布式 RL)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "core", + "year": 2018, + "summary_zh": "IMPALA 把 actor 和 learner 解耦:actor 不断把轨迹送给 learner,learner 用稍旧的策略数据更新参数。为了纠正这种异步带来的离策略偏差,作者提出了 V-trace 截断重要性采样,使分布式深度强化学习能在几千个 CPU 上稳定扩展并训练单一智能体同时玩几十个任务。", + "label": "IMPALA", + "degree": 2 + }, + { + "id": "paper:mpo", + "label_zh": "MPO(最大后验策略优化)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "core", + "year": 2018, + "summary_zh": "MPO 把策略改进步骤看成一次后验推断:先在 Q 值加权下找到一个非参数化的最优动作分布,再让参数化策略以 KL 投影去拟合这个分布。这种 E 步加 M 步的结构既保留了策略梯度的灵活性,又获得了类似自然梯度的稳定步长,成为 DeepMind 控制类工作的常用主力算法。", + "label": "MPO", + "degree": 4 + }, + { + "id": "paper:decision_transformer", + "label_zh": "Decision Transformer(把强化学习当作序列建模)", + "kind": "paper", + "tier": "S", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "Decision Transformer 用一个简单的 GPT 风格 transformer,把过去的回报到 go、状态和动作组成的序列直接当成语言来建模,预测下一个动作。它完全跳过了价值函数和策略梯度,只靠监督式的下一个 token 预测就在 D4RL 等离线数据集上达到与离线 RL 专门算法相当的水平。", + "label": "Decision Transformer", + "degree": 7 + }, + { + "id": "paper:trajectory_transformer", + "label_zh": "Trajectory Transformer(轨迹 token 化 + 束搜索)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "Trajectory Transformer 把状态、动作和奖励都离散化成 token,让一个 transformer 学会预测整条轨迹的联合分布。在部署时它对未来轨迹做束搜索,从中挑出预期回报最高的一条,从而把规划重新表述为高似然且高回报的序列搜索问题。", + "label": "Trajectory Transformer", + "degree": 6 + }, + { + "id": "paper:diffusion_policy_chi2023", + "label_zh": "Diffusion Policy(用扩散模型生成动作序列)", + "kind": "paper", + "tier": "S", + "topic": "deep_rl", + "phase": "frontier", + "year": 2023, + "summary_zh": "Diffusion Policy 把模仿学习中的策略改写成一个条件扩散模型,输入是最近若干帧观察,输出是未来若干步的动作序列。扩散过程的多模态表达能力让它能优雅处理人类示教中的动作多解性,从而在多种机器人操控任务上把成功率显著推高。", + "label": "Diffusion Policy", + "degree": 5 + }, + { + "id": "paper:redq", + "label_zh": "REDQ(高更新比的集成 Q)", + "kind": "paper", + "tier": "B", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "REDQ 在 SAC 之上做了两件事:维护一个由十个 Q 网络组成的集成、并且每次环境交互后做二十次梯度更新。集成中的方差用来抑制过估计偏差,使得激进的更新比也能稳定,从而把无模型连续控制的样本效率推到接近基于模型方法的水平。", + "label": "REDQ", + "degree": 2 + }, + { + "id": "paper:cql", + "label_zh": "CQL(保守 Q 学习)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "core", + "year": 2020, + "summary_zh": "CQL 在离线强化学习的 Q 损失里额外加了一个项,使得对数据外动作的 Q 估计被显式压低,从而得到真实 Q 的下界。这种保守化让从离线数据学到的策略在部署时不再倾向于挑那些没见过却看起来高分的动作,极大地缓解了离线 RL 的分布偏移问题。", + "label": "Conservative Q-Learning", + "degree": 5 + }, + { + "id": "paper:iql", + "label_zh": "IQL(隐式 Q 学习)", + "kind": "paper", + "tier": "S", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "IQL 用一个 expectile 回归来估计动作分布在状态下的高分位 Q,从而完全避免在 Bellman 备份里采样数据外动作。配合一个用优势加权的策略提取步骤,它在不显式接触分布外动作的情况下做出隐式的最大化,是当下最稳健的离线 RL 算法之一。", + "label": "Implicit Q-Learning", + "degree": 7 + }, + { + "id": "paper:calql", + "label_zh": "Cal-QL(校准 CQL 以支持离线到在线微调)", + "kind": "paper", + "tier": "B", + "topic": "deep_rl", + "phase": "frontier", + "year": 2023, + "summary_zh": "Cal-QL 发现 CQL 把分布外动作的 Q 值压得过低,会让随后的在线微调阶段一直在错误的方向上拉扯。它在 CQL 的下界项里加入一个校准项,把分布外 Q 钉在参考策略的真实回报附近,从而让同一份模型能从离线预训练顺畅过渡到在线强化学习。", + "label": "Cal-QL", + "degree": 6 + }, + { + "id": "paper:alphastar", + "label_zh": "AlphaStar(星际争霸 II 的大规模联盟训练)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "core", + "year": 2019, + "summary_zh": "AlphaStar 用监督学习从人类录像预热策略,然后通过一个由主智能体、利用者和联盟探索者构成的联盟博弈系统持续自我对弈,避免在策略空间里陷入循环。它最终在星际争霸 II 上达到大师级水准,展示了如何在巨大动作空间和长时博弈中把分布式强化学习推到极致。", + "label": "AlphaStar", + "degree": 4 + }, + { + "id": "paper:openai_five", + "label_zh": "OpenAI Five(Dota 2 大规模 PPO)", + "kind": "paper", + "tier": "A", + "topic": "deep_rl", + "phase": "core", + "year": 2019, + "summary_zh": "OpenAI Five 用一套相对简单的 PPO 算法,但把训练规模拉到数万 CPU 和成百上千 GPU、每天累积成千年的游戏经验。它通过随机化对手分布和团队奖励塑形把五人合作策略训练到能战胜世界冠军队伍,证明在合适规模下纯粹的策略梯度方法依然非常强大。", + "label": "OpenAI Five", + "degree": 3 + }, + { + "id": "paper:interfuser", + "label_zh": "InterFuser(CARLA 多模态融合 transformer)", + "kind": "paper", + "tier": "A", + "topic": "planning", + "phase": "core", + "year": 2022, + "summary_zh": "InterFuser 用一个 transformer 把多视角摄像头、激光雷达和高精地图特征融合成一个共享表示,并在解码器上同时输出可解释的中间结果,例如他车未来轨迹和交通灯状态。这种结构既提高了 CARLA 闭环驾驶分数,也让规划过程的依据可被人类审查。", + "label": "InterFuser", + "degree": 4 + }, + { + "id": "paper:roach", + "label_zh": "Roach(用特权 RL 教师蒸馏出 BC 学生)", + "kind": "paper", + "tier": "A", + "topic": "planning", + "phase": "core", + "year": 2021, + "summary_zh": "Roach 先在 CARLA 模拟器中用 PPO 训练一个能访问全部真值信息的强化学习教师智能体,再用这个教师生成大规模数据集,让一个只看摄像头的学生策略用监督学习去模仿。它把仿真特权信息变成了能下放给真实感知模型的免费监督信号,在 CARLA 排行榜上长期占据前列。", + "label": "Roach", + "degree": 5 + }, + { + "id": "paper:thinktwice", + "label_zh": "ThinkTwice(粗规划 + 细化的两阶段端到端)", + "kind": "paper", + "tier": "B", + "topic": "planning", + "phase": "frontier", + "year": 2023, + "summary_zh": "ThinkTwice 把端到端驾驶分成两个 transformer 阶段:第一阶段从感知特征里生成一个粗略的未来轨迹,第二阶段用这条粗轨迹回到特征上做条件式的细化预测,把可能的碰撞和遮挡反馈到下一次迭代。这种两次思考的结构让它在 CARLA 复杂场景上取得当年最好的闭环得分。", + "label": "ThinkTwice", + "degree": 3 + }, + { + "id": "paper:mile_driving", + "label_zh": "MILE(驾驶世界模型 + 模仿学习)", + "kind": "paper", + "tier": "A", + "topic": "world_models", + "phase": "frontier", + "year": 2022, + "summary_zh": "MILE 把 Wayve 内部的 BEV 世界模型与模仿学习联合训练,让策略不仅模仿专家动作,还要在隐世界模型中预测未来的语义 BEV 占用图。该联合目标显著提升了在城市路况下的泛化能力,也是较早把驾驶世界模型用于离线策略学习的代表性工作。", + "label": "MILE", + "degree": 4 + }, + { + "id": "paper:nuplan_baselines", + "label_zh": "nuPlan 规划基线(IDM, PDM, GameFormer)", + "kind": "paper", + "tier": "B", + "topic": "planning", + "phase": "core", + "year": 2023, + "summary_zh": "nuPlan 团队和社区围绕这一首个大规模真实驾驶规划基准发布了一系列基线,包括基于规则的 IDM 跟车、PDM 预测驱动的轨迹打分,以及 GameFormer 这样的博弈式 transformer 规划器。这些基线共同表明,规则方法在闭环度量上依然非常顽强,纯模仿学习离不开仔细的轨迹后处理。", + "label": "nuPlan baselines", + "degree": 4 + }, + { + "id": "paper:cilqr", + "label_zh": "CILQR(带约束的迭代 LQR)", + "kind": "paper", + "tier": "A", + "topic": "control", + "phase": "core", + "year": 2017, + "summary_zh": "CILQR 在 iLQR 的二阶迭代解算器上加入了对状态和控制的不等式约束,用对数障碍函数把约束变成可微项。它使得轨迹优化器可以同时考虑车辆动力学、加速度极限和侧向稳定性,成为很多工业级运动规划模块的核心数值求解器。", + "label": "CILQR", + "degree": 4 + }, + { + "id": "paper:ilqr_classic", + "label_zh": "iLQR(迭代 LQR)", + "kind": "paper", + "tier": "A", + "topic": "control", + "phase": "prereq", + "year": 2004, + "summary_zh": "iLQR 把非线性最优控制问题在当前名义轨迹附近线性化,套用线性二次调节器解出一次反馈律,然后沿反馈律前向滚动得到新的名义轨迹,反复迭代直到收敛。它兼具 LQR 的解析效率和直接配点法的非线性表达力,是机器人和自动驾驶轨迹优化的标准工具。", + "label": "iLQR", + "degree": 3 + }, + { + "id": "paper:mpc_book", + "label_zh": "MPC(模型预测控制,经典教科书)", + "kind": "paper", + "tier": "S", + "topic": "control", + "phase": "prereq", + "year": 2002, + "summary_zh": "模型预测控制在每一步都用当前模型预测未来若干步的状态轨迹,求解一个带约束的最优控制问题,只执行最优序列的第一个动作然后滚动重新求解。它天然支持显式约束和多输入多输出耦合,是化工、机械和自动驾驶纵向控制最广泛部署的最优控制方法。", + "label": "Model Predictive Control", + "degree": 5 + }, + { + "id": "paper:lqr_classic", + "label_zh": "LQR(线性二次调节器)", + "kind": "paper", + "tier": "A", + "topic": "control", + "phase": "prereq", + "year": 1960, + "summary_zh": "LQR 假设系统动力学是线性的、代价函数是状态和控制的二次型,并通过求解一个代数 Riccati 方程得到最优线性反馈律。它是最优控制最早也是最有教学意义的解析结果,几乎所有更先进的非线性最优控制方法都以它作为局部子问题或起步基线。", + "label": "Linear Quadratic Regulator", + "degree": 4 + }, + { + "id": "paper:cpo_safe_rl", + "label_zh": "CPO(带约束的策略优化)", + "kind": "paper", + "tier": "A", + "topic": "safety", + "phase": "core", + "year": 2017, + "summary_zh": "CPO 把 TRPO 的信赖域思想推广到带约束的马尔可夫决策过程,在每一步策略更新里同时保证新的策略在期望回报上有改进,并且不会让某些预期约束代价超过预算。它是第一个在大规模深度强化学习里直接对安全约束做硬性保证的算法,奠定了后续安全 RL 的范式。", + "label": "Constrained Policy Optimization", + "degree": 6 + }, + { + "id": "paper:lagrangian_safe_rl", + "label_zh": "Lagrangian 安全 RL(约束 RL 的对偶方法)", + "kind": "paper", + "tier": "B", + "topic": "safety", + "phase": "core", + "year": 2019, + "summary_zh": "Lagrangian 风格的安全强化学习把每个约束写成一个不等式,引入对偶变量并和策略参数一起做交替优化。这种方法实现简单,可以套在 PPO、SAC 等任意策略梯度算法上,是安全 RL 的工业基线,但对超参数和奖励 / 约束尺度比较敏感。", + "label": "Lagrangian Safe RL", + "degree": 6 + }, + { + "id": "paper:shielded_rl", + "label_zh": "屏蔽式 RL(形式化安全屏蔽)", + "kind": "paper", + "tier": "B", + "topic": "safety", + "phase": "core", + "year": 2018, + "summary_zh": "屏蔽式强化学习在学习到的策略外面套一层用形式化方法或可达性分析得到的安全监督器:当策略想执行某动作时,屏蔽器检查该动作是否会进入预定义的不安全状态集,并在必要时替换为安全替代动作。这种结构让 RL 的探索能够在硬安全保证下进行。", + "label": "Shielded RL", + "degree": 4 + }, + { + "id": "paper:pebble", + "label_zh": "PEBBLE(无监督预训练 + 偏好反馈 RL)", + "kind": "paper", + "tier": "B", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "PEBBLE 先用无监督的内在奖励 RL 让智能体学到一组多样化策略,再向人类反复展示成对的轨迹片段征询偏好,用偏好数据训练一个奖励模型,最后再用这个奖励模型驱动正式的 RL 学习。它把偏好式 RL 的样本效率提升到能在几百次查询内学会复杂控制任务的水平。", + "label": "PEBBLE", + "degree": 5 + }, + { + "id": "paper:bpref", + "label_zh": "B-Pref(偏好式 RL 基准)", + "kind": "paper", + "tier": "B", + "topic": "deep_rl", + "phase": "core", + "year": 2021, + "summary_zh": "B-Pref 提出了一组覆盖不同标注误差模式的人类偏好模拟器,并把当时所有主要的偏好式 RL 方法在统一接口下对照评估。它揭示了真实人类反馈中的不一致与延迟所带来的鲁棒性问题,是奖励建模与对齐研究的常用沙盒。", + "label": "B-Pref", + "degree": 4 + }, + { + "id": "paper:trajeglish", + "label_zh": "Trajeglish(驾驶轨迹的 token 语言模型)", + "kind": "paper", + "tier": "B", + "topic": "world_models", + "phase": "frontier", + "year": 2023, + "summary_zh": "Trajeglish 把多智能体的连续驾驶轨迹离散化成动作 token 序列,然后训练一个像语言模型一样自回归预测下一步行为的 transformer。它生成的交通场景在 Waymo Open Sim Agents 基准上拿到当时的最佳成绩,把驾驶仿真智能体推向以序列建模为中心的范式。", + "label": "Trajeglish", + "degree": 7 + }, + { + "id": "paper:most_simagents", + "label_zh": "MoST(多智能体场景 token 化生成)", + "kind": "paper", + "tier": "B", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "MoST 把仿真场景中所有车辆共同的状态序列编码成统一的 token 流,由一个 transformer 联合生成所有智能体未来若干秒的行为。这种把场景看成单一长序列的视角能够内生地捕捉车辆之间的交互,从而显著提升合成交通流的逼真度。", + "label": "MoST", + "degree": 4 + }, + { + "id": "paper:codetraj", + "label_zh": "CodeTrajectory(以代码描述驾驶轨迹)", + "kind": "paper", + "tier": "B", + "topic": "planning", + "phase": "frontier", + "year": 2024, + "summary_zh": "CodeTrajectory 让大语言模型把规划任务表述成一段 Python 风格的轨迹生成代码,再交给数值求解器或仿真器执行得到具体轨迹。这种以代码作为中间表示的方法使 LLM 能在抽象语义层面思考决策,同时把数值优化的精度问题留给专门工具。", + "label": "CodeTrajectory", + "degree": 4 + }, + { + "id": "paper:diffusion_planner", + "label_zh": "Diffuser 规划器扩展(条件扩散轨迹生成)", + "kind": "paper", + "tier": "B", + "topic": "planning", + "phase": "frontier", + "year": 2023, + "summary_zh": "在原始 Diffuser 的基础上,多项后续工作把条件扩散模型应用到驾驶轨迹规划:在采样过程中通过梯度引导加入碰撞代价、舒适度代价或目标到达约束,从而在不改动主干的情况下灵活注入安全和性能目标。", + "label": "Diffuser Planner extensions", + "degree": 3 + }, + { + "id": "paper:mbrl_pets", + "label_zh": "PETS(概率集成 + 轨迹采样的基于模型 RL)", + "kind": "paper", + "tier": "B", + "topic": "world_models", + "phase": "core", + "year": 2018, + "summary_zh": "PETS 用一个由概率神经网络集成构成的动力学模型,捕捉认知与随机两类不确定性,然后通过交叉熵方法对动作序列做采样规划。它证明只要正确建模不确定性,基于模型的 RL 在样本效率上可以和最强的无模型方法拉开数量级的差距。", + "label": "PETS", + "degree": 4 + }, + { + "id": "move:learn_world_model_then_plan_in_latent_imagination", + "label_zh": "学得世界模型后在隐空间想象中做规划", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "core", + "year": 2018, + "summary_zh": "训练一个能够预测未来隐状态、奖励和终止信号的世界模型,然后让策略或规划器完全在这个内部模拟器里展开多步推演。真实环境只用来收集数据和最终评估,从而把样本效率与昂贵交互彻底解耦。", + "label": "Learn world model then plan in latent imagination", + "degree": 7 + }, + { + "id": "move:plan_with_mcts_in_learned_model", + "label_zh": "在学得模型上跑蒙特卡洛树搜索", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "frontier", + "year": 2019, + "summary_zh": "把传统上需要真实模拟器的蒙特卡洛树搜索接到一个学得的隐式动力学模型上,让搜索可以扩展到没有先验规则的领域。这个动作让 AlphaZero 的强大规划能力解锁到雅达利、机器人乃至驾驶任务。", + "label": "Plan with MCTS in a learned model", + "degree": 3 + }, + { + "id": "move:discrete_latent_state_for_world_model", + "label_zh": "用离散隐变量构造世界模型状态", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "core", + "year": 2020, + "summary_zh": "把世界模型隐状态从连续高斯换成离散随机变量,再用直通梯度估计反向传播。这种结构能稳定捕捉多模态未来,是 DreamerV2 之后绝大多数世界模型的默认设计。", + "label": "Discrete latent state for world model", + "degree": 3 + }, + { + "id": "move:tokenize_pixel_frames_for_autoregressive_world_model", + "label_zh": "把像素帧 token 化成离散符号让 transformer 自回归预测", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "frontier", + "year": 2022, + "summary_zh": "先用 VQ-VAE 把每一帧图像压缩成几十到几百个离散 token,再让 transformer 把环境演化当成 token 序列预测。这把世界模型变成了一个像 GPT 一样可扩展、可条件化的生成模型。", + "label": "Tokenize pixel frames for autoregressive world model", + "degree": 4 + }, + { + "id": "move:replace_explicit_critic_with_diffusion_score", + "label_zh": "用扩散得分函数代替显式评论员", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "frontier", + "year": 2022, + "summary_zh": "把策略输出从一个简单高斯换成一个条件扩散模型,让动作分布的对数密度梯度自然承担起评论员的角色。这种动作让策略具备多模态表达能力,特别适合人类示教数据中常见的一态多动作场景。", + "label": "Replace explicit critic with diffusion score", + "degree": 2 + }, + { + "id": "move:bootstrap_target_network_to_stabilize_off_policy_learning", + "label_zh": "用慢更新目标网络稳定自举学习", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "prereq", + "year": 2015, + "summary_zh": "为价值函数维护一份缓慢更新或定期复制的目标参数,让 Bellman 备份的目标值在短时间内保持不变。这显著抑制了 Q 学习在深度网络上常见的发散现象。", + "label": "Use a target network to stabilize bootstrapping", + "degree": 4 + }, + { + "id": "move:add_entropy_bonus_to_encourage_exploration", + "label_zh": "在目标函数里加策略熵奖励促进探索", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "在策略优化的目标里加入策略熵项,鼓励策略在高回报附近仍保留一定随机性。这一动作既缓解了局部最优陷阱,也让最大熵强化学习有了原则性的损失函数定义。", + "label": "Add entropy bonus to encourage exploration", + "degree": 4 + }, + { + "id": "move:turn_offline_dataset_into_supervised_sequence_prediction", + "label_zh": "把离线数据集改写成监督序列预测", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "把回报到 go、状态、动作排成序列,让 transformer 用普通的下一个 token 监督学习目标来训练。这一动作绕开了价值函数与策略梯度,让强化学习直接受益于成熟的大模型工具链。", + "label": "Turn offline dataset into supervised sequence prediction", + "degree": 3 + }, + { + "id": "move:replace_value_function_with_implicit_max_via_expectile", + "label_zh": "用 expectile 回归隐式取最大值替代显式 Q 最大化", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "在离线 RL 中用 expectile 回归直接拟合 Q 在动作上的高分位,而不显式枚举或采样动作。这种动作让算法完全只看见数据集内的动作,自然回避了离线 RL 最棘手的分布外动作问题。", + "label": "Replace explicit value function with expectile implicit max", + "degree": 1 + }, + { + "id": "move:use_pretrained_language_model_as_action_prior", + "label_zh": "把预训练大语言模型当作动作序列先验", + "kind": "move", + "tier": "move", + "topic": "planning", + "phase": "frontier", + "year": 2023, + "summary_zh": "用预训练大语言模型在符号或代码层面提出候选动作序列,再交给数值优化器或仿真器评估。这种动作把 LLM 的常识与场景理解嫁接到了细粒度控制,是 LLM 规划器的关键构件。", + "label": "Use pretrained LLM as prior over action sequences", + "degree": 4 + }, + { + "id": "move:add_lagrangian_safety_constraint_to_actor_critic", + "label_zh": "在 actor-critic 上挂安全约束的 Lagrangian", + "kind": "move", + "tier": "move", + "topic": "safety", + "phase": "core", + "year": 2017, + "summary_zh": "把碰撞概率或代价违规等安全指标写成不等式约束,引入对偶变量做交替优化,从而把安全 RL 直接架在已有的 PPO 或 SAC 上。它让带约束的策略改进有了一个工业可落地的实现路径。", + "label": "Add Lagrangian safety constraint to actor-critic", + "degree": 4 + }, + { + "id": "move:treat_planning_as_conditional_generation", + "label_zh": "把规划看作条件生成问题", + "kind": "move", + "tier": "move", + "topic": "planning", + "phase": "frontier", + "year": 2022, + "summary_zh": "把轨迹规划重新表述成给定当前状态、目标和约束的条件分布采样问题,让扩散模型或自回归 transformer 等生成模型直接输出整条候选轨迹。这一动作把规划从优化范式拉进了生成范式。", + "label": "Treat planning as conditional generation", + "degree": 6 + }, + { + "id": "move:cast_continuous_action_as_discretized_token_sequence", + "label_zh": "把连续动作离散化成 token 序列", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "把每一维连续动作按分位数或均匀离散化成有限种类的 token,从而让 transformer 用分类交叉熵学习。该动作打开了语言建模工具箱直接服务控制问题的大门。", + "label": "Cast continuous action as discretized token sequence", + "degree": 4 + }, + { + "id": "move:use_n_step_returns_to_trade_bias_for_variance", + "label_zh": "用 n 步回报折中偏差与方差", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "prereq", + "year": 1988, + "summary_zh": "把 1 步 TD 备份和蒙特卡洛回报间的 n 步混合作为价值学习目标,在偏差和方差之间显式调参。这一基本动作贯穿从经典 TD 到 Rainbow 再到 R2D2 的几乎所有改进。", + "label": "Use n-step returns to trade bias for variance", + "degree": 2 + }, + { + "id": "move:add_intrinsic_motivation_via_novelty_or_curiosity", + "label_zh": "通过新颖性或好奇心给探索加内在动机", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "在外部奖励之外再加一项基于预测误差、状态访问计数或表征相似度的内在奖励,鼓励智能体主动探索没见过的状态。这是稀疏奖励下让 RL 仍能学到东西的关键动作。", + "label": "Add intrinsic motivation via novelty or curiosity", + "degree": 1 + }, + { + "id": "move:apply_gae_to_smooth_advantage_estimation", + "label_zh": "用广义优势估计平滑优势函数", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2015, + "summary_zh": "用一个折扣系数 lambda 把不同 n 步优势的几何加权混合起来,在偏差和方差之间得到平滑的优势估计。这是 PPO、A3C 等所有现代策略梯度方法的标准配件。", + "label": "Apply GAE to smooth advantage estimation", + "degree": 3 + }, + { + "id": "move:use_prioritized_replay_buffer", + "label_zh": "用优先级回放提升样本效率", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2015, + "summary_zh": "在经验回放里按 TD 误差大小为转移分配采样概率,让网络更多见到学习信号大的样本。这一动作通常能让 DQN 系列在相同样本数下提升数倍的最终表现。", + "label": "Use priority-weighted replay buffer", + "degree": 2 + }, + { + "id": "move:cotrain_dynamics_model_with_policy_to_share_representations", + "label_zh": "联合训练动力学模型与策略共享表示", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "core", + "year": 2018, + "summary_zh": "让感知主干同时承担预测下一时刻状态和输出动作两个任务,并共享中间表征。这迫使表征同时编码动力学相关和决策相关信息,从而比纯模仿学习更具结构。", + "label": "Co-train dynamics model with policy to share representations", + "degree": 2 + }, + { + "id": "move:warm_start_rl_with_imitation_then_anneal", + "label_zh": "用模仿数据预热 RL 再退火到纯 RL", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2019, + "summary_zh": "先用专家示范监督学习训练策略和评论员,再逐步降低模仿损失权重、提高 RL 损失权重,最后切换到纯环境奖励训练。这一动作显著降低早期探索失败和大规模分布式 RL 的冷启动成本。", + "label": "Warm start RL with imitation data then anneal", + "degree": 5 + }, + { + "id": "move:double_q_to_reduce_overestimation", + "label_zh": "用双 Q 结构抑制过估计偏差", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2016, + "summary_zh": "维护两个独立训练的 Q 网络,并用其中一个选取动作、另一个估值,或简单地在两者中取较小值。该动作有效地消除了 Q 学习以 max 操作为代价付出的系统性高估。", + "label": "Double Q to reduce overestimation", + "degree": 5 + }, + { + "id": "move:expert_iteration_self_distillation", + "label_zh": "专家迭代式自蒸馏", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "用一个慢但强的搜索专家(例如 MCTS)生成训练标签,让一个快但弱的网络去模仿;模仿后的网络又用作下一轮搜索的先验。这一循环是 AlphaZero 和后续多项工作的核心动作。", + "label": "Expert iteration via self-distillation", + "degree": 2 + }, + { + "id": "move:distill_privileged_teacher_to_sensor_student", + "label_zh": "将特权信息教师蒸馏到只看传感器的学生策略", + "kind": "move", + "tier": "move", + "topic": "planning", + "phase": "core", + "year": 2021, + "summary_zh": "先在仿真中训练一个能读到全部真值信息的教师策略,再让一个只能看相机或激光雷达的学生策略用监督学习去模仿。它把仿真特权信息变成了对真实部署可用的免费监督。", + "label": "Distill privileged teacher to sensor-only student", + "degree": 3 + }, + { + "id": "move:trust_region_step_for_monotonic_improvement", + "label_zh": "用信赖域步长保证策略单调改进", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2015, + "summary_zh": "把策略更新限制在新旧策略 KL 散度不超过给定阈值的范围内,给出有理论保证的近似单调改进。从 TRPO 的硬约束到 PPO 的截断比都是这一动作的不同实现形式。", + "label": "Trust region step for monotonic improvement", + "degree": 4 + }, + { + "id": "move:expectile_or_quantile_target_for_distributional_robustness", + "label_zh": "用 expectile 或 quantile 目标做分布式 RL", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "用 expectile 或 quantile 回归学习回报分布的高阶矩,而不仅是其均值。这一动作既是分布式 DQN 的基石,也启发了 IQL 等离线 RL 算法回避分布外动作。", + "label": "Expectile or quantile target for distributional RL", + "degree": 1 + }, + { + "id": "move:hindsight_experience_relabeling", + "label_zh": "用事后经验重标注扩充稀疏奖励数据", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "把失败轨迹的真实终点临时当作虚拟目标,让原本零奖励的转移变成对该虚拟任务的成功示例。该动作显著缓解了多目标稀疏奖励 RL 的样本效率问题。", + "label": "Hindsight experience relabeling", + "degree": 1 + }, + { + "id": "move:safety_shield_filters_unsafe_actions", + "label_zh": "用安全屏蔽器过滤不安全动作", + "kind": "move", + "tier": "move", + "topic": "safety", + "phase": "core", + "year": 2018, + "summary_zh": "在策略输出和执行之间插入一个由形式化验证或可达性分析得到的过滤器,对会进入不安全状态集的动作做替换。这是把硬安全保证与软学习策略结合的关键工程动作。", + "label": "Safety shield filters unsafe actions", + "degree": 1 + }, + { + "id": "move:reward_model_from_pairwise_human_preferences", + "label_zh": "从成对人类偏好学奖励模型", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "让人类标注员在两条候选轨迹之间二选一,用 Bradley-Terry 模型拟合一个标量奖励,再用这个奖励驱动后续 RL。该动作把人类直觉编码进了可微的奖励信号,是 RLHF 的核心。", + "label": "Reward model from pairwise human preferences", + "degree": 4 + }, + { + "id": "move:guided_sampling_through_classifier_gradients_at_inference", + "label_zh": "在推理时用分类器梯度引导扩散采样", + "kind": "move", + "tier": "move", + "topic": "planning", + "phase": "frontier", + "year": 2022, + "summary_zh": "在扩散模型的反向采样过程中按某个外部代价函数的梯度对样本做扰动,从而把碰撞、舒适度、目标到达等约束在不重新训练模型的情况下注入轨迹生成。", + "label": "Guided sampling via classifier gradients at inference", + "degree": 3 + }, + { + "id": "move:plan_via_cross_entropy_method_on_dynamics_model", + "label_zh": "在学得动力学模型上用交叉熵方法做规划", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "core", + "year": 2018, + "summary_zh": "把动作序列看成一个分布,反复采样、按预测回报筛选精英、再拟合精英拟合一个新分布,反复迭代直至收敛。该动作让基于模型的 RL 不依赖反向传播也能做长程规划。", + "label": "Plan via cross-entropy method on a learned dynamics model", + "degree": 2 + }, + { + "id": "move:two_stage_coarse_to_fine_trajectory", + "label_zh": "粗规划再细化的两阶段轨迹生成", + "kind": "move", + "tier": "move", + "topic": "planning", + "phase": "frontier", + "year": 2023, + "summary_zh": "先用一个简单解码器输出粗略未来轨迹,再把这条轨迹作为条件回到主干特征上做精细化预测。这一动作让模型既能高效地确定大致方向,又能在第二阶段集中算力处理障碍和交互细节。", + "label": "Two-stage coarse-to-fine trajectory", + "degree": 2 + }, + { + "id": "move:league_play_for_policy_diversity", + "label_zh": "用联盟博弈维持策略多样性", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "frontier", + "year": 2019, + "summary_zh": "在自我对弈中显式维护主智能体、利用者和探索者等多种角色,让训练池涵盖多种风格的对手。这一动作有效防止了纯自我对弈陷入策略循环,是 AlphaStar 的核心创新。", + "label": "League play for policy diversity", + "degree": 1 + }, + { + "id": "problem:reward_specification_for_safe_polite_driving", + "label_zh": "如何为安全且礼让的驾驶设计奖励", + "kind": "problem", + "tier": "problem", + "topic": "safety", + "phase": "core", + "year": 2018, + "summary_zh": "驾驶任务的奖励既要鼓励到达目标,又要惩罚危险、不舒适和不文明的行为,而这些维度往往相互冲突且难以量化。如何设计一个数学上明确、又能在真实数据中可计算的奖励,是端到端规划长期未解的问题。", + "label": "Reward specification for safe and polite driving", + "degree": 2 + }, + { + "id": "problem:long_horizon_credit_assignment_in_driving", + "label_zh": "驾驶任务中的长时信用分配", + "kind": "problem", + "tier": "problem", + "topic": "rl_foundations", + "phase": "core", + "year": 2018, + "summary_zh": "在城市驾驶里,一次错过的并道决策可能要几十秒后才在事故或拥堵中体现,使得 RL 的回报信号极度稀疏且延迟。如何在这种长视野下把奖励正确归因到关键决策,是把 RL 真正用于规划面临的核心难题。", + "label": "Long-horizon credit assignment in driving", + "degree": 3 + }, + { + "id": "problem:distributional_shift_between_offline_data_and_deployment", + "label_zh": "离线数据与在线部署之间的分布偏移", + "kind": "problem", + "tier": "problem", + "topic": "deep_rl", + "phase": "core", + "year": 2020, + "summary_zh": "用历史驾驶日志训练得到的策略一旦上路就会进入数据集没有覆盖的状态-动作组合,价值估计在这些点上往往严重高估。如何在不重新采集真实数据的情况下抑制这种偏移,是离线 RL 与模仿学习的共同难题。", + "label": "Distributional shift between offline data and deployment", + "degree": 5 + }, + { + "id": "problem:closed_loop_simulation_fidelity_gap", + "label_zh": "闭环仿真与真实世界的逼真度差距", + "kind": "problem", + "tier": "problem", + "topic": "planning", + "phase": "core", + "year": 2020, + "summary_zh": "闭环仿真要同时模拟感知、预测和其它交通参与者的反应,任何一个环节失真都会让在仿真里训练或评估的策略在真实路况下表现迥异。如何刻画并缩小这种 sim-to-real 差距,是规划研究最核心的方法论瓶颈之一。", + "label": "Closed-loop simulation fidelity gap", + "degree": 4 + }, + { + "id": "problem:multi_agent_interaction_modeling_in_dense_traffic", + "label_zh": "密集车流中的多智能体交互建模", + "kind": "problem", + "tier": "problem", + "topic": "planning", + "phase": "frontier", + "year": 2021, + "summary_zh": "在城市路口或匝道,自车决策与他车行为相互耦合,单边的预测或单边的规划都无法捕捉真实博弈过程。如何在不让模型规模与计算复杂度爆炸的情况下表达这种多智能体交互,是行为预测和规划的关键挑战。", + "label": "Multi-agent interaction modeling in dense traffic", + "degree": 2 + }, + { + "id": "problem:rare_event_evaluation_with_no_ground_truth", + "label_zh": "缺少真值的稀有事件评测", + "kind": "problem", + "tier": "problem", + "topic": "safety", + "phase": "frontier", + "year": 2021, + "summary_zh": "致命驾驶事故每数千万公里才发生一次,常规度量根本无法以统计置信度度量这种长尾,重要性采样和合成场景又面临真值缺失的问题。如何让稀有事件评测可信且可比较,是工业界与监管层的共同未解题。", + "label": "Rare-event evaluation without ground truth", + "degree": 2 + }, + { + "id": "problem:exploration_in_safety_critical_systems", + "label_zh": "安全关键系统中的探索问题", + "kind": "problem", + "tier": "problem", + "topic": "safety", + "phase": "core", + "year": 2019, + "summary_zh": "RL 需要主动尝试未知动作以学习更好的策略,但在驾驶或机器人手术等领域,错误探索代价无法承受。如何在硬安全约束下保持有效探索是安全 RL 与控制理论的长期共同难题。", + "label": "Exploration in safety-critical systems", + "degree": 3 + }, + { + "id": "problem:planning_horizon_vs_compute_budget_tradeoff", + "label_zh": "规划时域与算力预算的权衡", + "kind": "problem", + "tier": "problem", + "topic": "control", + "phase": "core", + "year": 2010, + "summary_zh": "把规划做得更深可以预见更远的后果,但计算成本随时域指数或多项式增长,而车载算力又必须支持十赫兹以上的实时刷新。如何在不牺牲反应速度的前提下扩展有效规划时域,是车载规划系统的永恒折中。", + "label": "Planning horizon vs compute budget tradeoff", + "degree": 2 + }, + { + "id": "problem:behavior_cloning_compounds_errors_over_time", + "label_zh": "行为克隆误差随时间复合", + "kind": "problem", + "tier": "problem", + "topic": "deep_rl", + "phase": "core", + "year": 2010, + "summary_zh": "纯监督式的行为克隆只看见专家轨迹,一旦在部署时偏离哪怕一点点就会进入训练分布之外,下一步又在更偏的位置预测,错误像雪球一样滚大。这是 DAgger、对抗模仿学习等大量后续方法所共同针对的问题。", + "label": "Behavior cloning compounds errors over time", + "degree": 4 + }, + { + "id": "problem:reward_hacking_in_learned_objectives", + "label_zh": "学习奖励容易被钻空子", + "kind": "problem", + "tier": "problem", + "topic": "safety", + "phase": "frontier", + "year": 2020, + "summary_zh": "无论是手写代价还是从人类偏好学到的奖励模型,都不可能完整反映真实意图,因此优化过头时策略会找到奇形怪状但高分的行为。如何检测并修补这种 reward hacking,是 RLHF 与控制对齐的共同未解问题。", + "label": "Reward hacking in learned objectives", + "degree": 3 + }, + { + "id": "insight:imitation_learning_alone_cannot_recover_from_compounding_errors", + "label_zh": "纯模仿学习无法从复合误差中自我恢复", + "kind": "insight", + "tier": "insight", + "topic": "deep_rl", + "phase": "core", + "year": 2011, + "summary_zh": "由于训练阶段只看专家状态而部署阶段必须自己应对自己产生的状态,模仿学习的误差会沿时间复合。任何想要把模仿学习推到长视野任务的方法都必须显式补救这一点,要么通过交互式重标注,要么通过引入价值或世界模型。", + "label": "Imitation learning alone cannot recover from compounding errors", + "degree": 1 + }, + { + "id": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", + "label_zh": "世界模型作为内部模拟器解锁长时规划", + "kind": "insight", + "tier": "insight", + "topic": "world_models", + "phase": "core", + "year": 2018, + "summary_zh": "一旦智能体内部拥有可微、可分支的环境近似,规划就不再受真实交互成本限制,可以在想象中跑成千上万次试错。这条洞见把 RL 从样本贵的范式带向了样本高效的范式。", + "label": "World model as inner simulator unlocks long-horizon planning", + "degree": 4 + }, + { + "id": "insight:human_demonstrations_compress_implicit_reward_function", + "label_zh": "人类示教其实把隐式奖励压缩在轨迹里", + "kind": "insight", + "tier": "insight", + "topic": "deep_rl", + "phase": "core", + "year": 2016, + "summary_zh": "一组好的示教不只是状态-动作对,更是对某个未明说的奖励函数的优解。一旦认识到这点,逆强化学习、偏好学习、扩散策略都可以被理解为不同方式去解码这份隐式奖励。", + "label": "Human demonstrations compress an implicit reward function", + "degree": 2 + }, + { + "id": "insight:safety_emerges_from_constraint_lagrangian_not_reward_shaping", + "label_zh": "安全要靠约束 Lagrangian 而非奖励塑形", + "kind": "insight", + "tier": "insight", + "topic": "safety", + "phase": "core", + "year": 2017, + "summary_zh": "把碰撞惩罚塞进标量奖励里会被到达目标的回报抵消,策略仍可能选择高风险路径。把安全单独写成约束并用 Lagrangian 优化,让安全要求与性能要求分别有各自的对偶变量调控,是更稳健的安全 RL 范式。", + "label": "Safety emerges from constraint Lagrangian not reward shaping", + "degree": 3 + }, + { + "id": "insight:offline_rl_is_actually_constrained_dynamic_programming", + "label_zh": "离线 RL 本质上是带约束的动态规划", + "kind": "insight", + "tier": "insight", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "之所以 CQL、IQL、Cal-QL 等离线 RL 方法都奏效,是因为它们用不同方式把价值迭代约束在数据集支撑内。一旦明白离线 RL 等价于在数据集分布上做带约束的动态规划,就能从单一框架推出大量算法变体。", + "label": "Offline RL is actually constrained dynamic programming", + "degree": 4 + }, + { + "id": "insight:tokenized_trajectories_let_planning_borrow_from_language_modeling", + "label_zh": "把轨迹 token 化让规划可以借用语言模型工具", + "kind": "insight", + "tier": "insight", + "topic": "planning", + "phase": "frontier", + "year": 2021, + "summary_zh": "一旦把状态、动作和奖励都编码成离散 token,规划就变成了一个序列生成问题,可以直接套用 transformer、束搜索、扩散采样等成熟工具。这是 Decision Transformer、Trajeglish、CodeTrajectory 等工作背后的共同范式。", + "label": "Tokenized trajectories let planning borrow from language modeling", + "degree": 3 + }, + { + "id": "insight:bigger_model_plus_more_data_beats_clever_priors", + "label_zh": "更大模型加更多数据胜过精巧先验", + "kind": "insight", + "tier": "insight", + "topic": "deep_rl", + "phase": "core", + "year": 2019, + "summary_zh": "在驾驶规划与 RL 的多项基准上,朴素的大模型 + 大数据组合一次次追上甚至超过手写规则与精巧结构。这一观察既是苦涩教训在决策领域的延伸,也是采取保守归纳偏置时必须正视的事实。", + "label": "Bigger model plus more data beats clever priors", + "degree": 3 + }, + { + "id": "insight:control_theory_and_rl_meet_in_optimal_control", + "label_zh": "控制论与强化学习在最优控制处汇合", + "kind": "insight", + "tier": "insight", + "topic": "control", + "phase": "core", + "year": 2000, + "summary_zh": "LQR、iLQR、MPC 与 DP、值迭代、actor-critic 其实是从两条不同传统出发解决同一类带约束最优化问题。一旦认识到这一点,许多看似割裂的方法都能在 Bellman 方程的统一视角下被推导和对比。", + "label": "Control theory and RL meet in optimal control", + "degree": 3 + }, + { + "id": "paradigm:model_based_rl", + "label_zh": "基于模型的强化学习范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "world_models", + "phase": "core", + "year": 2018, + "summary_zh": "基于模型的 RL 把环境的转移和奖励学习成一个可查询的模型,再在该模型内用规划或想象训练策略。它的关键卖点是样本效率,代价是要承担模型偏差带来的策略偏离。", + "label": "Model-based RL", + "degree": 8 + }, + { + "id": "paradigm:model_free_rl", + "label_zh": "无模型强化学习范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "deep_rl", + "phase": "core", + "year": 1989, + "summary_zh": "无模型 RL 不显式学习环境,而是直接从经验里估计价值或策略梯度。它实现简单、收敛保证清晰,但在样本成本高的真实任务中往往需要海量交互。", + "label": "Model-free RL", + "degree": 13 + }, + { + "id": "paradigm:offline_rl", + "label_zh": "离线强化学习范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "deep_rl", + "phase": "core", + "year": 2020, + "summary_zh": "离线 RL 从一份固定的历史数据集中学习策略,部署前不再与环境交互。它把强化学习与监督式机器学习的实践模式拉近,但必须正面解决分布外动作带来的过估计问题。", + "label": "Offline RL", + "degree": 5 + }, + { + "id": "paradigm:imitation_learning", + "label_zh": "模仿学习范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "deep_rl", + "phase": "prereq", + "year": 1989, + "summary_zh": "模仿学习把策略学习当作监督学习:用专家轨迹做标签训练策略去复现专家行为。它实现简单、训练稳定,但在分布偏移和奖励缺失两方面有原则性的限制。", + "label": "Imitation learning", + "degree": 6 + }, + { + "id": "paradigm:optimal_control", + "label_zh": "最优控制范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "control", + "phase": "prereq", + "year": 1960, + "summary_zh": "最优控制把决策问题写成一个带动力学约束的最优化问题,用变分法或动态规划求解。LQR、iLQR、MPC、CILQR 都是它的不同求解策略,是工业自动驾驶规划栈的理论基石。", + "label": "Optimal control", + "degree": 6 + }, + { + "id": "paradigm:safe_rl", + "label_zh": "安全强化学习范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "safety", + "phase": "core", + "year": 2017, + "summary_zh": "安全 RL 把碰撞或代价违规等硬约束显式建模成约束马尔可夫决策过程,并用 Lagrangian、信赖域或形式化屏蔽确保策略改进的同时不破坏安全。它处于纯 RL 与控制论的交叉地带。", + "label": "Safe RL", + "degree": 4 + }, + { + "id": "paradigm:sequence_modeling_for_decision", + "label_zh": "决策的序列建模范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "planning", + "phase": "frontier", + "year": 2021, + "summary_zh": "该范式把决策过程整个看成一个序列建模问题,让 transformer 或扩散模型直接学习状态-动作-奖励序列的联合分布。Decision Transformer、Trajeglish、CodeTrajectory 都是它在不同任务上的实例。", + "label": "Sequence modeling for decision", + "degree": 6 + }, + { + "id": "paper:gpt4", + "label_zh": "GPT-4(多模态闭源大模型)", + "kind": "paper", + "tier": "S", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "GPT-4 是 OpenAI 在 GPT-3 之后推出的更大规模混合专家架构语言模型,在专业考试、代码与多步推理上首次接近人类专家水平。它通过引入指令微调与基于人类反馈的强化学习,把模型行为从纯模仿语料压向有用、诚实与无害三个目标,并成为后续所有商用闭源大模型的对比基准。", + "label": "GPT-4", + "degree": 10 + }, + { + "id": "paper:gpt4v", + "label_zh": "GPT-4V(视觉多模态扩展)", + "kind": "paper", + "tier": "S", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "summary_zh": "GPT-4V 在 GPT-4 文本主干之上插入图像编码器,使闭源大模型第一次具备读图、读图表和读复杂场景照片的能力。它在驾驶研究中被广泛用作零样本场景描述与异常检测基线,但同时也暴露了视觉语言模型在精细空间几何与遮挡推理上的系统性弱点。", + "label": "GPT-4V", + "degree": 2 + }, + { + "id": "paper:claude", + "label_zh": "Claude 系列(Anthropic 大模型)", + "kind": "paper", + "tier": "S", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "Claude 是 Anthropic 推出的对话式大模型,使用宪法式人工智能流程在没有人类逐条偏好打分的情况下完成自我对齐。它把一组自然语言原则当作显式宪法,让模型用自身改写答案以满足这些原则,是开源社区研究无人标注偏好对齐的主要参考实现。", + "label": "Claude", + "degree": 2 + }, + { + "id": "paper:gemini", + "label_zh": "Gemini(Google 多模态原生大模型)", + "kind": "paper", + "tier": "S", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "Gemini 是 Google DeepMind 把文本、图像、音频、视频与代码端到端原生混合训练的大模型,强调真正的多模态联合预训练而不是事后拼装编码器。它的长上下文窗口与对工具调用的工程化支持,使其成为机器人与驾驶领域研究长时域决策时的常用闭源参考。", + "label": "Gemini", + "degree": 4 + }, + { + "id": "paper:llama", + "label_zh": "LLaMA 系列(Meta 开源大模型)", + "kind": "paper", + "tier": "S", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "LLaMA 是 Meta 发布的开源大语言模型权重族,第一次以学术许可让研究者获得百亿到千亿参数规模的强基线。它直接催生了 Alpaca、Vicuna、Llama-2-Chat 等指令微调分支,使 VLM 与 VLA 研究不必再依赖封闭 API 就能复现训练流程。", + "label": "LLaMA family", + "degree": 6 + }, + { + "id": "paper:mistral", + "label_zh": "Mistral 与 Mixtral(稀疏专家开源大模型)", + "kind": "paper", + "tier": "A", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "Mistral 用滑动窗口注意力和分组查询注意力把稠密小模型推到同尺寸 LLaMA 之上,Mixtral 进一步引入稀疏专家路由把推理成本控制在激活参数量上。这一线证明在固定算力下结构层面的工程优化仍然有显著回报,为车端部署小型 VLM 提供模板。", + "label": "Mistral / Mixtral", + "degree": 4 + }, + { + "id": "paper:qwen", + "label_zh": "Qwen 与 Qwen-VL 系列", + "kind": "paper", + "tier": "A", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "Qwen 是阿里发布的中文与多语种开源大模型族,其 Qwen-VL 与 Qwen2-VL 分支把视觉编码器与语言主干联合微调到中文场景对话与文档理解上。它在国内驾驶相关 VLM 研究中是最常见的开源基座之一,许多 DriveLM 与 Senna 类工作直接基于 Qwen-VL 续训。", + "label": "Qwen series", + "degree": 3 + }, + { + "id": "paper:instructgpt", + "label_zh": "InstructGPT(指令微调 + RLHF)", + "kind": "paper", + "tier": "S", + "topic": "alignment", + "phase": "core", + "year": 2022, + "summary_zh": "InstructGPT 是 OpenAI 把指令式数据集与人类偏好强化学习串成完整对齐流程的奠基论文。它证明了一个相对较小但经过 RLHF 的模型在用户偏好上能稳定击败更大却只做语言建模的基线,从此奠定了所有商用大模型的三段式训练范式。", + "label": "InstructGPT", + "degree": 5 + }, + { + "id": "paper:constitutional_ai", + "label_zh": "Constitutional AI(宪法式自我对齐)", + "kind": "paper", + "tier": "A", + "topic": "alignment", + "phase": "core", + "year": 2022, + "summary_zh": "Constitutional AI 用一组写好的自然语言原则替代人类逐条偏好标注,让模型先自评再自改,最后再用强化学习把这些自评作为奖励信号。它的核心贡献是把对齐的人力瓶颈从打分员搬到原则书写者,给小团队复现 RLHF 提供了可行路径。", + "label": "Constitutional AI", + "degree": 5 + }, + { + "id": "paper:react", + "label_zh": "ReAct(思考与行动交替)", + "kind": "paper", + "tier": "S", + "topic": "llm_agent", + "phase": "core", + "year": 2022, + "summary_zh": "ReAct 在提示中显式交替写出推理和工具调用两类 token,使语言模型既能像思维链一样自我推演,又能像工具使用者一样调用搜索与计算器纠正自身幻觉。它是绝大多数现代语言模型代理框架以及 Agent-Driver 这一类驾驶认知代理工作的提示骨架。", + "label": "ReAct", + "degree": 9 + }, + { + "id": "paper:reflexion", + "label_zh": "Reflexion(语言式自我反思)", + "kind": "paper", + "tier": "A", + "topic": "llm_agent", + "phase": "core", + "year": 2023, + "summary_zh": "Reflexion 在每一轮失败之后让语言模型用自然语言写一段自我批评,并把这段批评作为下一轮提示的一部分,等价于一个文本梯度下降式的策略改进算子。它把强化学习里的奖励信号替换为自我生成的语言反馈,是 DiLu 等驾驶反思框架的直接灵感来源。", + "label": "Reflexion", + "degree": 4 + }, + { + "id": "paper:tot", + "label_zh": "Tree-of-Thoughts(思维树搜索)", + "kind": "paper", + "tier": "A", + "topic": "reasoning", + "phase": "core", + "year": 2023, + "summary_zh": "Tree-of-Thoughts 把单链思维链推广成对中间思考分支显式展开与回溯的搜索过程,让语言模型在每一步评估多条潜在思路再决定继续展开哪一条。它把推理本身建模为带启发式估值的搜索,为驾驶规划中的多假设展开提供了直接的语义类比。", + "label": "Tree-of-Thoughts", + "degree": 3 + }, + { + "id": "paper:toolformer", + "label_zh": "Toolformer(自监督学会工具调用)", + "kind": "paper", + "tier": "A", + "topic": "llm_agent", + "phase": "core", + "year": 2023, + "summary_zh": "Toolformer 让语言模型自己生成大量带工具调用标记的训练样本,并用调用结果是否降低后续 token 困惑度作为筛选信号,从而无需人工示范学会调用搜索、计算器与翻译器。这种自监督工具学习方法是把 API 集成进语言模型最干净的一条路径。", + "label": "Toolformer", + "degree": 3 + }, + { + "id": "paper:voyager", + "label_zh": "VOYAGER(Minecraft 终身学习代理)", + "kind": "paper", + "tier": "A", + "topic": "llm_agent", + "phase": "frontier", + "year": 2023, + "summary_zh": "VOYAGER 在 Minecraft 中用 GPT-4 维护一个可执行技能库,把每个新学到的技能写成可调用的代码并不断累积成层次化课程,从而展示出无监督开放式探索能力。它是把语言模型作为持续学习元控制器的代表作,与驾驶中的长时域终身学习思路同构。", + "label": "VOYAGER", + "degree": 6 + }, + { + "id": "paper:swiftsage", + "label_zh": "SwiftSage(双系统语言代理)", + "kind": "paper", + "tier": "B", + "topic": "llm_agent", + "phase": "frontier", + "year": 2023, + "summary_zh": "SwiftSage 把代理拆成一个快速反应的小模型 Swift 与一个谨慎反思的大模型 Sage,按任务困难度在两者之间切换。它是 Daniel Kahneman 双系统假说在语言代理上的直接工程化,与 DriveVLM-Dual 在驾驶上的快慢双环架构思想完全一致。", + "label": "SwiftSage", + "degree": 4 + }, + { + "id": "paper:flamingo", + "label_zh": "Flamingo(少样本视觉语言模型)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "prereq", + "year": 2022, + "summary_zh": "Flamingo 在冻结的语言模型中插入交叉注意力门,让视觉特征以加性方式注入而不破坏原始语言能力,从而实现真正的少样本图文上下文学习。它是后来 LLaVA 与 BLIP-2 等开源 VLM 注入视觉特征到 LLM 的概念原型。", + "label": "Flamingo", + "degree": 5 + }, + { + "id": "paper:palme", + "label_zh": "PaLM-E(具身多模态大模型)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "summary_zh": "PaLM-E 把视觉、状态向量与语言一起编码成连续 token 序列送入 PaLM 主干,使一个模型可以同时回答视觉问答与生成机器人控制指令。它第一次系统性地展示了大语言模型预训练带来的常识可以正迁移到具身控制策略上。", + "label": "PaLM-E", + "degree": 5 + }, + { + "id": "paper:rt1", + "label_zh": "RT-1(机器人 transformer 1)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "core", + "year": 2022, + "summary_zh": "RT-1 是 Google 用机器人采集数据训练的端到端 transformer 控制器,把图像与语言指令编码后离散化输出末端动作。它建立了机器人通用控制器的训练管线模板,也是 RT-2 与 OpenVLA 一系列后续视觉语言动作模型的直接前身。", + "label": "RT-1", + "degree": 1 + }, + { + "id": "paper:rt2", + "label_zh": "RT-2(视觉语言动作模型)", + "kind": "paper", + "tier": "S", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "RT-2 把动作输出重新编码成 PaLI-X 视觉语言模型词表中的文本 token,让一个原本只做视觉问答的大模型直接生成机器人动作序列。这一动作 token 化思想把控制问题转写成统一的自回归生成,是当前 VLA 范式的奠基工作。", + "label": "RT-2", + "degree": 10 + }, + { + "id": "paper:rtx", + "label_zh": "RT-X 与 Open X-Embodiment", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "RT-X 把来自二十多家实验室、几十种机器人本体的演示数据统一成 OpenX 格式并联合训练,证明跨本体训练能换来正迁移而非冲突。它把机器人视觉语言动作模型推进到大规模联合预训练阶段,是开源机器人基础模型的数据基石。", + "label": "RT-X / Open X-Embodiment", + "degree": 3 + }, + { + "id": "paper:openvla", + "label_zh": "OpenVLA(开源视觉语言动作模型)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "OpenVLA 在 LLaMA-2 与 SigLIP 之上复现并开放了 RT-2 风格的动作 token 训练管线,给学术界第一份可下载权重的 7B 级 VLA 基线。它把 VLA 研究的入门门槛从大厂资源拉回到单机多卡,是开源机器人控制基础模型的标志性节点。", + "label": "OpenVLA", + "degree": 9 + }, + { + "id": "paper:octo", + "label_zh": "Octo(开源跨本体策略)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "Octo 用基于 transformer 的扩散动作头在 Open X-Embodiment 数据上训练出一个支持多本体多视角接口的通用策略。它的核心贡献是把多模态条件、多种动作空间和多个传感器配置统一进同一个可微管线,是另一条与 OpenVLA 并行的开源 VLA 路线。", + "label": "Octo", + "degree": 3 + }, + { + "id": "paper:florence", + "label_zh": "Florence-2(统一视觉基础模型)", + "kind": "paper", + "tier": "B", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "summary_zh": "Florence-2 把检测、分割、关键点、描述等多种视觉任务都改写为 prompt 引导的文本生成,并在大规模图文数据上联合训练。它代表了一条把视觉任务全部归约为语言生成的统一基础模型路线,与 DETR 类纯结构通用化形成对照。", + "label": "Florence-2", + "degree": 3 + }, + { + "id": "paper:internvl", + "label_zh": "InternVL(大规模开源 VLM)", + "kind": "paper", + "tier": "B", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "summary_zh": "InternVL 把视觉编码器规模拉到与语言模型对齐的数十亿参数级别,并采用渐进式对齐策略训练,使开源 VLM 在中英文基准上首次接近 GPT-4V。它是后续国内驾驶领域 VLM 续训和评估的常用基座之一。", + "label": "InternVL", + "degree": 2 + }, + { + "id": "paper:cambrian", + "label_zh": "Cambrian-1(以视觉为中心的 VLM)", + "kind": "paper", + "tier": "B", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "Cambrian-1 系统性比较了 20 多种视觉编码器接入大语言模型的方式,并提出空间视觉聚合连接器以缓解高分辨率视觉特征的稀释。它强调当前 VLM 瓶颈不在语言侧而在视觉侧,对驾驶等需要精细空间理解的领域有直接指导意义。", + "label": "Cambrian-1", + "degree": 4 + }, + { + "id": "paper:sora", + "label_zh": "Sora(视频扩散基础模型)", + "kind": "paper", + "tier": "S", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "Sora 用扩散 transformer 在压缩潜空间中生成长时高分辨率视频,把图像扩散模型的范式推广到时空联合建模。OpenAI 把它定位为通用物理世界模拟器原型,也使视频生成与世界模型的边界开始消失。", + "label": "Sora", + "degree": 7 + }, + { + "id": "paper:veo", + "label_zh": "Veo(Google 视频生成模型)", + "kind": "paper", + "tier": "B", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "Veo 是 Google DeepMind 推出的高分辨率长视频生成模型,强调对相机轨迹和场景指令的可控条件输入。它与 Sora 并列代表视频生成迈向可控物理模拟器的工业实践,对驾驶仿真数据合成有直接溢出价值。", + "label": "Veo", + "degree": 1 + }, + { + "id": "paper:cosmos", + "label_zh": "NVIDIA Cosmos(物理 AI 世界基础模型)", + "kind": "paper", + "tier": "A", + "topic": "world_models", + "phase": "frontier", + "year": 2025, + "summary_zh": "NVIDIA Cosmos 是面向具身与自动驾驶的视频世界基础模型族,提供扩散与自回归两条主干以及驾驶专用的条件控制接口。它把视频生成模型显式定位为机器人与车辆训练用的反事实数据合成器,是世界模型工程化最完整的工业方案之一。", + "label": "NVIDIA Cosmos", + "degree": 13 + }, + { + "id": "paper:dit", + "label_zh": "DiT(扩散 transformer)", + "kind": "paper", + "tier": "A", + "topic": "world_models", + "phase": "core", + "year": 2022, + "summary_zh": "DiT 用纯 transformer 主干替换 U-Net 作为扩散模型的去噪网络,证明在大规模图像扩散上 transformer 同样能享受 scaling law。它是 Sora、Stable Diffusion 3 与 Cosmos 等视频世界模型的共同结构基础。", + "label": "DiT", + "degree": 3 + }, + { + "id": "paper:svd", + "label_zh": "Stable Video Diffusion", + "kind": "paper", + "tier": "B", + "topic": "world_models", + "phase": "core", + "year": 2023, + "summary_zh": "Stable Video Diffusion 把图像扩散模型在时间维度上加入卷积与注意力扩展,并发布开源权重,使学术界能够在视频生成与世界模型之间自由切换。它是 GAIA-1、DriveDreamer 等驾驶世界模型在权重层面常用的开源参考点。", + "label": "Stable Video Diffusion", + "degree": 3 + }, + { + "id": "paper:senna", + "label_zh": "Senna(驾驶 VLM 元动作框架)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "Senna 把驾驶决策拆成 VLM 输出语言元动作和小型端到端模型把元动作翻译成具体轨迹两层,让大模型只承担需要常识与意图理解的高层判断。它是驾驶领域元动作中介范式的代表,与 DriveVLM-Dual 并列。", + "label": "Senna", + "degree": 7 + }, + { + "id": "paper:emma", + "label_zh": "EMMA(端到端多模态 Waymo 模型)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "EMMA 是 Waymo 基于 Gemini 主干的端到端驾驶模型,将图像、地图与文本指令编码为 token,让 VLM 直接输出未来轨迹与解释性思维链。它展示了把闭源大模型作为驾驶单一神经栈的工业级可行性,但同时也带来推理延迟与可解释性新的挑战。", + "label": "EMMA", + "degree": 10 + }, + { + "id": "paper:drivelm", + "label_zh": "DriveLM(图结构问答驾驶数据集与模型)", + "kind": "paper", + "tier": "A", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "DriveLM 在 nuScenes 上构造按感知、预测、规划组织的图结构问答数据集,并提出在 VLM 上做图链式推理的训练范式。它把驾驶任务显式建模成一个可被语言模型展开的有向因果图,为驾驶领域思维链评测提供基准。", + "label": "DriveLM", + "degree": 3 + }, + { + "id": "paper:drivemlm", + "label_zh": "DriveMLM(多模态闭环驾驶大模型)", + "kind": "paper", + "tier": "B", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "DriveMLM 在 CARLA 闭环中把多视角图像、激光点云与语言指令一起送入 LLM,并输出与规则栈对接的离散行为决策。它是较早系统验证闭环条件下 LLM 决策可用性的工作,也暴露了模型延迟与决策一致性之间的硬约束。", + "label": "DriveMLM", + "degree": 4 + }, + { + "id": "paper:gpt_driver", + "label_zh": "GPT-Driver(语言模型驾驶规划)", + "kind": "paper", + "tier": "B", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "GPT-Driver 把感知输出序列化为文本场景描述送入 GPT,让大语言模型以坐标 token 形式输出未来轨迹。它把驾驶规划完全压成自回归文本生成任务,为后续轨迹 token 化与动作 token 化研究提供了最简版本的概念证明。", + "label": "GPT-Driver", + "degree": 4 + }, + { + "id": "paper:lmdrive", + "label_zh": "LMDrive(闭环语言驾驶代理)", + "kind": "paper", + "tier": "B", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "LMDrive 在 CARLA 中接入 LLaMA 类大模型,用自然语言指令实时驱动车辆,并提出语言指令到航点的端到端转换机制。它是开源社区可复现的闭环语言驾驶代理基线,与商用闭环系统形成对照实验。", + "label": "LMDrive", + "degree": 4 + }, + { + "id": "paper:prism1", + "label_zh": "Wayve PRISM-1(神经场驾驶仿真)", + "kind": "paper", + "tier": "B", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "Wayve PRISM-1 用四维神经场把真实采集的驾驶日志重建成可任意视角与可干预的动态场景,是 GAIA 系列世界模型的仿真互补工具。它把世界模型分成生成式与重建式两条线索,使闭环训练在没有外部模拟器的情况下也能进行。", + "label": "Wayve PRISM-1", + "degree": 4 + }, + { + "id": "paper:cot_wei2022", + "label_zh": "Chain-of-Thought 提示", + "kind": "paper", + "tier": "S", + "topic": "reasoning", + "phase": "core", + "year": 2022, + "summary_zh": "Chain-of-Thought 论文系统性展示了在提示中加入中间推理步骤可以让大语言模型在算术、常识与符号推理任务上获得显著质变。它是后续 ReAct、Tree-of-Thoughts、Reflexion 以及所有 VLM 驾驶解释链工作的概念起点。", + "label": "Chain-of-Thought", + "degree": 5 + }, + { + "id": "paper:self_consistency", + "label_zh": "自一致性解码", + "kind": "paper", + "tier": "A", + "topic": "reasoning", + "phase": "core", + "year": 2022, + "summary_zh": "自一致性解码在思维链之上采样多条独立推理路径然后对最终答案投票,把单点推理升级为蒙特卡洛式集成。它是几乎无成本就能显著提升语言模型推理可靠性的标准技术,也常被驾驶 VLM 用作多假设规划的轻量化版本。", + "label": "Self-Consistency", + "degree": 2 + }, + { + "id": "paper:debate", + "label_zh": "多智能体辩论", + "kind": "paper", + "tier": "B", + "topic": "reasoning", + "phase": "frontier", + "year": 2023, + "summary_zh": "多智能体辩论让多个语言模型实例针对同一问题相互质询并修正,最终由裁判模型综合出结论。它把对齐与可靠性问题转写为可扩展的多角色协作过程,为高风险驾驶决策提供另一条不依赖外部验证器的纠错机制。", + "label": "Multi-Agent Debate", + "degree": 3 + }, + { + "id": "paper:verifier", + "label_zh": "过程验证器与结果验证器", + "kind": "paper", + "tier": "B", + "topic": "reasoning", + "phase": "frontier", + "year": 2023, + "summary_zh": "过程验证器对推理链的每一步分别打分,结果验证器只评估最终答案;二者都用监督数据训练成单独模型并作为生成器的搜索向导。它在 OpenAI 与 DeepMind 的数学推理工作中被反复验证,是把强化学习风格价值函数引回语言推理的关键桥梁。", + "label": "Process / Outcome Verifier", + "degree": 4 + }, + { + "id": "move:scale_data_then_let_emergent_capabilities_appear", + "label_zh": "先把数据与算力堆上去再观察涌现能力", + "kind": "move", + "tier": "move", + "topic": "foundation_models", + "phase": "core", + "year": 2020, + "summary_zh": "这是一种刻意把规模放在结构创新之前的研究姿态,方法者承认无法预测哪些能力会突然出现,因此先沿着已知 scaling law 把训练规模拉到下一级,再回过头去刻画新冒出来的能力。它是 GPT-3、PaLM 和 Sora 等里程碑工作背后共同的方法论起手式。", + "label": "Scale then let capabilities emerge", + "degree": 4 + }, + { + "id": "move:pretrain_with_contrastive_alignment_between_modalities", + "label_zh": "用跨模态对比学习把不同模态对齐到同一空间", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "prereq", + "year": 2021, + "summary_zh": "这一招把两条模态的编码器训练成把语义对应的样本互相拉近、把无关样本互相推远,从而在零样本任务中复用语言空间。CLIP 是其代表,但同样的对比对齐思想也驱动了 SigLIP、ImageBind 等多模态基础模型,是把视觉信号桥接到语言模型的最简洁可扩展手段。", + "label": "Cross-modal contrastive pretraining", + "degree": 2 + }, + { + "id": "move:fine_tune_with_instruction_data_then_align_with_preferences", + "label_zh": "先用指令数据微调再用偏好数据对齐", + "kind": "move", + "tier": "move", + "topic": "alignment", + "phase": "core", + "year": 2022, + "summary_zh": "这一招把对齐过程拆成两段:先用大量带格式的指令对让模型学会服从任务约定,再用人类或宪法式偏好对让模型学会在多个合规候选中挑出更好的那一个。它已经成为所有商用大模型的默认训练阶段划分,也是把基座模型变成可交付产品的最小工艺集。", + "label": "Instruction tune then preference align", + "degree": 3 + }, + { + "id": "move:plug_in_modality_encoder_to_frozen_language_model_via_projection", + "label_zh": "用一个投影头把模态编码器插进冻结语言模型", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "summary_zh": "这一招保留预训练语言模型权重不动,只训练一个把视觉或音频特征线性映射到语言 token 空间的小投影模块,从而最大化复用昂贵语言能力。LLaVA、BLIP-2 与 MiniGPT-4 都是这一移动的实例,它也成为学术实验室能用单卡复现多模态大模型的关键工艺。", + "label": "Plug modality encoder via projection", + "degree": 3 + }, + { + "id": "move:wrap_language_model_with_tool_calling_loop", + "label_zh": "在语言模型外面套一层工具调用循环", + "kind": "move", + "tier": "move", + "topic": "llm_agent", + "phase": "core", + "year": 2023, + "summary_zh": "把语言模型当成可以生成函数调用 token 的策略,再把调用结果回灌成新的上下文,循环直到任务完成。这一招把无状态生成器升级成可以与环境交互的代理,是 ReAct、Toolformer、Agent-Driver 等大量代理框架的共有骨架。", + "label": "Wrap LM with tool-calling loop", + "degree": 3 + }, + { + "id": "move:add_reflection_step_so_agent_critiques_its_own_output", + "label_zh": "在代理循环中加入自我反思与批评步骤", + "kind": "move", + "tier": "move", + "topic": "llm_agent", + "phase": "core", + "year": 2023, + "summary_zh": "在每一轮行动之后专门留一段空间让模型回顾刚才的决策并写出文字批评,然后把这段批评作为下一轮的额外输入。它把强化学习中的奖励信号换成自然语言反馈,是 Reflexion、DiLu 等让代理在不更新权重的情况下持续改进的关键招式。", + "label": "Add reflection step for self-critique", + "degree": 4 + }, + { + "id": "move:replace_explicit_action_head_with_tokenized_action_sequence", + "label_zh": "把显式动作头替换为离散化的动作 token 序列", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "把连续控制量量化成有限词表,再让语言模型像写句子一样自回归生成动作 token,从而把控制与语言生成统一进同一套预训练。RT-2、OpenVLA、GPT-Driver 都用这一招把驾驶或机械臂控制问题改写为大模型友好的形式。", + "label": "Tokenize action sequence", + "degree": 5 + }, + { + "id": "move:augment_supervised_training_with_counterfactual_or_synthetic_data", + "label_zh": "用反事实或合成数据扩充监督训练", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "这一招用世界模型或仿真器生成被有意修改过条件的反事实数据,让监督训练接触到现实数据里从未出现但物理上合理的极端情形。它是缓解长尾分布、解耦虚假相关、训练稳健驾驶策略的关键工具,CF-VLA 与 Cosmos 都把它作为核心机制。", + "label": "Augment with counterfactual synthetic data", + "degree": 3 + }, + { + "id": "move:condition_video_generative_model_on_control_action_for_world_model", + "label_zh": "用控制动作条件化视频生成模型构成世界模型", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "frontier", + "year": 2023, + "summary_zh": "在视频扩散或自回归生成模型上加入未来动作或控制信号作为条件输入,使模型能回答如果车辆这样行动场景会如何演化。GAIA-1、DriveDreamer 与 Cosmos 都用这一招把生成模型升级成可被规划器查询的隐式物理引擎。", + "label": "Condition video generator on action", + "degree": 3 + }, + { + "id": "move:use_retrieval_augmented_memory_to_extend_context", + "label_zh": "用检索增强的外部记忆扩展上下文", + "kind": "move", + "tier": "move", + "topic": "llm_agent", + "phase": "core", + "year": 2022, + "summary_zh": "在固定窗口的语言模型外面挂一个可读写的向量索引,把长期经验存到外部库里,需要时检索回来拼进上下文。这一招把有限上下文窗口扩展成事实上的无限记忆,是构建终身学习代理与驾驶经验库的标准工程模式。", + "label": "Retrieval-augmented memory", + "degree": 3 + }, + { + "id": "move:cast_reasoning_as_search_over_thought_tree", + "label_zh": "把推理过程刻画为思维树上的搜索", + "kind": "move", + "tier": "move", + "topic": "reasoning", + "phase": "frontier", + "year": 2023, + "summary_zh": "在每一步推理之后保留多个候选展开方向并用启发式或验证器评估,再选择有前途的分支继续展开,必要时回溯。Tree-of-Thoughts、AlphaCode 与各类数学推理工作都用这一招把语言生成提升到带显式状态搜索的层级。", + "label": "Cast reasoning as tree search", + "degree": 3 + }, + { + "id": "move:co_finetune_language_model_with_action_data_jointly", + "label_zh": "把动作数据与语言数据联合微调到同一模型", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "在 VLA 训练中把机器人或驾驶动作数据与通用图文问答数据按比例混合进行联合微调,避免动作微调灾难性遗忘语言能力。RT-2、OpenVLA 与 EMMA 都依赖这一招在不丧失视觉问答能力的前提下学会输出动作。", + "label": "Co-finetune LM with action data", + "degree": 4 + }, + { + "id": "move:use_self_play_to_generate_unlimited_training_signal", + "label_zh": "用自博弈生成无尽训练信号", + "kind": "move", + "tier": "move", + "topic": "reasoning", + "phase": "core", + "year": 2017, + "summary_zh": "让模型与自己博弈或互相批评,从而把训练信号的瓶颈从人工标注搬到计算量上。AlphaZero 是其经典例证,多智能体辩论与宪法式自我改写则是其在语言模型时代的等价物,对扩展驾驶 corner case 训练同样有启示。", + "label": "Self-play for unlimited signal", + "degree": 3 + }, + { + "id": "move:distill_large_model_into_specialist_for_deployment", + "label_zh": "把大模型蒸馏成可部署的专用模型", + "kind": "move", + "tier": "move", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "把缓慢但能干的大模型当成教师生成行为数据或软标签,再用一个小很多的学生网络去拟合,使最终上车的模型既保留大模型常识又能满足实时约束。它是把 VLM 与 VLA 落到车端控制器的几乎所有工业线的必走步骤。", + "label": "Distill into deployable specialist", + "degree": 4 + }, + { + "id": "move:rewrite_continuous_video_as_token_sequence_for_transformer_world_model", + "label_zh": "把连续视频改写成离散 token 喂给 transformer 世界模型", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "frontier", + "year": 2023, + "summary_zh": "用 VAE 或 VQ-VAE 把每一帧压缩成离散视觉 token,再把整段视频拼成 token 序列让 transformer 像处理语言那样建模。GAIA-1、Genie 与 Cosmos 自回归分支都依赖这一招把世界模型嵌入到统一 token 化框架。", + "label": "Tokenize video for transformer world model", + "degree": 2 + }, + { + "id": "move:condition_on_language_meta_action_then_emit_low_level_action", + "label_zh": "先输出语言元动作再翻译成低层动作", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "在驾驶或机械臂决策中先让 VLM 输出诸如让行、跟车、避让等少量语言元动作,再让小型专用模型把元动作转译为连续轨迹或电机指令。Senna 与 DriveVLM-Dual 都是这一招的代表,把语义判断与几何细化解耦。", + "label": "Language meta-action then low-level action", + "degree": 3 + }, + { + "id": "move:cache_kv_state_to_amortize_long_context", + "label_zh": "缓存键值状态以摊销长上下文成本", + "kind": "move", + "tier": "move", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "把推理时的注意力键值张量保存下来,下一步只追加新的 token 并复用旧缓存,把长序列推理成本从二次降到线性。它是把大模型推理时延控制到车端可接受范围的最常用工程招式,也是 vLLM 与 TensorRT-LLM 的核心机制。", + "label": "Cache KV state to amortize context", + "degree": 3 + }, + { + "id": "move:speculative_decoding_with_draft_model", + "label_zh": "用小草稿模型推测式解码加速大模型", + "kind": "move", + "tier": "move", + "topic": "foundation_models", + "phase": "frontier", + "year": 2023, + "summary_zh": "用一个便宜的小模型先生成若干候选 token,再让大模型一次性并行验证这些候选并接受其中可被复现的前缀,从而在不改变分布的前提下显著降低延迟。它是让大模型在驾驶等实时场景里勉强可用的关键推理优化手段。", + "label": "Speculative decoding with draft model", + "degree": 3 + }, + { + "id": "move:freeze_visual_encoder_and_only_train_connector", + "label_zh": "冻结视觉编码器只训练连接器", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "summary_zh": "把代价昂贵的视觉编码器和语言模型都冻结,仅训练中间的小型连接器或交叉注意力,使多模态对齐可以在很小算力下完成。这是 BLIP-2、LLaVA 初代以及众多驾驶 VLM 落地工作的事实标准入门姿态。", + "label": "Freeze visual encoder, train connector", + "degree": 3 + }, + { + "id": "move:use_diffusion_head_for_continuous_action", + "label_zh": "用扩散头建模连续动作分布", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "在策略网络末端用条件扩散模型采样连续控制量,从而自然刻画多模态动作分布和不确定性,避开离散 token 量化误差。Octo、Diffusion Policy 与多种轨迹规划工作都采用这一招在精细控制场景下替代单点回归。", + "label": "Diffusion head for continuous action", + "degree": 2 + }, + { + "id": "move:treat_planning_as_autoregressive_trajectory_generation", + "label_zh": "把规划问题视为自回归轨迹生成", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "把未来一段时间的航点序列当作 token 序列让 transformer 一步一步生成,把规划与语言生成统一进同一接口。GPT-Driver、EMMA 与多种端到端规划网络都用这一招把规划架在大模型生成能力之上。", + "label": "Planning as autoregressive trajectory", + "degree": 3 + }, + { + "id": "move:use_world_model_rollout_as_critic_for_policy", + "label_zh": "用世界模型滚动作为策略的批评者", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "让世界模型对策略提议的多个候选动作分别模拟若干步未来,再用一个评估函数比较结果,从而把世界模型当成可微或可查询的批评者。Dreamer 家族、CF-VLA 与基于 Sora 类模型的规划研究都共享这一招的思想。", + "label": "World-model rollout as critic", + "degree": 5 + }, + { + "id": "move:long_horizon_via_hierarchical_subgoal", + "label_zh": "用层次化子目标处理长时域决策", + "kind": "move", + "tier": "move", + "topic": "llm_agent", + "phase": "frontier", + "year": 2023, + "summary_zh": "把长时任务先用语言模型分解成有序子目标列表,再为每个子目标调用低层策略或者再次递归分解。VOYAGER、SwiftSage 与多种规划代理都用这一招对抗有限上下文窗口与稀疏奖励,把任务难度向下展开成可解决的小步。", + "label": "Long-horizon via hierarchical subgoal", + "degree": 4 + }, + { + "id": "move:prompt_chain_with_explicit_persona_roles", + "label_zh": "用显式角色化的提示链分工", + "kind": "move", + "tier": "move", + "topic": "llm_agent", + "phase": "core", + "year": 2023, + "summary_zh": "把同一个大模型用不同系统提示扮演规划者、批评者、执行者等角色再彼此对话,把单体语言模型变成可解释的多角色管线。它把对齐与协作问题搬到提示工程层面,是大量驾驶认知代理工作的低成本起步形态。", + "label": "Prompt chain with persona roles", + "degree": 3 + }, + { + "id": "move:contrast_corner_case_against_normal_case_in_training", + "label_zh": "在训练中显式对比 corner case 与常规情况", + "kind": "move", + "tier": "move", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "在数据组织阶段刻意把同一场景的正常版本与被扰动出 corner case 的版本配对呈现,使模型学到的不只是平均行为而是变化的边界。这一招在驾驶安全和反事实训练里尤为关键,是 CF-VLA 与某些 Cosmos 子任务的核心设计。", + "label": "Contrast corner vs normal cases", + "degree": 3 + }, + { + "id": "move:evaluate_open_loop_then_close_loop_for_realism", + "label_zh": "先做开环评估再切到闭环验证现实性", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "core", + "year": 2022, + "summary_zh": "先在静态数据集上跑指标筛掉明显不合格的模型,再在仿真或路测里做闭环以暴露分布偏移与累积误差。它是驾驶研究从论文走向工程必经的两阶段评估姿态,避免开环指标好看但闭环失稳的常见陷阱。", + "label": "Open-loop then closed-loop eval", + "degree": 4 + }, + { + "id": "move:use_language_explanation_as_auxiliary_supervision", + "label_zh": "把语言解释作为辅助监督信号", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "core", + "year": 2024, + "summary_zh": "在训练驾驶策略或感知网络时加入对决策原因的自然语言解释作为附加输出,要求模型同时回答做了什么和为什么。LINGO、DriveLM 与多种带解释 VLA 工作都用这一招把模型逼向人类可审计的内部表征。", + "label": "Language explanation as auxiliary supervision", + "degree": 3 + }, + { + "id": "problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop", + "label_zh": "VLM 在安全关键回路中产生幻觉动作", + "kind": "problem", + "tier": "problem", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "视觉语言模型擅长生成看起来合理的解释和动作,但这种合理性并不蕴含物理可执行性,一旦把它直接闭环到车辆控制就可能输出违反约束甚至危险的动作。如何在保持表达力的同时给 VLA 加上硬约束验证,是当前安全可部署的核心难题。", + "label": "Hallucinated action in safety-critical loop", + "degree": 3 + }, + { + "id": "problem:grounding_language_token_to_continuous_physical_world", + "label_zh": "把离散语言 token 接地到连续物理世界", + "kind": "problem", + "tier": "problem", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "语言模型在离散符号空间中训练,但要驱动机械臂或车辆就必须输出有度量含义的连续量,这之间存在天然的语义到物理的接地鸿沟。如何系统性地学到这种接地,而不是依赖手工动作 token 表,是 VLA 与具身智能的根本未解问题。", + "label": "Grounding language to physical world", + "degree": 3 + }, + { + "id": "problem:latency_budget_for_large_model_in_realtime_control", + "label_zh": "大模型在实时控制中的延迟预算", + "kind": "problem", + "tier": "problem", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "车辆控制循环需要在十毫秒量级响应,而当前 VLM 与 VLA 在车端硬件上的单次推理常常超过这个预算几个数量级。如何用蒸馏、缓存、推测式解码与双系统架构把大模型推理压进控制周期,决定了 VLA 是否真能上车。", + "label": "Latency budget for large model control", + "degree": 4 + }, + { + "id": "problem:long_horizon_reasoning_with_finite_context_window", + "label_zh": "有限上下文窗口下的长时域推理", + "kind": "problem", + "tier": "problem", + "topic": "llm_agent", + "phase": "core", + "year": 2023, + "summary_zh": "驾驶任务跨越分钟甚至小时尺度,但语言模型的注意力代价随窗口长度二次增长,必须在固定窗口内表达远超窗口的历史信息。如何用外部记忆、层次化总结与世界模型来弥补这一矛盾,是把代理范式推进到真实长时任务的瓶颈。", + "label": "Long-horizon reasoning with finite context", + "degree": 3 + }, + { + "id": "problem:zero_shot_generalization_to_unseen_driving_scenes", + "label_zh": "对未见驾驶场景的零样本泛化", + "kind": "problem", + "tier": "problem", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "驾驶世界几乎不可能被采集穷尽,模型必须在训练分布之外保持合理行为。VLM 因为预训练数据广泛而被寄予厚望,但实证显示它们在精细几何与新国家路况上仍频繁失败,因此零样本泛化的真正机制与边界还远未被理解。", + "label": "Zero-shot generalization to unseen scenes", + "degree": 2 + }, + { + "id": "problem:fine_grained_spatial_understanding_in_vision_language_model", + "label_zh": "视觉语言模型的精细空间理解", + "kind": "problem", + "tier": "problem", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "当前 VLM 在描述场景大意时表现良好,但要它说出两辆车之间的精确距离、车道几何或视差关系时几乎一律失败。这种细粒度空间推理的缺失直接限制了 VLM 在驾驶感知与规划中的可信度,是 Cambrian 等工作集中攻击的方向。", + "label": "Fine-grained spatial understanding in VLM", + "degree": 2 + }, + { + "id": "problem:counterfactual_reasoning_about_other_agents_intent", + "label_zh": "对他车意图的反事实推理", + "kind": "problem", + "tier": "problem", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "安全驾驶要求模型能想象其他交通参与者在不同假设下会怎样反应,而不是只对实际观察到的轨迹做拟合。如何让 VLA 或世界模型显式表示并搜索这些反事实分支,是 CF-VLA、Cosmos 等工作正在试图把握的研究问题。", + "label": "Counterfactual reasoning about other agents", + "degree": 3 + }, + { + "id": "problem:open_world_corner_case_synthesis_for_training", + "label_zh": "开放世界 corner case 的合成", + "kind": "problem", + "tier": "problem", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "现实道路里真正危险的 corner case 罕见到无法靠采集解决,必须由生成模型在保持物理合理性的前提下大量合成。如何衡量合成 corner case 的覆盖率与有效性,又如何避免模型只学到合成数据上的虚假相关,是世界模型工程化的核心未解问题。", + "label": "Open-world corner case synthesis", + "degree": 3 + }, + { + "id": "problem:evaluation_gap_between_offline_benchmark_and_closed_loop", + "label_zh": "离线基准与闭环表现之间的评估鸿沟", + "kind": "problem", + "tier": "problem", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "summary_zh": "目前的离线 VQA 与轨迹相似度指标无法可靠预测模型在闭环里的真实安全性,许多在 DriveLM 上得高分的 VLM 在 CARLA 闭环里反而失败。如何设计能贯通开闭环的评估指标,是 VLA 研究是否能形成可累积进展的方法学前提。", + "label": "Offline-to-closed-loop evaluation gap", + "degree": 3 + }, + { + "id": "problem:catastrophic_forgetting_after_action_finetuning", + "label_zh": "动作微调后的灾难性遗忘", + "kind": "problem", + "tier": "problem", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "当 VLM 被进一步用动作数据微调成 VLA 时,原本强大的视觉问答与语言推理能力常常显著退化,使得模型在需要常识的边界情况下反而比微调前更糟。如何在保持语言能力的同时引入动作能力,是 RT-2、OpenVLA 等工作面对的工程难题。", + "label": "Catastrophic forgetting after action finetune", + "degree": 3 + }, + { + "id": "insight:language_is_compressed_world_model_for_human_priors", + "label_zh": "语言是人类先验的压缩世界模型", + "kind": "insight", + "tier": "insight", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "人类用语言把对世界因果、物理和社会的判断压缩在文本里,使得在大规模文本上预训练得到的模型自动继承了这些先验。这一洞察解释了为什么纯文本预训练的 LLM 在驾驶常识、社会博弈等领域不需要进一步训练就能给出合理答案,也支撑了语言作为通用世界模型代理的研究路线。", + "label": "Language as compressed world model", + "degree": 3 + }, + { + "id": "insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models", + "label_zh": "扩大规模解锁小模型上不存在的能力", + "kind": "insight", + "tier": "insight", + "topic": "foundation_models", + "phase": "core", + "year": 2022, + "summary_zh": "经验上某些能力例如多步推理、零样本指令跟随只在模型规模与数据量越过某个阈值后才突然出现,而无法在小模型上靠精调获得。这一洞察支撑了苦涩教训所倡导的把工程资源投入到通用扩展而非领域规则上的方法论选择。", + "label": "Scaling unlocks emergent capabilities", + "degree": 3 + }, + { + "id": "insight:world_model_video_diffusion_is_implicit_physics_engine", + "label_zh": "视频扩散模型是隐式物理引擎", + "kind": "insight", + "tier": "insight", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "在足够大的视频数据上训练的扩散模型自动学到了惯性、碰撞、光照与物体持久性等近似物理规律,使它们成为不需要显式方程的可查询世界模型。这一洞察使 Sora、GAIA、Cosmos 等模型被重新定位为机器人和驾驶的通用模拟器,而不仅仅是内容生成器。", + "label": "Video diffusion as implicit physics engine", + "degree": 3 + }, + { + "id": "insight:agent_loop_is_just_iterated_conditional_generation", + "label_zh": "代理循环本质是反复条件生成", + "kind": "insight", + "tier": "insight", + "topic": "llm_agent", + "phase": "core", + "year": 2023, + "summary_zh": "ReAct、Reflexion、VOYAGER 等代理框架虽然形态各异,但都可以被统一理解为把工具结果与历史观察拼回上下文后再次条件生成下一段动作。这一洞察把代理设计简化为如何构造每一步的条件输入与如何聚合长期记忆两件事,是搭建驾驶认知代理的统一抽象。", + "label": "Agent loop is iterated conditional generation", + "degree": 3 + }, + { + "id": "insight:tool_use_extends_language_model_into_environment_grounded_actor", + "label_zh": "工具使用把语言模型扩展为接地于环境的执行者", + "kind": "insight", + "tier": "insight", + "topic": "llm_agent", + "phase": "core", + "year": 2023, + "summary_zh": "一旦语言模型可以生成结构化函数调用并消费其返回值,它就从无状态文本生成器变成了可以查询事实、操作仿真器与控制车辆的环境接地执行者。这一洞察是 Toolformer、ReAct 与 Agent-Driver 共同的方法论核心,也把对齐重心从输出文本搬到了选择动作。", + "label": "Tool use extends LM into actor", + "degree": 3 + }, + { + "id": "insight:counterfactual_replanning_separates_intent_from_execution", + "label_zh": "反事实重规划把意图与执行解耦", + "kind": "insight", + "tier": "insight", + "topic": "world_models", + "phase": "frontier", + "year": 2025, + "summary_zh": "通过让模型对相同高层意图模拟多种动作轨迹再选择最优,可以把语义层面的我想做什么与几何层面的我该怎么做分离开来。CF-VLA 把这一思想工程化,使 VLM 输出的意图可以经由世界模型反事实验证后再交给底层控制器执行。", + "label": "Counterfactual replanning separates intent from execution", + "degree": 2 + }, + { + "id": "insight:foundation_model_decouples_perception_from_task_specific_training", + "label_zh": "基础模型把感知与任务特定训练解耦", + "kind": "insight", + "tier": "insight", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "DINOv2、SAM、Florence 等通用感知基础模型把以前各任务都要重复训练的视觉骨干变成可复用的冻结服务,使下游 AD 工作只需要在轻量任务头上微调。这一洞察重塑了感知研究的劳动分工,把绝大多数科研价值压到任务设计与数据策展上。", + "label": "Foundation model decouples perception from task training", + "degree": 3 + }, + { + "id": "insight:dual_system_fast_slow_loop_marries_reactive_and_deliberative_control", + "label_zh": "快慢双系统循环融合反应式与审议式控制", + "kind": "insight", + "tier": "insight", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "Kahneman 的快慢思维启发的双系统架构在 SwiftSage 与 DriveVLM-Dual 中被工程化为高频小模型与低频大模型并行的控制结构。这一洞察既保留了大模型常识又满足车端实时性,是当前 VLA 落地的事实标准形态。", + "label": "Dual-system fast-slow loop", + "degree": 2 + }, + { + "id": "insight:emergent_planning_from_next_token_prediction_alone", + "label_zh": "纯下一 token 预测中涌现出的规划能力", + "kind": "insight", + "tier": "insight", + "topic": "reasoning", + "phase": "frontier", + "year": 2023, + "summary_zh": "尽管语言模型只优化下一个 token 的对数似然,但在足够规模与数据下它们表现出对未来若干步的隐式规划行为,包括在数学证明与代码生成中明显的目标驱动结构。这一洞察暗示规划与生成在足够大的模型上趋同,对端到端驾驶研究有深远启示。", + "label": "Emergent planning from next-token prediction", + "degree": 2 + }, + { + "id": "insight:alignment_is_constraint_satisfaction_over_generation", + "label_zh": "对齐本质是对生成的约束满足", + "kind": "insight", + "tier": "insight", + "topic": "alignment", + "phase": "core", + "year": 2023, + "summary_zh": "RLHF、DPO 与宪法式自我对齐都可以被看作在已经强大的生成分布上添加额外约束,让满足偏好或原则的样本概率上升而不满足的样本概率下降。这一视角统一了多种对齐技术,也把安全驾驶决策的对齐问题嵌入相同的形式框架。", + "label": "Alignment as constraint satisfaction", + "degree": 3 + }, + { + "id": "insight:open_weight_release_compounds_research_velocity", + "label_zh": "开放权重发布复利化研究速度", + "kind": "insight", + "tier": "insight", + "topic": "foundation_models", + "phase": "core", + "year": 2023, + "summary_zh": "LLaMA、Mistral、Qwen、OpenVLA 等开源权重的发布让任何研究者都能在强基线上做受控实验,而不必从零训练,使整个领域的实验速度形成正反馈。这一洞察解释了为什么开源策略在长期上对方法论进步的贡献往往超过单点最强闭源模型。", + "label": "Open weights compound research velocity", + "degree": 4 + }, + { + "id": "paradigm:foundation_model_axis", + "label_zh": "基础模型与 VLA 轴范式总览", + "kind": "paradigm", + "tier": "paradigm", + "topic": "foundation_models", + "phase": "core", + "year": 2024, + "summary_zh": "这一范式把所有具备通用能力的视觉、语言、视频、动作模型组织在同一研究轴上,强调它们共享 transformer 主干、自监督预训练与下游联合微调的相同方法论。它是组织从 GPT-3 到 Cosmos、再到 EMMA 与 CF-VLA 这一巨大谱系的统一坐标系。", + "label": "Foundation Model Axis", + "degree": 6 + }, + { + "id": "paradigm:world_model_paradigm", + "label_zh": "世界模型范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "world_models", + "phase": "frontier", + "year": 2024, + "summary_zh": "世界模型范式主张把环境动力学独立学到一个可被规划器查询或可被策略联合训练的生成模型里,从而把决策与感知解耦。从 Ha 与 Schmidhuber 的 World Models 到 Dreamer、GAIA-1、DriveDreamer 和 Cosmos,这一范式贯穿强化学习与自动驾驶研究的多个时代。", + "label": "World Model Paradigm", + "degree": 5 + }, + { + "id": "paradigm:llm_agent_paradigm", + "label_zh": "大模型代理范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "llm_agent", + "phase": "frontier", + "year": 2024, + "summary_zh": "大模型代理范式把语言模型当成具备状态、工具、记忆的通用决策器,并通过工具循环、反思与层次分解构造长时域行为。它在驾驶领域以 Agent-Driver、DiLu、DriveVLM-Dual 等形态出现,是把语言模型常识转化为可执行驾驶策略的主要研究路径。", + "label": "LLM Agent Paradigm", + "degree": 5 + }, + { + "id": "paradigm:vla_paradigm", + "label_zh": "视觉语言动作范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "视觉语言动作范式把视觉、语言与控制动作统一进同一个自回归或扩散生成模型,让一个网络同时承担感知、推理与控制三个传统上分离的角色。RT-2、OpenVLA、EMMA、CF-VLA 都是这一范式在机器人与驾驶上的代表实例。", + "label": "VLA Paradigm", + "degree": 6 + }, + { + "id": "move:residual_connection", + "label_zh": "残差连接(恒等捷径)", + "kind": "move", + "tier": "move", + "topic": "math_foundations", + "phase": "prereq", + "year": 2015, + "summary_zh": "残差连接是一种方法学原语,它让网络的每一层学习相对于输入的偏差而非完整映射,从而把恒等函数作为优化的默认起点。这一移动将深度网络的可训练性问题转化为残差函数的学习问题,使梯度可以沿恒等通路直达浅层。它最早在 ResNet 中被系统化,随后被 Transformer、扩散模型、Diffusion Policy 和残差策略学习反复借用。在自动驾驶中,残差策略可以叠加在规则规划器之上,从而在保留可解释安全行为的同时让数据驱动模块进行局部修正。", + "building_blocks": [], + "label": "Residual Connection", + "degree": 3 + }, + { + "id": "move:patchify_tokenization", + "label_zh": "Patch 分块与 token 化", + "kind": "move", + "tier": "move", + "topic": "math_foundations", + "phase": "prereq", + "year": 2020, + "summary_zh": "Patch 化是将连续高维信号(图像、点云、视频、语音)切分为离散 token 序列的方法学动作,目的是让 transformer 这类与序列长度近似线性相关的架构能够直接处理非语言模态。这一移动在 ViT 中第一次把图像视为 16×16 的 patch 序列,在 VideoMAE 中扩展到时空管道,在驾驶感知中扩展到 BEV 网格 token 与点云体素 token。它揭示了一个普适规律,即模态之间的差异往往可以通过 tokenizer 的设计而非主干结构的差异来消化。", + "building_blocks": [], + "label": "Patch / Token Reshaping", + "degree": 3 + }, + { + "id": "move:masking_for_pretext", + "label_zh": "掩码预测自监督任务", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "core", + "year": 2018, + "summary_zh": "掩码并预测是一类基础的自监督移动,其核心是把输入的一部分隐藏起来并要求模型从可见部分重构被掩盖的部分,从而免费获得无穷多的监督信号。BERT 把它用于文本片段,MAE 把它用于图像块,VideoMAE 把它用于时空体,占据栅格预训练把它用于驾驶 BEV 占据。在自动驾驶中,将未来帧的占据栅格或邻车轨迹掩码起来再预测,可以作为大规模无标签预训练任务来缓解长尾驾驶事件的标注稀缺。", + "building_blocks": [], + "label": "Masked Prediction Pretext", + "degree": 3 + }, + { + "id": "move:contrastive_alignment", + "label_zh": "对比对齐", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "core", + "year": 2020, + "summary_zh": "对比对齐通过把成对的样本在嵌入空间拉近、把非成对样本推远来学习一个共享表征,这一移动在 SimCLR 中用于图像增广对,在 CLIP 中用于图文配对,在 Audio-CLIP 中扩展到音频。其本质是用相对的相似性结构而非绝对标签来定义任务,从而能够利用互联网规模的天然配对数据。在自动驾驶中,将驾驶日志片段与人类自然语言解释做对比对齐,可以使模型实现零样本场景检索和自然语言条件下的轨迹生成。", + "building_blocks": [], + "label": "Contrastive Alignment", + "degree": 3 + }, + { + "id": "move:cross_attention_query", + "label_zh": "类型化 cross-attention query", + "kind": "move", + "tier": "move", + "topic": "math_foundations", + "phase": "core", + "year": 2020, + "summary_zh": "把任何关心的实体表达为一组可学习的 query 向量,让它们通过 cross-attention 从一个公共特征记忆中拉取信息,是一类极其普适的方法学移动。DETR 把它用于目标检测的对象 query,BEVFormer 把它用于鸟瞰图网格 query,UniAD 进一步把它用于代理 query 与地图 query。在自动驾驶中,每一种司机关心的对象都可以被参数化为 query,并通过 cross-attention 实现感知、预测、规划在共享潜空间中的对话。", + "building_blocks": [], + "label": "Typed Cross-Attention Query", + "degree": 9 + }, + { + "id": "move:lift_2d_to_3d", + "label_zh": "2D 升至 3D 的 lift-splat", + "kind": "move", + "tier": "move", + "topic": "e2e_ad", + "phase": "prereq", + "year": 2020, + "summary_zh": "Lift-Splat 是一类把二维图像特征通过深度概率或可学习投影抬升到三维空间、再投影到鸟瞰图的方法学移动。该移动建立了一个跨视角共享的几何一致表征,使得多相机感知、占据预测与运动规划可以在同一坐标系中协作。它最早在 Lift-Splat-Shoot 中明确提出,在 BEVDet、BEVFormer 中演化出深度监督与时空 query 变体,并成为现代端到端自动驾驶的事实标准前端。", + "building_blocks": [], + "label": "Lift-Splat to BEV", + "degree": 3 + }, + { + "id": "move:set_prediction_with_hungarian", + "label_zh": "匈牙利匹配下的集合预测", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "core", + "year": 2020, + "summary_zh": "集合预测把检测、跟踪、规划等任务统一为输出一个无序集合,并通过匈牙利算法在预测与真值之间做最优二分匹配以计算损失,从而消除了对手工设计的非极大值抑制和 anchor 的依赖。DETR 把它用于目标检测,DETR3D 把它用于三维检测,PlanT 与 UniAD 把它用于代理级的规划预测。这一移动在自动驾驶中提供了端到端可微的输出层范式,使后续模块可以无歧义地拼接到感知输出之后。", + "building_blocks": [], + "label": "Set Prediction with Hungarian Matching", + "degree": 4 + }, + { + "id": "move:diffusion_denoise_sampling", + "label_zh": "基于得分的去噪采样", + "kind": "move", + "tier": "move", + "topic": "math_foundations", + "phase": "core", + "year": 2020, + "summary_zh": "扩散方法把生成问题转化为反向去噪过程,模型只需学习预测每一步的噪声或得分函数,并在推断时迭代采样得到样本。这一移动在图像生成中由 DDPM 奠基,在视频中扩展为视频扩散,在控制领域被 Diffusion Policy 重新解释为以条件为状态、以动作为样本的策略学习。在自动驾驶中,扩散过程可以同时生成多模态轨迹候选和反事实场景,统一了生成与决策两个传统上分离的问题。", + "building_blocks": [], + "label": "Score-based Denoising Sampling", + "degree": 3 + }, + { + "id": "move:dual_system_fast_slow", + "label_zh": "快慢双系统分解", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "快慢双系统是一种受 Kahneman 双过程理论启发的方法学分解,让一个低延迟的小模型负责常规情况下的实时响应,让一个高延迟的大模型在异常或复杂情境下提供慢速推理。AlphaGo 用快速 policy 加 MCTS 体现了它,OpenAI o1 用思考链体现了它,DriveVLM-Dual 用快规划加 VLM 慢系统体现了它。在自动驾驶中这一移动直接回应了延迟和质量之间的硬性权衡,是落地系统中最实用的范式之一。", + "building_blocks": [], + "label": "Dual System Fast-Slow Decomposition", + "degree": 3 + }, + { + "id": "move:tool_use_function_calling", + "label_zh": "工具调用与函数调用", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "core", + "year": 2023, + "summary_zh": "工具调用让语言模型把外部 API、地图查询、轨迹优化器、规则检查器视为可调用函数,从而把符号系统的精确性嫁接到神经语言模型的灵活推理之上。Toolformer 演示了从无监督语料学习工具触发,ReAct 引入了思考与行动交替的循环,Agent-Driver 则把它落到驾驶决策栈中。在自动驾驶里这一移动使 LLM 不需要内化所有数值能力,而可以通过调用专家组件解决精确控制和约束验证问题。", + "building_blocks": [], + "label": "Tool Use / Function Calling", + "degree": 1 + }, + { + "id": "move:counterfactual_replan", + "label_zh": "反事实重规划", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2025, + "summary_zh": "反事实重规划让系统主动构造若干并未发生的对照场景并比较其后果,从而把决策从被动响应升级为对潜在后果的主动评估。它在因果推断、强化学习中的 model-based rollout、对抗训练中都有体现,在 CF-VLA 中被显式实现为对替代轨迹的 VLM 评分。在自动驾驶中这一移动让系统能够回答如果我变道而非保持车道会发生什么这类问题,是迈向人类级因果驾驶推理的关键路径。", + "building_blocks": [], + "label": "Counterfactual Replan", + "degree": 3 + }, + { + "id": "move:tokenize_modalities", + "label_zh": "模态统一 tokenize", + "kind": "move", + "tier": "move", + "topic": "vlm_vla", + "phase": "core", + "year": 2022, + "summary_zh": "模态统一 tokenize 是一种深刻的简化:与其为每种模态设计独立的编码器,不如把图像、语音、动作、点云都映射到同一个离散或连续 token 空间,然后让一个统一的序列模型处理它们。Gato 在多模态控制中实现了它,RT-1、RT-2 在机器人动作中扩展了它,Wayve LINGO-2 在驾驶中把传感、语言、动作放进同一个序列。在自动驾驶中这一移动允许同一个基础模型在驾驶日志、人类解说、控制信号上联合训练。", + "building_blocks": [], + "label": "Universal Modality Tokenization", + "degree": 2 + }, + { + "id": "move:replay_and_target_net", + "label_zh": "经验回放与目标网络", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2015, + "summary_zh": "经验回放与目标网络是稳定值函数学习的两个方法学原语,前者通过从大池子均匀采样打破数据的时序相关性,后者通过周期同步的复制网络打破贝尔曼自举的反馈循环。DQN 同时引入它们使深度强化学习首次稳定收敛,DDPG、TD3、Rainbow 等继承并扩展了这一移动。在自动驾驶中这一移动是任何基于值函数的离线强化学习方法的基础前置条件。", + "building_blocks": [], + "label": "Replay Buffer with Target Network", + "degree": 1 + }, + { + "id": "move:dataset_aggregation", + "label_zh": "迭代数据集聚合", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2011, + "summary_zh": "迭代数据集聚合让学习到的策略与环境交互、由专家在新访问到的状态上重新标注、再把这些状态与标签加入训练集进行下一轮训练。这一移动直接对应于模仿学习中协变量偏移的成因并提供了理论上有界的解决方案,是 DAgger 的核心。在自动驾驶中这一移动启发了影子模式数据收集、规划失败案例自动挖掘等数据飞轮设计。", + "building_blocks": [], + "label": "Iterative Dataset Aggregation", + "degree": 1 + }, + { + "id": "move:clipped_surrogate_objective", + "label_zh": "剪裁替代目标", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "剪裁替代目标用一个对策略比率施加上下界限制的目标函数来近似信赖域约束,从而在保留高效一阶优化的同时避免单步策略更新过大。这是 PPO 的方法学心脏,被广泛复用于 RLHF、DPO 的早期变体和很多机器人在线策略优化。在自动驾驶相关的训练后期,这一移动是把模仿学习初始化的策略安全地推向更高奖励的标准工具。", + "building_blocks": [], + "label": "Clipped Surrogate Objective", + "degree": 2 + }, + { + "id": "move:self_play_with_search", + "label_zh": "自对弈与蒙特卡洛树搜索", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "自对弈让一个策略对抗自身的副本以生成无穷数据,蒙特卡洛树搜索利用学习到的价值与策略进行未来推演,二者结合形成正反馈式的能力提升。AlphaZero 在棋类中实现了它,MuZero 把它推广到无模型环境,最近的 Multi-Agent Driving Self-Play 把它探索性地引入交通博弈。在自动驾驶中这一移动是大规模仿真器中多智能体策略学习的核心范式。", + "building_blocks": [], + "label": "Self-Play with Tree Search", + "degree": 2 + }, + { + "id": "move:latent_imagination_rollout", + "label_zh": "潜空间想象 rollout", + "kind": "move", + "tier": "move", + "topic": "deep_rl", + "phase": "frontier", + "year": 2018, + "summary_zh": "潜空间想象让策略和价值在一个学到的紧凑潜世界模型中进行多步推演,而非在像素或原始传感空间中代价高昂地展开。Ha 与 Schmidhuber 的 World Models 首次系统化它,Dreamer 系列把它做到大规模可训练,GAIA-1 与 DriveDreamer 把它扩展到驾驶视频。在自动驾驶中这一移动是稀缺真实事故数据条件下大规模 model-based 强化学习的关键。", + "building_blocks": [], + "label": "Latent Imagination Rollout", + "degree": 5 + }, + { + "id": "move:spike_event_compute", + "label_zh": "脉冲与事件驱动计算", + "kind": "move", + "tier": "move", + "topic": "brain_inspired", + "phase": "frontier", + "year": 1997, + "summary_zh": "脉冲与事件驱动计算让神经元只有在阈值被跨越时才发放离散事件,从而把信息表示为稀疏时序脉冲流,大幅降低能耗和延迟。脉冲神经网络在神经形态硬件、事件相机视觉、Spike-driven Transformer 中反复出现。在自动驾驶中这一移动直接对应车载端能耗、延迟与冗余度的硬性约束,是边缘端低功耗感知与决策的方向之一。", + "building_blocks": [], + "label": "Spike / Event-driven Compute", + "degree": 3 + }, + { + "id": "insight:residual_learning_unlocks_arbitrary_depth", + "label_zh": "残差学习解锁任意深度", + "kind": "insight", + "tier": "insight", + "topic": "math_foundations", + "phase": "prereq", + "year": 2015, + "summary_zh": "残差学习是一条跨领域的普适洞见,其抽象内核是把任意复杂的映射改写为恒等加上一个小的偏差并训练这个偏差。这一洞见在视觉中由 ResNet 首先打通了百层网络的训练,在 Transformer 中以残差子层与层归一化的组合再次出现,在扩散模型中以预测噪声残差的形式作为生成的方法学基石,在强化学习中以残差策略叠加在规则控制器之上而获得安全可控的微调。在自动驾驶研究中这意味着规则规划器并不需要被推翻,而是可以作为残差策略的恒等基线,由数据驱动模块只学习相对修正,从而把传统工程与神经端到端融合。", + "building_blocks": [ + "move:residual_connection", + "paper:he2015_resnet" + ], + "label": "Residual Learning Unlocks Arbitrary Depth", + "degree": 3 + }, + { + "id": "insight:masked_prediction_yields_self_supervised_signal", + "label_zh": "掩码预测提取自监督信号", + "kind": "insight", + "tier": "insight", + "topic": "ssl_vision", + "phase": "core", + "year": 2018, + "summary_zh": "掩码并预测是一条贯穿语言、视觉、视频、机器人等模态的洞见,其内核是任何含有冗余结构的数据都可以通过遮蔽一部分并让模型重构被遮蔽部分而提供任务无关的监督。BERT 把它带入语言并奠定了预训练后微调的范式,MAE 把它带入图像并显著降低了视觉预训练的标签成本,VideoMAE 与 OmniMAE 把它扩展到时空,占据栅格掩码预训练则把它引入自动驾驶。对于自动驾驶研究的含义是,可以将下一秒 BEV 占据或邻车轨迹作为掩码目标进行大规模无标签预训练,从而把街道上海量的未标注行驶数据转化为通用驾驶表征。", + "building_blocks": [ + "move:masking_for_pretext", + "concept:ssl" + ], + "label": "Masked Prediction Yields Self-Supervised Signal", + "degree": 4 + }, + { + "id": "insight:attention_is_typed_entity_communication", + "label_zh": "注意力是类型化实体之间的通信", + "kind": "insight", + "tier": "insight", + "topic": "math_foundations", + "phase": "core", + "year": 2017, + "summary_zh": "注意力机制可以被理解为一群类型化的实体在共享潜空间中互相发送和接收消息的协议,每个 query 是一个有指定意图的探针,每个 key-value 对是一个可被检索的事实。DETR 把检测对象编码为 query,BEVFormer 把鸟瞰网格点编码为 query,UniAD 把代理、地图元素与运动模式全部编码为 query 并让它们在同一注意力栈中对话。在自动驾驶研究的含义是,任何被司机关心的实体(路口、信号灯、邻车意图、占据网格、自车未来)都可以被声明为 query 并接入一个统一的注意力工厂,从而把感知、预测、规划写成同一种结构。", + "building_blocks": [ + "move:cross_attention_query", + "concept:self_attention", + "concept:detr_query" + ], + "label": "Attention is Communication Between Typed Entities", + "degree": 7 + }, + { + "id": "insight:contrastive_alignment_creates_zero_shot_transfer", + "label_zh": "对比对齐造就零样本迁移", + "kind": "insight", + "tier": "insight", + "topic": "ssl_vision", + "phase": "core", + "year": 2021, + "summary_zh": "对比对齐告诉我们任何两种自然成对出现的模态都可以通过对齐它们的嵌入空间而获得相互检索和零样本分类能力。CLIP 在图文上展示了这一点,ALIGN 用更脏的网络数据进一步扩大规模,CLAP 与 AudioCLIP 把它推广到音频文本,DriveCLIP 与场景文本对齐工作把它带入驾驶日志检索。对于自动驾驶研究的含义是,可以把驾驶视频与人类驾驶员的解说、事故报告、城市规则文本进行对比对齐,从而获得零样本的场景检索能力和自然语言条件下的轨迹生成基础。", + "building_blocks": [ + "move:contrastive_alignment", + "concept:vlm" + ], + "label": "Contrastive Alignment Creates Zero-shot Transfer", + "degree": 3 + }, + { + "id": "insight:diffusion_unifies_generation_and_decision", + "label_zh": "扩散统一生成与决策", + "kind": "insight", + "tier": "insight", + "topic": "math_foundations", + "phase": "core", + "year": 2022, + "summary_zh": "扩散与得分匹配揭示了一条把生成、推断、控制统一起来的洞见,其内核是任何复杂分布的样本都可以通过反向去噪过程逐步生成,而条件信息可以作为得分网络的额外输入控制采样。它在图像生成中由 DDPM 与 Stable Diffusion 推广,在视频中通过 Video Diffusion 实现,在控制中由 Diffusion Policy、Decision Diffuser 把动作序列视为待去噪样本,在驾驶中由轨迹扩散与场景扩散世界模型同时使用。对于自动驾驶研究的含义是,多模态轨迹候选与反事实场景可以从同一个扩散过程中以不同条件采样得到,从而消除生成模块和规划模块之间的人为边界。", + "building_blocks": [ + "move:diffusion_denoise_sampling", + "paper:diffuser" + ], + "label": "Diffusion Unifies Generation and Decision", + "degree": 3 + }, + { + "id": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", + "label_zh": "信号足够强时端到端可微胜过手工中间表示", + "kind": "insight", + "tier": "insight", + "topic": "e2e_ad", + "phase": "core", + "year": 2017, + "summary_zh": "端到端可微优化在监督信号足够强、数据足够多时往往优于由人类手工设计的中间接口,因为后者人为限制了表征空间。Listen-Attend-Spell 在语音识别中消化了声学加发音加语言模型的级联,seq2seq 在机器翻译中消化了短语对齐与句法分析的级联,UniAD 在驾驶中消化了感知到预测到规划的级联。对于自动驾驶研究的含义是,当具备完整的端到端反馈(仿真器奖励、真实日志的人类轨迹、安全度量)时,应当主动溶解模块边界、让梯度自由流动;同时需要清醒地承认在反馈稀疏或安全敏感的子任务上模块化仍是稳健的折衷。", + "building_blocks": [ + "paper:2212.10156", + "essay:bitter_lesson" + ], + "label": "End-to-End Differentiable Beats Handcraft When Signal is Strong", + "degree": 4 + }, + { + "id": "insight:dual_system_handles_latency_quality_tradeoff", + "label_zh": "双系统化解延迟与质量权衡", + "kind": "insight", + "tier": "insight", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "Kahneman 的快慢系统给出了一条跨学科洞见,复杂决策系统应当用一条低延迟廉价的反应通路覆盖大多数常规情境,并用一条高延迟昂贵的反思通路在异常或风险情境下提供慢速推理。AlphaGo 用快速 policy 加 MCTS 实现了它,OpenAI o1 用 chain-of-thought 思考时间换质量,DriveVLM-Dual 用快规划加 VLM 慢系统融合实现了它。对于自动驾驶研究的含义是,应当在生产堆栈中显式分离 30 Hz 控制循环和 1 Hz 反思循环,并把后者用于罕见且高代价的决策点,而非试图用单一大模型既快又好。", + "building_blocks": [ + "move:dual_system_fast_slow", + "paper:2402.12289" + ], + "label": "Dual System Handles the Latency-Quality Tradeoff", + "degree": 4 + }, + { + "id": "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "label_zh": "符号中间表示提供可解释性与可迁移性", + "kind": "insight", + "tier": "insight", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "在两个神经组件之间插入一个人类可读的符号中间表示往往同时提升可解释性和跨任务迁移能力,因为符号既约束了表达空间又对人类审计开放。Code-as-Policies 用 Python 代码作为机器人计划的中间表示,LMDrive 与 DriveLM 用自然语言作为感知与规划之间的接口,PROGPROMPT 用结构化提示驱动家庭机器人。对于自动驾驶研究的含义是,把语言或结构化场景描述显式插入感知到规划之间,可以在解释失败案例、跨车型迁移、安全审计上带来巨大杠杆,而代价仅仅是中间一次额外的模型推理。", + "building_blocks": [ + "concept:cot", + "paper:2309.16292" + ], + "label": "Symbolic Intermediate Enables Interpretability and Transfer", + "degree": 6 + }, + { + "id": "insight:long_tail_solved_by_synthesis_not_data_alone", + "label_zh": "长尾问题靠合成而非单纯增加数据解决", + "kind": "insight", + "tier": "insight", + "topic": "e2e_ad", + "phase": "frontier", + "year": 2023, + "summary_zh": "长尾问题的解决依赖主动合成而非被动收集,因为现实世界中真正稀有的情境出现频率太低使得纯数据扩张的边际收益递减。机器人控制中通过域随机化合成了千万级仿真轨迹,文本到 3D 中通过扩散世界模型合成了不存在的物体,自动驾驶中通过场景生成器和反事实重写合成了边角案例。对于自动驾驶研究的含义是,每多收集一千小时真实数据可能不如训练一个高质量的反事实场景生成器更划算,反事实重规划与合成驱动的安全验证应当成为核心工具链而非辅助工具。", + "building_blocks": [ + "move:counterfactual_replan", + "paper:2512.24426", + "paper:drivedreamer" + ], + "label": "Long Tail Solved by Synthesis Rather than Data Alone", + "degree": 7 + }, + { + "id": "insight:scaling_laws_predict_capability_emergence", + "label_zh": "Scaling 定律预测能力涌现", + "kind": "insight", + "tier": "insight", + "topic": "meta_philosophy", + "phase": "core", + "year": 2020, + "summary_zh": "Kaplan 等人的 scaling 定律显示模型损失对参数量、数据量、计算量呈幂律下降,并允许研究者在做小实验后预测大规模训练的能力。它在语言模型中由 GPT-3 验证、在视觉中由 DINOv2 验证、在多模态中由 Flamingo 与 Gemini 验证。对于自动驾驶研究的含义是,应当在每一项关键架构决策之前先做小规模 scaling 扫描以验证其外推性,而非在小模型上得到的局部最优结论上做大规模工程投入。", + "building_blocks": [ + "essay:bitter_lesson", + "concept:scaling_vs_knowledge", + "paper:gpt3" + ], + "label": "Scaling Laws Predict Capability Emergence", + "degree": 7 + }, + { + "id": "insight:foundation_pretraining_decouples_data_from_task", + "label_zh": "基础模型预训练把数据与任务解耦", + "kind": "insight", + "tier": "insight", + "topic": "ssl_vision", + "phase": "core", + "year": 2021, + "summary_zh": "基础模型范式揭示了把任务无关的大规模预训练与任务相关的小规模微调彻底分离的可行性,由此使下游任务的边际数据需求大幅下降。BERT 与 GPT 在语言上演示了它,CLIP 与 DINOv2、DINOv3 在视觉上演示了它,SAM 在分割上演示了它。对于自动驾驶研究的含义是,应当首先投入资源训练驾驶通用主干(视觉、占据、轨迹)而非每个任务从零训练,并通过适配器和提示工程在下游低成本部署。", + "building_blocks": [ + "paper:dinov2", + "paper:2508.10104", + "concept:ssl" + ], + "label": "Foundation Pretraining Decouples Data from Task", + "degree": 6 + }, + { + "id": "insight:test_time_compute_substitutes_train_time_via_search", + "label_zh": "测试时计算可经搜索替代训练时计算", + "kind": "insight", + "tier": "insight", + "topic": "deep_rl", + "phase": "frontier", + "year": 2017, + "summary_zh": "测试时计算与训练时计算之间存在可替换关系,通过在推断时引入搜索、采样、自洽性投票等开销可以补偿训练阶段的能力不足。AlphaGo 用 MCTS 在弱策略上做搜索拿到超人类表现,OpenAI o1 用更长思维链交换更高准确率,AlphaCode 用大规模采样加过滤换取代码竞赛能力。对于自动驾驶研究的含义是,在生产端可以将昂贵车队训练与可控的车载推断搜索(轨迹采样加约束检查、VLM 反思)结合起来,从而以可承担的车载预算获得更高的决策质量。", + "building_blocks": [ + "move:self_play_with_search", + "paper:silver2017_alphazero" + ], + "label": "Test-Time Compute Can Substitute Train-Time via Search", + "degree": 3 + }, + { + "id": "insight:imitation_data_compresses_unspecified_reward", + "label_zh": "模仿数据压缩了未明示的奖励函数", + "kind": "insight", + "tier": "insight", + "topic": "deep_rl", + "phase": "core", + "year": 2011, + "summary_zh": "模仿学习的洞见是专家演示隐式编码了一个研究者难以手工指定的奖励函数,从而避开了奖励设计的难题。它在 ALVINN 中首次用于驾驶,在 GAIL、AIRL 中通过对抗逆强化学习显式提取了奖励,在 DriveGPT 系列中被推到大规模驾驶日志预训练。对于自动驾驶研究的含义是,与其试图手工写出涵盖舒适、安全、效率、社会礼仪的复合奖励,不如先用大规模驾驶日志做模仿预训练,再用偏好对齐与少量精细奖励做微调。", + "building_blocks": [ + "concept:imitation_learning", + "paper:ross2011_dagger" + ], + "label": "Imitation Data Compresses an Unspecified Reward", + "degree": 5 + }, + { + "id": "insight:world_models_let_planning_be_done_in_imagination", + "label_zh": "世界模型让规划在想象中进行", + "kind": "insight", + "tier": "insight", + "topic": "deep_rl", + "phase": "core", + "year": 2018, + "summary_zh": "世界模型把环境动力学压缩进一个学到的潜空间模型,使策略和价值可以在该潜空间里以极低代价进行多步推演而非每次都与昂贵或危险的真实环境交互。Ha 与 Schmidhuber 第一次系统化了它,Dreamer 系列把它扩展到 Atari 与机器人,GAIA-1 与 DriveDreamer 把它扩展到驾驶视频。对于自动驾驶研究的含义是,未来的安全验证、罕见事故训练、反事实问答都可以建立在驾驶世界模型之上,把今天昂贵的实车测试逐步替换为模型内想象。", + "building_blocks": [ + "move:latent_imagination_rollout", + "paper:world_models", + "paper:gaia1" + ], + "label": "World Models Let Planning Be Done in Imagination", + "degree": 7 + }, + { + "id": "insight:tokenization_collapses_modality_gap", + "label_zh": "Tokenize 抹平模态差距", + "kind": "insight", + "tier": "insight", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2022, + "summary_zh": "把每种模态都映射为一个共享的 token 序列后,处理它们的主干网络几乎可以是同一个,模态差异退化为 tokenizer 的设计差异。ViT 把图像 tokenize 为 patch,VQ-VAE 把语音和视频 tokenize 为离散码本,Gato 与 RT-2 把动作 tokenize 为符号,使同一个 transformer 既能聊天又能操作。对于自动驾驶研究的含义是,可以把视觉、点云、HD 地图、控制信号统一 tokenize 进同一序列,从而让基础模型在驾驶日志、人类讲解、控制信号上联合训练,向真正的多模态驾驶基础模型迈进。", + "building_blocks": [ + "move:tokenize_modalities", + "move:patchify_tokenization" + ], + "label": "Tokenization Collapses the Modality Gap", + "degree": 4 + }, + { + "id": "insight:set_prediction_eliminates_postprocessing_heuristics", + "label_zh": "集合预测消除后处理启发式", + "kind": "insight", + "tier": "insight", + "topic": "ssl_vision", + "phase": "core", + "year": 2020, + "summary_zh": "把多对象输出建模为集合并用匈牙利匹配计算损失,可以一次性消除诸如非极大值抑制、锚框设计、阈值调节等大量手工后处理。这一洞见在 DETR 中第一次系统化,在 DETR3D、Sparse R-CNN 中推广到三维与稀疏检测,在 PlanT 与 UniAD 中扩展为对智能体级别规划的集合预测。对于自动驾驶研究的含义是,每当下游任务可以被表述为一个无序的输出集合时,集合预测往往能用更少的代码、更直接的梯度信号取代级联的启发式后处理,使整个 stack 真正端到端可微。", + "building_blocks": [ + "move:set_prediction_with_hungarian", + "paper:carion2020" + ], + "label": "Set Prediction Eliminates Post-processing Heuristics", + "degree": 4 + }, + { + "id": "insight:in_context_learning_emerges_at_scale", + "label_zh": "上下文学习在规模上涌现", + "kind": "insight", + "tier": "insight", + "topic": "vlm_vla", + "phase": "core", + "year": 2020, + "summary_zh": "大规模自回归语言模型展示了在不更新参数的前提下从提示中的少量例子中学习新任务的能力,这是参数规模与数据规模累积之后的涌现现象。GPT-3 第一次把它放到聚光灯下,Flamingo 把它扩展到图像文本交错,VLM 工具如 Gemini 与 Claude 已经在驾驶问答中演示了少样本场景理解。对于自动驾驶研究的含义是,许多边角案例可以通过精心设计的提示中包含少量类似情境的示例与解决方案而被解决,而不必为每一种长尾微调一个新模型。", + "building_blocks": [ + "paper:gpt3", + "concept:cot" + ], + "label": "In-Context Learning Emerges at Scale", + "degree": 6 + }, + { + "id": "insight:safety_constraints_via_lagrangian_dual", + "label_zh": "安全约束借助拉格朗日对偶实现", + "kind": "insight", + "tier": "insight", + "topic": "deep_rl", + "phase": "core", + "year": 2019, + "summary_zh": "拉格朗日对偶为把硬安全约束嵌入到策略优化提供了一条原则化道路,做法是把约束乘以可学习的乘子并将其加入目标函数,乘子由约束违反程度驱动上升。Constrained Policy Optimization 把它和信赖域结合,RCPO 把它和 PPO 结合,自动驾驶的 Safe-DRL 工作把它用于限制碰撞概率和加速度。对于自动驾驶研究的含义是,可以用拉格朗日方法把碰撞率、加速度、横向加速度等硬指标作为约束接入端到端策略优化,从而让性能与安全在统一框架下被对齐。", + "building_blocks": [ + "concept:policy_gradient", + "concept:actor_critic" + ], + "label": "Safety Constraints via Lagrangian Duality", + "degree": 3 + }, + { + "id": "insight:event_sparse_compute_matches_energy_budget", + "label_zh": "事件稀疏计算匹配边缘能耗预算", + "kind": "insight", + "tier": "insight", + "topic": "brain_inspired", + "phase": "frontier", + "year": 2023, + "summary_zh": "事件驱动稀疏计算的核心洞见是大多数自然信号在时间或空间上都是高度稀疏的,因此只在事件发生时进行计算可以节省数量级的能耗。事件相机在视觉中演示了它,脉冲神经网络在感知中演示了它,Spike-driven Transformer 把它与注意力结构融合。对于自动驾驶研究的含义是,车端有严格能耗与延迟预算,事件稀疏感知与决策可以作为补充昂贵密集 GPU 推断的轻量备份,特别适用于高频低复杂度的旁路任务。", + "building_blocks": [ + "move:spike_event_compute", + "concept:spiking_nn", + "paper:2307.01694" + ], + "label": "Event-Sparse Compute Matches Edge Energy Budget", + "degree": 5 + }, + { + "id": "validation:trace_unified_planning_oriented_e2e_driving", + "label_zh": "再发现:UniAD(统一规划导向端到端驾驶)", + "kind": "validation", + "tier": "validation", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "summary_zh": "如果 UniAD 不存在,要从零再发明它,研究者必须在图谱中具备以下完整的构件链。第一是 BEVFormer 提供的时空鸟瞰图特征作为统一坐标系,第二是 DETR 风格的对象 query 与集合预测作为可微输出接口,第三是把每一种子任务(检测、跟踪、地图、运动预测、占据预测、规划)都改写为 query 在共享 BEV 上 cross-attention 的统一语言。第四是端到端可微即在信号足够强时溶解模块边界的洞见,使最终规划损失能够把梯度回传到感知。第五是模仿学习作为大规模驾驶日志监督的载体。把这五条放在一起,研究者就能自然推导出 UniAD 这种以规划为目标、各感知模块通过 query 协作的统一架构。", + "building_blocks": [ + "paper:li2022bevformer", + "paper:carion2020", + "concept:bev", + "concept:detr_query", + "move:cross_attention_query", + "move:set_prediction_with_hungarian", + "concept:imitation_learning", + "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", + "insight:attention_is_typed_entity_communication" + ], + "label": "Trace: Unified Planning-Oriented E2E Driving", + "degree": 10 + }, + { + "id": "validation:trace_object_level_planning_transformer", + "label_zh": "再发现:PlanT(对象级规划 transformer)", + "kind": "validation", + "tier": "validation", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "summary_zh": "如果 PlanT 不存在,要重新发明它,研究者需要的构件是:第一,已经具备成熟的检测器输出对象级别的紧凑场景表示(位置、速度、类别);第二,Transformer 作为一种把变长无序对象集合编码并产生序列化输出的通用工具;第三,集合预测与匈牙利匹配作为损失结构以处理对象的无序性;第四,模仿学习把人类驾驶轨迹作为目标输出;第五,对端到端模仿在 CARLA 等闭环仿真中可验证的认识。把这些组合起来便可自然得到一个用 transformer 直接消费对象 token 并输出未来路点的对象级规划器,并理解为何它比像素级端到端更样本高效与可解释。", + "building_blocks": [ + "paper:vaswani2017", + "paper:carion2020", + "concept:transformer", + "move:cross_attention_query", + "move:set_prediction_with_hungarian", + "concept:imitation_learning", + "paper:ad_benchmarks", + "paper:transfuser" + ], + "label": "Trace: Object-Level Planning Transformer", + "degree": 9 + }, + { + "id": "validation:trace_vision_language_action_dual_loop", + "label_zh": "再发现:DriveVLM-Dual(VLA 双系统)", + "kind": "validation", + "tier": "validation", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "如果 DriveVLM 与其双系统变体不存在,要把它再发明出来,研究者需要具备的构件是:一个能将视觉与语言对齐并产生符号级输出的 VLM 主干(如 LLaVA);BEV 与代理 query 作为快规划器的几何输入;chain-of-thought 推理作为慢系统的核心能力;快慢双系统范式作为对延迟与质量权衡的方法学回答;以及一个把语言场景描述与轨迹候选互相校验的接口。把它们组合起来,研究者会自然提出一个 30 Hz 的快规划器加 1 Hz 的 VLM 反思回路在关键时刻提供高阶决策的双重架构。", + "building_blocks": [ + "paper:llava", + "concept:vlm", + "concept:vla", + "concept:cot", + "move:dual_system_fast_slow", + "move:cross_attention_query", + "insight:dual_system_handles_latency_quality_tradeoff", + "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "paper:li2022bevformer" + ], + "label": "Trace: Vision-Language-Action Dual Loop", + "degree": 10 + }, + { + "id": "validation:trace_llm_decision_agent_for_driving", + "label_zh": "再发现:Agent-Driver(LLM 决策智能体)", + "kind": "validation", + "tier": "validation", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "如果 Agent-Driver 不存在,要从零再发明它,研究者需要的构件是:第一,GPT-3 量级的语言模型展示的少样本上下文学习能力;第二,工具调用范式让 LLM 能调用外部的检测器、地图查询、轨迹优化器以补偿其几何与控制的不足;第三,ReAct 形式的交替思考与行动循环;第四,对象级感知作为对环境的紧凑描述以馈入语言提示;第五,模仿学习专家轨迹作为对比基线以评估 LLM 的决策质量。组合起来即可推出一个把 LLM 作为高阶决策核心、把所有数值技能委托给外部工具的驾驶认知智能体。", + "building_blocks": [ + "paper:gpt3", + "concept:vlm", + "concept:cot", + "concept:tool_use", + "move:tool_use_function_calling", + "paper:2210.14222", + "insight:in_context_learning_emerges_at_scale", + "insight:symbolic_intermediate_enables_interpretability_and_transfer" + ], + "label": "Trace: LLM Decision Agent for Driving", + "degree": 9 + }, + { + "id": "validation:trace_knowledge_driven_reflective_agent", + "label_zh": "再发现:DiLu(知识驱动反思智能体)", + "kind": "validation", + "tier": "validation", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "如果 DiLu 不存在,要重新发明它,研究者需要的构件是:GPT-3 级别的语言推理能力作为通用决策器;chain-of-thought 让模型显式写出选择车道、避让的理由;类似 Reflexion 的外部记忆与反思机制把过去错误的决策回溯并修正;驾驶仿真器作为决策闭环;以及对照 Sutton 苦涩教训的批判性立场,承认在数据稀缺的小型试点中知识与符号反思可以暂时弥补 scaling 的不足。把这些组合起来便能得到一个写自然语言决策日志、维护可检索经验池、用反思修正错误的知识驱动驾驶 agent。", + "building_blocks": [ + "paper:gpt3", + "concept:vlm", + "concept:cot", + "concept:tool_use", + "essay:bitter_lesson", + "paper:ad_benchmarks", + "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "insight:in_context_learning_emerges_at_scale" + ], + "label": "Trace: Knowledge-Driven Reflective Agent", + "degree": 9 + }, + { + "id": "validation:trace_brain_inspired_spike_attention", + "label_zh": "再发现:Spike-driven Transformer", + "kind": "validation", + "tier": "validation", + "topic": "brain_inspired", + "phase": "frontier", + "year": 2023, + "summary_zh": "如果 Spike-driven Transformer 不存在,要把它再发明出来,研究者需要的构件是:脉冲神经网络的发放-阈值-膜电位计算模型;transformer 中的自注意力作为通用 mixer;ResNet 启发的残差结构保证深层可训练性;把 query-key 点积重写为脉冲触发的稀疏外积的代数技巧;事件稀疏计算可匹配边缘能耗预算的洞见以及对苦涩教训的逆向思考即接受在能耗约束硬性主导的边缘场景里仍需引入硬件友好的归纳偏置。组合起来即可得到一个用脉冲事件代替密集激活、保留注意力表达力又显著节能的模型。", + "building_blocks": [ + "concept:spiking_nn", + "paper:vaswani2017", + "paper:vit", + "paper:he2015_resnet", + "concept:self_attention", + "move:residual_connection", + "move:spike_event_compute", + "insight:event_sparse_compute_matches_energy_budget", + "essay:bitter_lesson" + ], + "label": "Trace: Brain-inspired Spike Attention", + "degree": 10 + }, + { + "id": "validation:trace_scalable_self_supervised_vision_backbone", + "label_zh": "再发现:DINOv3(可规模化自监督视觉主干)", + "kind": "validation", + "tier": "validation", + "topic": "ssl_vision", + "phase": "frontier", + "year": 2025, + "summary_zh": "如果 DINOv3 不存在,要把它再发明出来,研究者需要的构件是:DINOv2 提供的自蒸馏多裁剪自监督训练配方;ViT 提供的可扩展架构;掩码图像建模作为辅助监督;scaling 定律告诉我们更大的模型与数据继续提升下游能力;基础模型预训练把数据与任务解耦的洞见使得驾驶等下游可以零样本受益;以及对苦涩教训的承诺即长期看大规模无监督预训练胜过手工特征。把这些组合起来便可推出一个把数据规模、模型规模、训练步数同时再推一档以提供下一代驾驶视觉表征的工作。", + "building_blocks": [ + "paper:dinov2", + "paper:vit", + "concept:ssl", + "move:masking_for_pretext", + "essay:bitter_lesson", + "insight:foundation_pretraining_decouples_data_from_task", + "insight:scaling_laws_predict_capability_emergence", + "insight:masked_prediction_yields_self_supervised_signal" + ], + "label": "Trace: Scalable Self-Supervised Vision Backbone", + "degree": 9 + }, + { + "id": "validation:trace_counterfactual_vla_replanner", + "label_zh": "再发现:CF-VLA(反事实 VLA 重规划器)", + "kind": "validation", + "tier": "validation", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2025, + "summary_zh": "如果 CF-VLA 不存在,要把它再发明出来,研究者需要的构件是:DriveVLM 等已经稳定的 VLA 主干提供基础的多模态感知与符号决策;meta-action 作为高层语义动作的离散接口以便枚举反事实;驾驶世界模型用以快速模拟反事实轨迹的结果;RLHF 与偏好对齐作为对反事实之间偏好打分的训练手段;反事实重规划作为方法学移动;以及长尾要靠合成而非更多真实数据解决的洞见。组合起来即可推出一个在每个决策点枚举若干 meta-action 替代方案、用世界模型推演并用偏好对齐挑选最优方案的反事实 VLA 重规划框架。", + "building_blocks": [ + "paper:2402.12289", + "paper:llava", + "paper:world_models", + "paper:gaia1", + "paper:drivedreamer", + "paper:rlhf_dpo", + "concept:vla", + "concept:counterfactual", + "concept:meta_action", + "move:counterfactual_replan", + "move:latent_imagination_rollout", + "insight:long_tail_solved_by_synthesis_not_data_alone", + "insight:world_models_let_planning_be_done_in_imagination" + ], + "label": "Trace: Counterfactual VLA Replanner", + "degree": 14 + }, + { + "id": "validation:trace_set_prediction_with_object_queries", + "label_zh": "再发现:DETR(基于对象 query 的集合预测)", + "kind": "validation", + "tier": "validation", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2020, + "summary_zh": "如果 DETR 不存在,要再发明它,研究者需要的构件是:Transformer 的编码器解码器作为对图像 patch 与对象之间任意远依赖的建模工具;ViT 启发的把图像切成 patch 的 tokenize 习惯(或 CNN 主干输出特征图);匈牙利算法在二分图上做最优分配的经典工具;模仿与监督学习中对集合输出的损失设计经验;以及把每个待检测对象抽象为一个 query 与图像 cross-attention 的关键洞见。组合起来即可得到一个用固定数量的 query、通过 cross-attention 从图像中拉取对象信息并经匈牙利匹配计算损失的检测器,从而彻底去除锚框和非极大值抑制。", + "building_blocks": [ + "paper:vaswani2017", + "concept:transformer", + "concept:self_attention", + "move:cross_attention_query", + "move:set_prediction_with_hungarian", + "insight:attention_is_typed_entity_communication", + "insight:set_prediction_eliminates_postprocessing_heuristics" + ], + "label": "Trace: Set Prediction with Object Queries", + "degree": 8 + }, + { + "id": "validation:trace_self_attention_replaces_recurrence", + "label_zh": "再发现:Transformer(自注意力取代循环)", + "kind": "validation", + "tier": "validation", + "topic": "math_foundations", + "phase": "prereq", + "year": 2017, + "summary_zh": "如果 Transformer 不存在,要把它再发明出来,研究者需要的构件是:seq2seq 翻译框架已经引入的注意力作为序列对齐机制;ResNet 已经普及的残差连接与层归一化作为深网络训练的稳定子;多头机制对应了把若干并行子空间的检索合并起来的直觉;位置编码作为对序列顺序的可加表示;以及把所有时间步的计算并行化以摆脱循环网络瓶颈的工程动机。组合起来便可推出一个完全靠 self-attention 与残差子层堆叠、能完全并行训练、对长距离依赖一阶建模的纯注意力网络。", + "building_blocks": [ + "paper:he2015_resnet", + "concept:self_attention", + "concept:transformer", + "move:residual_connection", + "move:cross_attention_query", + "insight:residual_learning_unlocks_arbitrary_depth", + "insight:attention_is_typed_entity_communication" + ], + "label": "Trace: Self-Attention Replaces Recurrence", + "degree": 8 + }, + { + "id": "validation:trace_image_transformer_via_patch_tokenization", + "label_zh": "再发现:ViT(patch tokenize 的图像 transformer)", + "kind": "validation", + "tier": "validation", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2020, + "summary_zh": "如果 ViT 不存在,要再发明它,研究者需要的构件是:Transformer 已经在文本上证明其在大规模数据上的优势;ResNet 时代积累的图像分类基准与数据集;把图像切成 16×16 patch 并展平为序列作为 tokenize 移动;位置编码引入图像中的空间结构;以及对 scaling 定律的信念使得研究者愿意接受 ViT 在小数据上不如 CNN 但在大数据上会反超的实验直觉。组合起来即可得到一个把图像视作 patch 序列、用纯 transformer 主干完成分类的架构,由此打开视觉基础模型的大门。", + "building_blocks": [ + "paper:vaswani2017", + "concept:transformer", + "concept:self_attention", + "move:patchify_tokenization", + "insight:tokenization_collapses_modality_gap", + "insight:scaling_laws_predict_capability_emergence" + ], + "label": "Trace: Image Transformer via Patch Tokenization", + "degree": 7 + }, + { + "id": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "label_zh": "再发现:BEVFormer(时空 BEV transformer)", + "kind": "validation", + "tier": "validation", + "topic": "e2e_ad", + "phase": "prereq", + "year": 2022, + "summary_zh": "如果 BEVFormer 不存在,要再发明它,研究者需要的构件是:DETR 提供的 query-based 检测范式;ViT 提供的图像 patch 编码主干;Lift-Splat-Shoot 思想的 2D 升 3D 投影;多相机标定与时序对齐的工程基础;以及把每个 BEV 网格点作为一个 query、对各相机特征做 deformable cross-attention、并把上一帧的 BEV 作为时间记忆做时序自注意力这一关键设计。组合起来即可推出一个直接在鸟瞰图坐标系下统一多相机感知、时序传递场景信息的 transformer 主干,为下游统一规划提供共享底座。", + "building_blocks": [ + "paper:vaswani2017", + "paper:vit", + "paper:carion2020", + "concept:transformer", + "concept:detr_query", + "concept:bev", + "move:cross_attention_query", + "move:lift_2d_to_3d", + "move:patchify_tokenization", + "insight:attention_is_typed_entity_communication" + ], + "label": "Trace: BEV Transformer with Temporal Aggregation", + "degree": 11 + }, + { + "id": "validation:trace_few_shot_in_context_learning_at_scale", + "label_zh": "再发现:GPT-3(大规模 few-shot 上下文学习)", + "kind": "validation", + "tier": "validation", + "topic": "vlm_vla", + "phase": "prereq", + "year": 2020, + "summary_zh": "如果 GPT-3 不存在,要再发明它,研究者需要的构件是:Transformer 解码器作为自回归语言建模主干;BERT 与 GPT-2 已经验证的预训练加微调范式;CommonCrawl 量级的网络文本作为预训练数据;模型与数据的 scaling 定律告诉我们继续把规模再放大一两个数量级会带来质变;以及对苦涩教训的承诺即放弃精细任务工程、把所有任务用统一的语言模型预训练目标处理。组合起来即可得到一个一百多亿到千亿参数级、在不更新参数的前提下从提示中学习新任务的通用语言模型,为后来所有 LLM 应用奠基。", + "building_blocks": [ + "paper:vaswani2017", + "concept:transformer", + "essay:bitter_lesson", + "concept:scaling_vs_knowledge", + "insight:scaling_laws_predict_capability_emergence", + "insight:in_context_learning_emerges_at_scale" + ], + "label": "Trace: Few-shot In-Context Learning at Scale", + "degree": 7 + }, + { + "id": "validation:trace_clipped_policy_gradient_surrogate", + "label_zh": "再发现:PPO(剪裁策略梯度)", + "kind": "validation", + "tier": "validation", + "topic": "deep_rl", + "phase": "core", + "year": 2017, + "summary_zh": "如果 PPO 不存在,要再发明它,研究者需要的构件是:REINFORCE 形式的策略梯度作为基础;TRPO 提出的信赖域约束保证单步更新不过大;Actor-Critic 框架以共享值函数减小方差;广义优势估计 GAE 用以平衡偏差与方差;以及对工程友好性的强烈关注,希望避免 TRPO 的二阶共轭梯度计算。把这些组合起来便可推出一个用对策略比率施加上下界 clip 来近似信赖域、可以纯靠 SGD 训练、超参数少的策略梯度方法,并显著降低 RL 算法的工程门槛。", + "building_blocks": [ + "concept:policy_gradient", + "concept:actor_critic", + "concept:ppo", + "move:clipped_surrogate_objective", + "course:zhao_rl", + "paper:sutton_barto" + ], + "label": "Trace: Clipped Policy Gradient Surrogate", + "degree": 7 + }, + { + "id": "validation:trace_deep_q_network_with_replay_and_target", + "label_zh": "再发现:DQN(深度 Q 网络)", + "kind": "validation", + "tier": "validation", + "topic": "deep_rl", + "phase": "prereq", + "year": 2015, + "summary_zh": "如果 DQN 不存在,要再发明它,研究者需要的构件是:Q-learning 与 Bellman 最优方程作为值函数学习的目标;CNN 作为从像素到动作值的可学习函数逼近;经验回放打破数据时序相关性;目标网络打破贝尔曼自举反馈回路;以及 Atari 仿真器作为高频可重复的环境。组合起来即可推出一个用 CNN 估计 Q 值、用经验回放和目标网络稳定训练、能够从纯像素学习多种 Atari 游戏的端到端深度强化学习算法。", + "building_blocks": [ + "concept:mdp", + "concept:bellman_eq", + "concept:td_learning", + "concept:dqn", + "concept:replay_buffer", + "move:replay_and_target_net", + "course:zhao_rl", + "paper:sutton_barto" + ], + "label": "Trace: Deep Q Network with Replay and Target", + "degree": 9 + }, + { + "id": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", + "label_zh": "再发现:AlphaZero(自对弈与 MCTS 指导策略)", + "kind": "validation", + "tier": "validation", + "topic": "deep_rl", + "phase": "prereq", + "year": 2017, + "summary_zh": "如果 AlphaZero 不存在,要再发明它,研究者需要的构件是:策略网络与价值网络作为一对联合训练的近似器;蒙特卡洛树搜索作为基于先验策略和价值估计的前向推演;自对弈作为生成无穷训练数据的源头;明确的、零和、完全信息的棋类规则作为闭环奖励;以及对苦涩教训的承诺即弃用人类棋谱与手工特征。组合起来即可推出一个仅靠自对弈数据、用 MCTS 改良策略、把 MCTS 改良后的访问分布作为策略训练目标的通用棋类超人系统。", + "building_blocks": [ + "concept:mdp", + "concept:policy_gradient", + "concept:actor_critic", + "essay:bitter_lesson", + "move:self_play_with_search", + "insight:test_time_compute_substitutes_train_time_via_search" + ], + "label": "Trace: AlphaZero Self-Play with MCTS-Guided Policy", + "degree": 7 + }, + { + "id": "validation:trace_dataset_aggregation_for_imitation", + "label_zh": "再发现:DAgger(模仿学习中的数据集聚合)", + "kind": "validation", + "tier": "validation", + "topic": "deep_rl", + "phase": "core", + "year": 2011, + "summary_zh": "如果 DAgger 不存在,要再发明它,研究者需要的构件是:行为克隆作为最朴素的监督模仿基线;对监督学习中分布偏移即协变量偏移的深入认识;专家策略可以被反复查询这一假设;以及在线学习中的 Follow-the-Leader 等理论思想给出聚合策略可以获得 no-regret 保证的灵感。组合起来即可推出一种迭代算法,让当前策略与环境交互、由专家在新状态上重新标注、把新数据加入训练集,从而解决纯 BC 在长视野任务上的崩塌问题。", + "building_blocks": [ + "concept:imitation_learning", + "concept:covariate_shift", + "move:dataset_aggregation", + "course:cs285", + "insight:imitation_data_compresses_unspecified_reward" + ], + "label": "Trace: Dataset Aggregation for Imitation", + "degree": 6 + }, + { + "id": "validation:trace_world_model_in_latent_imagination", + "label_zh": "再发现:World Models / Dreamer(潜空间想象世界模型)", + "kind": "validation", + "tier": "validation", + "topic": "deep_rl", + "phase": "core", + "year": 2018, + "summary_zh": "如果 World Models 与 Dreamer 类工作不存在,要再发明它们,研究者需要的构件是:VAE 等概率潜变量模型作为对感知的紧凑编码;循环或状态空间网络作为对动力学的预测;策略与价值函数能够在潜空间 rollout 中训练;强化学习的策略梯度作为最终优化算法;以及把规划在想象中进行可以大幅降低环境样本复杂度的洞见。组合起来即可推出把感知压缩为潜表示、用 RNN 建模潜动力学、在潜空间中并行多步 rollout 训练策略的世界模型方法。", + "building_blocks": [ + "concept:mdp", + "concept:policy_gradient", + "concept:actor_critic", + "move:latent_imagination_rollout", + "insight:world_models_let_planning_be_done_in_imagination", + "paper:world_models" + ], + "label": "Trace: World Model in Latent Imagination", + "degree": 7 + }, + { + "id": "validation:trace_vision_language_pretrained_dual_encoder", + "label_zh": "再发现:CLIP(视觉语言对比预训练双编码器)", + "kind": "validation", + "tier": "validation", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2021, + "summary_zh": "如果 CLIP 不存在,要再发明它,研究者需要的构件是:ViT 或 ResNet 作为可扩展图像编码器;Transformer 作为文本编码器;互联网规模的图文配对作为天然监督;InfoNCE 等对比学习损失把成对样本拉近、非成对样本推远;以及对比对齐可造就零样本迁移的洞见。组合起来即可推出一个由图像编码器、文本编码器和 InfoNCE 对齐目标构成的双塔模型,并显示其在零样本图像分类与跨模态检索上的全面突破。", + "building_blocks": [ + "paper:vit", + "paper:vaswani2017", + "concept:vlm", + "move:contrastive_alignment", + "concept:ssl", + "insight:contrastive_alignment_creates_zero_shot_transfer", + "insight:foundation_pretraining_decouples_data_from_task" + ], + "label": "Trace: Vision-Language Pretrained Dual Encoder", + "degree": 8 + }, + { + "id": "validation:trace_diffusion_policy_as_score_based_action_sampler", + "label_zh": "再发现:Diffusion Policy(基于得分的动作采样策略)", + "kind": "validation", + "tier": "validation", + "topic": "deep_rl", + "phase": "frontier", + "year": 2022, + "summary_zh": "如果 Diffusion Policy 不存在,要再发明它,研究者需要的构件是:DDPM 等图像扩散模型已经验证的反向去噪生成范式;模仿学习提供的状态动作配对数据;条件生成的标准化做法即把观测嵌入作为去噪网络的条件;多模态行为分布无法被单峰高斯捕捉这一经验观察;以及扩散统一生成与决策的洞见。组合起来即可推出一种把动作序列视为待去噪样本、把当前观测作为条件、在推断时迭代采样得到多模态动作分布的模仿策略。", + "building_blocks": [ + "concept:imitation_learning", + "move:diffusion_denoise_sampling", + "paper:diffuser", + "insight:diffusion_unifies_generation_and_decision", + "insight:imitation_data_compresses_unspecified_reward" + ], + "label": "Trace: Diffusion Policy as Score-Based Action Sampler", + "degree": 6 + }, + { + "id": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "label_zh": "再发现:BEV 融合模块化感知流水线(LSS / BEVFormer 家族)", + "kind": "validation", + "tier": "validation", + "topic": "e2e_ad", + "phase": "prereq", + "year": 2020, + "summary_zh": "如果 Lift-Splat-Shoot 与现代 BEV 融合流水线不存在,要再发明它们,研究者需要的构件是:多相机外参标定经验;每像素的深度分布预测的可学习方法;体素和栅格表示作为统一坐标系;ViT 或 CNN 作为图像编码主干;DETR 风格的 query 用以从 BEV 特征中拉取下游任务;以及 2D 升 3D 投影是 AD 中跨视角融合的天然语言这一洞见。组合起来即可推出一个把每相机图像特征沿深度概率展平为视锥体素、splat 到 BEV 栅格、再供下游检测和规划共享的统一感知流水线。", + "building_blocks": [ + "paper:vit", + "paper:carion2020", + "paper:li2022bevformer", + "concept:bev", + "concept:detr_query", + "move:lift_2d_to_3d", + "move:cross_attention_query", + "paper:tesla_ai_day" + ], + "label": "Trace: Modular Perception Pipeline with BEV Fusion", + "degree": 9 + }, + { + "id": "validation:trace_neural_field_for_dynamic_driving_scene", + "label_zh": "再发现:EmerNeRF / DrivingGaussian(动态驾驶场神经场)", + "kind": "validation", + "tier": "validation", + "topic": "e2e_ad", + "phase": "frontier", + "year": 2023, + "summary_zh": "如果 EmerNeRF 与 DrivingGaussian 类方法不存在,要把它们再发明出来,研究者需要的构件是:NeRF 与 3D Gaussian Splatting 作为对静态三维场景的可微表示;驾驶日志中多相机加 LiDAR 加自车位姿作为输入;时空分解把场分为静态背景与动态对象的设计思想;以及把神经场视为可被仿真器消费的可微世界模型这一立场。组合起来即可推出一个能从真实驾驶日志中重建可视、可编辑、可重放的动态驾驶场,为大规模反事实生成和闭环仿真提供基础。", + "building_blocks": [ + "paper:vit", + "paper:gaia1", + "paper:drivedreamer", + "paper:world_models", + "move:latent_imagination_rollout", + "insight:long_tail_solved_by_synthesis_not_data_alone", + "insight:world_models_let_planning_be_done_in_imagination" + ], + "label": "Trace: Neural Field for Dynamic Driving Scene", + "degree": 8 + }, + { + "id": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "label_zh": "再发现:Decision Transformer(序列建模式离线 RL)", + "kind": "validation", + "tier": "validation", + "topic": "deep_rl", + "phase": "frontier", + "year": 2021, + "summary_zh": "如果 Decision Transformer 不存在,要再发明它,研究者需要的构件是:GPT 风格的自回归 transformer 作为序列建模主干;离线强化学习的轨迹数据;把 return-to-go、状态、动作三元组拼接为 token 序列的关键 tokenize 移动;模仿学习作为监督信号;以及一个简单但有力的认识即只要把目标回报作为提示输入,自回归预测下一动作就等价于条件策略学习。组合起来即可推出一个完全用监督学习训练、却能在推断时通过指定高目标回报而得到高质量策略的离线强化学习方法。", + "building_blocks": [ + "paper:vaswani2017", + "paper:gpt3", + "concept:transformer", + "concept:imitation_learning", + "move:tokenize_modalities", + "insight:tokenization_collapses_modality_gap", + "insight:imitation_data_compresses_unspecified_reward" + ], + "label": "Trace: Decision Transformer (Offline RL via Sequence Modeling)", + "degree": 8 + }, + { + "id": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", + "label_zh": "再发现:拉格朗日约束的安全强化学习", + "kind": "validation", + "tier": "validation", + "topic": "deep_rl", + "phase": "frontier", + "year": 2019, + "summary_zh": "如果拉格朗日安全强化学习方法不存在,要再发明它们,研究者需要的构件是:约束马尔可夫决策过程作为对硬安全限制的形式化;PPO 或 TRPO 等策略优化算法;拉格朗日对偶把约束转化为乘子调节的可微目标;估计成本函数与价值函数的方法;以及把安全视为与回报对偶而非奖励内塞软项的认识立场。组合起来即可推出一类把碰撞概率、加速度上限等约束乘以可学习乘子并与策略联合优化、在保证安全约束被满足前提下最大化驾驶性能的安全强化学习算法。", + "building_blocks": [ + "concept:mdp", + "concept:policy_gradient", + "concept:actor_critic", + "concept:ppo", + "move:clipped_surrogate_objective", + "insight:safety_constraints_via_lagrangian_dual" + ], + "label": "Trace: Safe RL via Lagrangian Constrained Optimization", + "degree": 7 + }, + { + "id": "paradigm:modular_perception_to_planning_pipeline", + "label_zh": "范式:模块化感知到规划流水线", + "kind": "paradigm", + "tier": "paradigm", + "topic": "e2e_ad", + "phase": "prereq", + "year": 2018, + "summary_zh": "模块化感知到规划范式把自动驾驶分解为检测、跟踪、预测、规划、控制等明确的模块并通过结构化中间表示串联,押注于每个模块都可以独立验证、独立迭代、独立替换。其立场是把工程可维护性与可解释性置于联合优化之前,代表工作包括 Tesla 早期 AI Day 架构、Waymo 经典分层、以及任何在 BEVFormer 上接非神经规划器的产品系统。它的主要局限是模块边界压抑了端到端反馈、人为接口可能丢失下游任务真正需要的信息,并且每个模块的最优解不一定加起来等于系统最优。", + "building_blocks": [ + "paper:tesla_ai_day", + "paper:li2022bevformer", + "paper:transfuser", + "concept:bev", + "move:lift_2d_to_3d", + "insight:set_prediction_eliminates_postprocessing_heuristics" + ], + "label": "Paradigm: Modular Perception-to-Planning Pipeline", + "degree": 6 + }, + { + "id": "paradigm:differentiable_end_to_end_imitation", + "label_zh": "范式:可微端到端模仿", + "kind": "paradigm", + "tier": "paradigm", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "summary_zh": "可微端到端模仿范式把感知到规划写成一个完全可微的神经网络并用人类驾驶日志作为监督,押注于在数据规模足够大时端到端反馈能够发现比手工接口更优的表征。代表工作包括 UniAD、PlanT、VADv2、TransFuser,它们的共同立场是模块化的人工接口最终会成为性能瓶颈。它的主要局限是难以验证、对协变量偏移敏感、奖励隐式且难以注入硬安全约束,且对车载推断与传感器变化的鲁棒性需要额外的工程努力。", + "building_blocks": [ + "paper:2212.10156", + "paper:2210.14222", + "paper:vadv2", + "paper:transfuser", + "concept:imitation_learning", + "move:cross_attention_query", + "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong" + ], + "label": "Paradigm: Differentiable End-to-End Imitation", + "degree": 7 + }, + { + "id": "paradigm:model_based_world_imagination_planning", + "label_zh": "范式:基于世界模型的想象规划", + "kind": "paradigm", + "tier": "paradigm", + "topic": "deep_rl", + "phase": "frontier", + "year": 2018, + "summary_zh": "基于世界模型的想象规划范式把环境动力学学习成一个可微生成模型,再让策略在想象的潜空间或像素空间中以低代价 rollout,押注于这种方式可以把样本复杂度从昂贵的真实交互转移到便宜的想象推演。代表工作包括 World Models、Dreamer 系列、GAIA-1、DriveDreamer 与 CF-VLA 中的扩散世界模型。它的主要局限是世界模型的预测漂移会污染长视野规划,且对稀有事件的覆盖仍然取决于训练分布,导致罕见但关键的事故场景未必能被想象出来。", + "building_blocks": [ + "paper:world_models", + "paper:gaia1", + "paper:drivedreamer", + "move:latent_imagination_rollout", + "move:diffusion_denoise_sampling", + "insight:world_models_let_planning_be_done_in_imagination", + "insight:long_tail_solved_by_synthesis_not_data_alone" + ], + "label": "Paradigm: Model-Based World Imagination Planning", + "degree": 7 + }, + { + "id": "paradigm:foundation_model_zero_shot_driving_agent", + "label_zh": "范式:基础模型零样本驾驶智能体", + "kind": "paradigm", + "tier": "paradigm", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "基础模型零样本驾驶智能体范式把通用预训练 VLM 或 LLM 当作驾驶决策核心,通过提示工程、少样本示例和工具调用解决长尾问题,押注于通用智能可以通过 scaling 与对齐迁移到驾驶。代表工作包括 DriveVLM、Agent-Driver、LINGO-2 以及 CF-VLA 的反思变体。它的主要局限是 VLM 的几何与控制精度不足、推断延迟高、对车载算力压力大、安全性难以严格验证,因此实际部署多采取与传统快规划器并行的双系统架构。", + "building_blocks": [ + "paper:gpt3", + "paper:llava", + "paper:2402.12289", + "paper:2311.10813", + "concept:vla", + "concept:tool_use", + "move:dual_system_fast_slow", + "insight:dual_system_handles_latency_quality_tradeoff", + "insight:in_context_learning_emerges_at_scale" + ], + "label": "Paradigm: Foundation Model Zero-shot Driving Agent", + "degree": 10 + }, + { + "id": "paradigm:brain_inspired_event_sparse_compute", + "label_zh": "范式:类脑事件稀疏计算", + "kind": "paradigm", + "tier": "paradigm", + "topic": "brain_inspired", + "phase": "frontier", + "year": 2020, + "summary_zh": "类脑事件稀疏计算范式把脉冲神经元、事件相机、神经形态硬件作为核心计算单元,押注于自然信号的稀疏性可以在能耗与延迟上换取数量级优势。代表工作包括 Loihi 系列芯片、Spike-driven Transformer、事件相机视觉栈以及汽车端的低功耗感知试验。它与 Sutton 苦涩教训形成鲜明对照,主要局限是训练算法不成熟、与主流密集模型生态脱节、在大规模数据上仍未达到与稠密 transformer 相当的精度,因此目前主要作为车载补充而非主路径。", + "building_blocks": [ + "concept:spiking_nn", + "paper:2307.01694", + "move:spike_event_compute", + "insight:event_sparse_compute_matches_energy_budget" + ], + "label": "Paradigm: Brain-Inspired Event-Sparse Compute", + "degree": 4 + }, + { + "id": "paradigm:counterfactual_data_centric_safety", + "label_zh": "范式:以反事实数据为中心的安全", + "kind": "paradigm", + "tier": "paradigm", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2024, + "summary_zh": "反事实数据中心安全范式把驾驶安全问题主要视为数据问题而非模型问题,依靠合成反事实场景、扰动安全约束、对抗性场景生成来覆盖真实世界采样不到的长尾事件。代表工作包括 CF-VLA、Waymo CCD、CARLA leaderboard 中的对抗场景以及反事实重规划方法。它的立场是更多的真实数据存在边际递减,应该把投入转向高质量的反事实合成。主要局限是合成数据的分布偏置可能与真实事故分布不一致,需要严格的分布匹配工具与人在回路审核。", + "building_blocks": [ + "paper:2512.24426", + "paper:gaia1", + "paper:drivedreamer", + "concept:counterfactual", + "concept:meta_action", + "move:counterfactual_replan", + "insight:long_tail_solved_by_synthesis_not_data_alone" + ], + "label": "Paradigm: Counterfactual Data-Centric Safety", + "degree": 7 + }, + { + "id": "paradigm:knowledge_driven_reflective_agent", + "label_zh": "范式:知识驱动反思智能体", + "kind": "paradigm", + "tier": "paradigm", + "topic": "vlm_vla", + "phase": "frontier", + "year": 2023, + "summary_zh": "知识驱动反思智能体范式把 LLM 视为一个可以读写自然语言记忆、做事后反思并把经验沉淀为可复用知识的认知核心,押注于符号反思可以在数据有限时弥补 scaling 的不足。代表工作包括 DiLu、Reflexion 启发的驾驶变体、以及把法律法规与道路手册显式作为提示注入的 Agent-Driver。它的主要立场是与 Sutton 苦涩教训形成有意识的对照,承认知识工程在小规模、数据稀缺、跨地域迁移时仍有杠杆。主要局限是反思与记忆的可靠性依赖 LLM 推理稳定性,并且记忆膨胀后检索的精度与延迟需要工程化解决。", + "building_blocks": [ + "paper:gpt3", + "paper:2309.16292", + "paper:2311.10813", + "concept:cot", + "concept:tool_use", + "essay:bitter_lesson", + "insight:symbolic_intermediate_enables_interpretability_and_transfer" + ], + "label": "Paradigm: Knowledge-Driven Reflective Agent", + "degree": 7 + }, + { + "id": "paradigm:scaling_data_with_self_supervision", + "label_zh": "范式:以自监督扩张数据规模", + "kind": "paradigm", + "tier": "paradigm", + "topic": "ssl_vision", + "phase": "core", + "year": 2020, + "summary_zh": "以自监督扩张数据范式承诺把所有可获得的无标签数据通过掩码预测、对比对齐、自蒸馏等方式变成训练信号,押注于这样得到的通用表征可以在下游驾驶任务上以少量标签微调而获得显著迁移。代表工作包括 BERT、MAE、DINOv2、DINOv3、SAM 以及驾驶领域的占据预训练与 GAIA-1 自监督世界模型。它的主要局限是预训练任务与下游驾驶任务并非完全对齐、对计算预算极敏感,且自监督本身并不能解决奖励、规划与因果反思这类问题。", + "building_blocks": [ + "paper:dinov2", + "paper:2508.10104", + "paper:sam", + "concept:ssl", + "move:masking_for_pretext", + "move:contrastive_alignment", + "insight:masked_prediction_yields_self_supervised_signal", + "insight:foundation_pretraining_decouples_data_from_task", + "insight:scaling_laws_predict_capability_emergence" + ], + "label": "Paradigm: Scaling Data with Self-Supervision", + "degree": 9 + }, + { + "id": "paper:nuplan", + "label": "nuPlan", + "label_zh": "nuPlan(首个大规模闭环规划基准)", + "kind": "paper", + "tier": "A", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2021, + "card": "paper_nuplan.md", + "summary_zh": "nuPlan 是 Motional 提出的首个大规模闭环自动驾驶规划基准,提供 1500 小时人类驾驶数据并定义了基于交通规则、舒适度与进展度的混合评分体系,使规划器评估首次从开环位移误差升级为闭环行为指标。", + "degree": 8 + }, + { + "id": "paper:waymo_motion", + "label": "Waymo Open Motion", + "label_zh": "Waymo Open Motion 数据集", + "kind": "paper", + "tier": "A", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2021, + "card": "paper_waymo_motion.md", + "summary_zh": "Waymo Open Motion Dataset 提供约十万段高质量交互场景,专门为行为预测与多智能体规划设计,配套的预测排行榜推动了 MultiPath、Wayformer 等结构的迭代。", + "degree": 4 + }, + { + "id": "paper:argoverse2", + "label": "Argoverse 2", + "label_zh": "Argoverse 2(运动预测与高精地图基准)", + "kind": "paper", + "tier": "A", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2022, + "card": "paper_argoverse2.md", + "summary_zh": "Argoverse 2 由 Argo AI 发布,包含 250000 段六类城市预测场景、激光雷达数据集与高精地图,并提出长时序多智能体预测指标 brier-minFDE,是预测与拓扑感知研究的事实标准。", + "degree": 4 + }, + { + "id": "paper:navsim", + "label": "NAVSIM", + "label_zh": "NAVSIM(非反应式闭环代理评测)", + "kind": "paper", + "tier": "A", + "topic": "evaluation_benchmark", + "phase": "frontier", + "year": 2024, + "card": "paper_navsim.md", + "summary_zh": "NAVSIM 通过在 nuPlan 与 OpenScene 之上构建非反应式短时滚动评估,提出 PDM Score 这一可与真实安全代理强相关的离线指标,弥合了完全开环与昂贵闭环之间的鸿沟。", + "degree": 5 + }, + { + "id": "paper:bench2drive", + "label": "Bench2Drive", + "label_zh": "Bench2Drive(端到端闭环 CARLA 基准)", + "kind": "paper", + "tier": "A", + "topic": "evaluation_benchmark", + "phase": "frontier", + "year": 2024, + "card": "paper_bench2drive.md", + "summary_zh": "Bench2Drive 在 CARLA Leaderboard 2.0 之上提供 44 个能力分桶与统一训练协议,使 UniAD、VAD 等端到端模型可在同一闭环环境下被公平比较,揭示了离线 L2 与闭环成功率之间的弱相关性。", + "degree": 7 + }, + { + "id": "paper:carla_lb2", + "label": "CARLA Leaderboard 2.0", + "label_zh": "CARLA Leaderboard 2.0", + "kind": "paper", + "tier": "B", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2023, + "card": "paper_carla_lb2.md", + "summary_zh": "CARLA Leaderboard 2.0 引入更长路线、更密集对抗事件与新评分公式,把驾驶成功率与违规分数解耦,迫使端到端模型在长尾事件上具备真正的恢复与避撞能力。", + "degree": 5 + }, + { + "id": "paper:highway_env", + "label": "HighwayEnv", + "label_zh": "HighwayEnv(轻量决策仿真环境)", + "kind": "paper", + "tier": "B", + "topic": "simulator", + "phase": "prereq", + "year": 2018, + "card": "paper_highway_env.md", + "summary_zh": "HighwayEnv 是基于 OpenAI Gym 的轻量级高速公路决策仿真器,支持 IDM、MOBIL 与多智能体设置,是验证 DQN/PPO 等算法在变道与汇入任务上的事实平台。", + "degree": 3 + }, + { + "id": "paper:metadrive", + "label": "MetaDrive", + "label_zh": "MetaDrive(程序生成驾驶模拟器)", + "kind": "paper", + "tier": "B", + "topic": "simulator", + "phase": "core", + "year": 2021, + "card": "paper_metadrive.md", + "summary_zh": "MetaDrive 通过程序化生成无穷场景与可重复随机种子,专门用于研究强化学习的泛化与安全性,并提供与 SUMO、Waymo Open Motion 重放兼容的驾驶接口。", + "degree": 5 + }, + { + "id": "paper:smarts", + "label": "SMARTS", + "label_zh": "SMARTS(多智能体驾驶仿真)", + "kind": "paper", + "tier": "B", + "topic": "simulator", + "phase": "core", + "year": 2020, + "card": "paper_smarts.md", + "summary_zh": "华为诺亚提出的 SMARTS 平台聚焦多智能体交互式驾驶,支持灵活的社交车辆行为模型与课程学习接口,被广泛用于多智能体 RL 与博弈式规划研究。", + "degree": 4 + }, + { + "id": "paper:commonroad", + "label": "CommonRoad", + "label_zh": "CommonRoad(形式化运动规划基准)", + "kind": "paper", + "tier": "B", + "topic": "simulator", + "phase": "core", + "year": 2017, + "card": "paper_commonroad.md", + "summary_zh": "CommonRoad 由 TU München 维护,提供可被形式化验证的运动规划场景与代价函数语言,是 MPC、采样规划与神经规划共同的基准。", + "degree": 3 + }, + { + "id": "paper:lyft_l5", + "label": "Lyft L5 Prediction", + "label_zh": "Lyft Level-5 预测数据集", + "kind": "paper", + "tier": "B", + "topic": "dataset", + "phase": "core", + "year": 2020, + "card": "paper_lyft_l5.md", + "summary_zh": "Lyft Level-5 数据集包含 1000 小时车队真实驾驶日志与对应栅格化语义地图,是第一个把行为预测当作大规模监督学习问题来求解的工业数据集。", + "degree": 1 + }, + { + "id": "paper:pandaset", + "label": "PandaSet", + "label_zh": "PandaSet(多模态感知数据集)", + "kind": "paper", + "tier": "B", + "topic": "dataset", + "phase": "core", + "year": 2020, + "card": "paper_pandaset.md", + "summary_zh": "Hesai 与 Scale AI 联合发布的 PandaSet 提供机械式与固态激光雷达双路数据及精细 3D 语义标签,是研究激光雷达硬件差异对感知影响的少见公开数据集。", + "degree": 1 + }, + { + "id": "paper:apolloscape", + "label": "ApolloScape", + "label_zh": "ApolloScape(百度多任务驾驶数据集)", + "kind": "paper", + "tier": "B", + "topic": "dataset", + "phase": "core", + "year": 2018, + "card": "paper_apolloscape.md", + "summary_zh": "百度 ApolloScape 涵盖语义分割、车道线、轨迹与立体匹配等多任务大规模标注,是国内最早开放的工业级自动驾驶数据集之一。", + "degree": 1 + }, + { + "id": "paper:bdd100k", + "label": "BDD100K", + "label_zh": "BDD100K(伯克利多样化驾驶视频)", + "kind": "paper", + "tier": "B", + "topic": "dataset", + "phase": "core", + "year": 2018, + "card": "paper_bdd100k.md", + "summary_zh": "BDD100K 提供十万段一分钟驾驶视频与十类多任务标签,覆盖夜间、雨雪、城市等多种工况,使研究者能在感知模型中明确量化领域偏移。", + "degree": 2 + }, + { + "id": "paper:womd_pred", + "label": "WOMD prediction benchmark", + "label_zh": "WOMD 预测基准", + "kind": "paper", + "tier": "B", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2021, + "card": "paper_womd_pred.md", + "summary_zh": "Waymo Open Motion Dataset 的预测排行榜定义了 minADE、minFDE、Miss Rate 与 mAP 等核心指标,并对长尾交互场景采用难度加权评分。", + "degree": 3 + }, + { + "id": "paper:interaction_dataset", + "label": "INTERACTION Dataset", + "label_zh": "INTERACTION 数据集", + "kind": "paper", + "tier": "B", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2019, + "card": "paper_interaction_dataset.md", + "summary_zh": "INTERACTION 数据集聚焦无信号路口、并道与环岛等高交互性场景,由全球多个研究组共同采集,是博弈论建模与社会力学研究的标准平台。", + "degree": 2 + }, + { + "id": "paper:shift_dataset", + "label": "SHIFT", + "label_zh": "SHIFT(连续域偏移合成数据集)", + "kind": "paper", + "tier": "B", + "topic": "dataset", + "phase": "frontier", + "year": 2022, + "card": "paper_shift_dataset.md", + "summary_zh": "SHIFT 在 CARLA 中合成连续变化的天气、时间、密度等参数,使研究者能精确量化分布偏移对感知模型的影响,是研究持续学习与领域适配的纯净基准。", + "degree": 5 + }, + { + "id": "paper:v2x_sim", + "label": "V2X-Sim", + "label_zh": "V2X-Sim(车路云协同仿真数据集)", + "kind": "paper", + "tier": "B", + "topic": "dataset", + "phase": "frontier", + "year": 2022, + "card": "paper_v2x_sim.md", + "summary_zh": "V2X-Sim 在 CARLA 中合成多车、路侧单元的协同感知数据,配套点云、图像与通信延迟模型,是协同感知与协同规划研究的早期标杆。", + "degree": 3 + }, + { + "id": "paper:flashattention", + "label": "FlashAttention", + "label_zh": "FlashAttention(IO 感知精确注意力)", + "kind": "paper", + "tier": "A", + "topic": "efficient_computing", + "phase": "core", + "year": 2022, + "card": "paper_flashattention.md", + "summary_zh": "FlashAttention 通过把注意力切片为可放入 GPU SRAM 的 tile,并将 softmax 与矩阵乘融合,实现了精确注意力的 2-4 倍加速,是长序列 transformer 实用化的关键工程突破。", + "degree": 7 + }, + { + "id": "paper:performer", + "label": "Performer", + "label_zh": "Performer(核近似线性注意力)", + "kind": "paper", + "tier": "B", + "topic": "efficient_computing", + "phase": "core", + "year": 2020, + "card": "paper_performer.md", + "summary_zh": "Performer 用 FAVOR+ 随机特征近似 softmax 核,把注意力复杂度降至 O(N),为长视频与高频感知序列提供了可扩展的替代方案。", + "degree": 5 + }, + { + "id": "paper:linear_attention", + "label": "Linear Attention", + "label_zh": "线性注意力(Katharopoulos 2020)", + "kind": "paper", + "tier": "B", + "topic": "efficient_computing", + "phase": "core", + "year": 2020, + "card": "paper_linear_attention.md", + "summary_zh": "Katharopoulos 等用核函数把注意力重新表达为线性递归形式,使其在自回归推理时与 RNN 等价,是当前许多高效驾驶序列模型的数学起点。", + "degree": 4 + }, + { + "id": "paper:gptq", + "label": "GPTQ", + "label_zh": "GPTQ(后训练 4bit 量化)", + "kind": "paper", + "tier": "B", + "topic": "efficient_computing", + "phase": "frontier", + "year": 2022, + "card": "paper_gptq.md", + "summary_zh": "GPTQ 利用近似二阶 Hessian 信息一次性完成大模型 4bit 量化,几乎不损失精度,使大型 VLM 能在车规算力下离线部署成为可能。", + "degree": 4 + }, + { + "id": "paper:awq", + "label": "AWQ", + "label_zh": "AWQ(激活感知权重量化)", + "kind": "paper", + "tier": "B", + "topic": "efficient_computing", + "phase": "frontier", + "year": 2023, + "card": "paper_awq.md", + "summary_zh": "AWQ 根据通道激活分布对权重进行尺度变换后再量化,避免显著通道被压扁,是车端 LLM/VLM 推理常用的工业级量化方案。", + "degree": 3 + }, + { + "id": "paper:distill_vlm", + "label": "DistillVLM", + "label_zh": "DistillVLM(VLM 蒸馏综述)", + "kind": "paper", + "tier": "B", + "topic": "efficient_computing", + "phase": "frontier", + "year": 2023, + "card": "paper_distill_vlm.md", + "summary_zh": "DistillVLM 系列工作把大尺寸视觉语言模型的推理能力蒸馏到 1-3B 量级专家模型,是车端实时驾驶解释与意图预测的主要落地路径。", + "degree": 4 + }, + { + "id": "paper:loihi2", + "label": "Intel Loihi 2", + "label_zh": "Intel Loihi 2 神经形态芯片", + "kind": "paper", + "tier": "A", + "topic": "neuromorphic_hardware", + "phase": "frontier", + "year": 2021, + "card": "paper_loihi2.md", + "summary_zh": "Intel Loihi 2 是异步事件驱动神经形态芯片,单芯片支持百万级可编程脉冲神经元与三因子塑性,是 SNN 算法在端侧实测能耗的首选硬件。", + "degree": 8 + }, + { + "id": "paper:truenorth", + "label": "IBM TrueNorth", + "label_zh": "IBM TrueNorth 神经形态芯片", + "kind": "paper", + "tier": "B", + "topic": "neuromorphic_hardware", + "phase": "core", + "year": 2014, + "card": "paper_truenorth.md", + "summary_zh": "TrueNorth 是 IBM 早期纯数字事件驱动神经形态芯片,单芯片 100 万神经元、仅 70 mW,证明了大规模 SNN 在嵌入式场景的能效优势。", + "degree": 3 + }, + { + "id": "paper:tianjic", + "label": "Tianjic", + "label_zh": "天机芯(清华混合范式芯片)", + "kind": "paper", + "tier": "B", + "topic": "neuromorphic_hardware", + "phase": "core", + "year": 2019, + "card": "paper_tianjic.md", + "summary_zh": "清华施路平团队的天机芯把 ANN 与 SNN 在同一硅片上统一调度,曾驱动自动驾驶自行车展示,是混合范式神经形态计算的代表作。", + "degree": 5 + }, + { + "id": "paper:grai", + "label": "GrAI Matter", + "label_zh": "GrAI Matter VIP 系列芯片", + "kind": "paper", + "tier": "B", + "topic": "neuromorphic_hardware", + "phase": "frontier", + "year": 2022, + "card": "paper_grai.md", + "summary_zh": "GrAI Matter 的 NeuronFlow VIP 系列把事件驱动稀疏计算与传统视觉流水线结合,主打超低延迟车载感知 ASIC。", + "degree": 2 + }, + { + "id": "paper:dvs_event_camera", + "label": "DVS event camera", + "label_zh": "DVS 事件相机", + "kind": "paper", + "tier": "A", + "topic": "neuromorphic_hardware", + "phase": "core", + "year": 2008, + "card": "paper_dvs_event_camera.md", + "summary_zh": "DVS 事件相机每像素异步输出亮度变化事件,具有微秒延迟与 120dB 动态范围,是高速运动与极端光照下感知的天然传感器。", + "degree": 5 + }, + { + "id": "paper:gs_for_ad", + "label": "3DGS for AD", + "label_zh": "3D 高斯泼溅用于自动驾驶(StreetGaussians 等)", + "kind": "paper", + "tier": "A", + "topic": "data_engine", + "phase": "frontier", + "year": 2024, + "card": "paper_gs_for_ad.md", + "summary_zh": "StreetGaussians、DrivingGaussian 等工作把 3D 高斯泼溅扩展到带运动物体的街景重建,使闭环仿真器能以照片级真实度回放并扰动真实日志。", + "degree": 6 + }, + { + "id": "paper:lidar_cam_calib", + "label": "Continuous-time LiDAR-Camera calibration", + "label_zh": "连续时间 LiDAR-相机时空同步标定", + "kind": "paper", + "tier": "B", + "topic": "data_engine", + "phase": "core", + "year": 2020, + "card": "paper_lidar_cam_calib.md", + "summary_zh": "连续时间样条标定把 LiDAR、相机、IMU 的时空外参联合优化为时间函数,能在运行中校正同步误差与振动漂移,是真实车队的工程基线。", + "degree": 1 + }, + { + "id": "paper:iso26262", + "label": "ISO 26262", + "label_zh": "ISO 26262 道路车辆功能安全", + "kind": "paper", + "tier": "A", + "topic": "safety_standard", + "phase": "core", + "year": 2011, + "card": "paper_iso26262.md", + "summary_zh": "ISO 26262 定义了汽车电子电气系统的功能安全生命周期与 ASIL 等级,是任何量产自动驾驶硬件与软件必须满足的功能安全基础。", + "degree": 4 + }, + { + "id": "paper:sotif_21448", + "label": "ISO 21448 SOTIF", + "label_zh": "ISO 21448 预期功能安全 SOTIF", + "kind": "paper", + "tier": "A", + "topic": "safety_standard", + "phase": "core", + "year": 2019, + "card": "paper_sotif_21448.md", + "summary_zh": "ISO 21448 SOTIF 把传统功能安全无法覆盖的功能不足与可预见误用纳入风险接受准则,是为机器学习驱动的自动驾驶量身定制的国际标准。", + "degree": 7 + }, + { + "id": "paper:ul4600", + "label": "ANSI/UL 4600", + "label_zh": "UL 4600 自动驾驶安全论证标准", + "kind": "paper", + "tier": "B", + "topic": "safety_standard", + "phase": "frontier", + "year": 2020, + "card": "paper_ul4600.md", + "summary_zh": "UL 4600 提出以结构化安全论证(safety case)作为自动驾驶安全证明的核心,要求基于声明—证据—假设的三元组进行可审计推理。", + "degree": 5 + }, + { + "id": "paper:tesla_autolabel", + "label": "Tesla Auto-Labeling", + "label_zh": "Tesla 自动标注与数据引擎", + "kind": "paper", + "tier": "A", + "topic": "data_engine", + "phase": "core", + "year": 2022, + "card": "paper_tesla_autolabel.md", + "summary_zh": "Tesla AI Day 披露的离线 4D 重建自动标注流水线,把车队回传片段离线超算重投影为高精轨迹标签,是数据引擎闭环的代表性工业实现。", + "degree": 6 + }, + { + "id": "paper:waymo_scenario_mining", + "label": "Waymo Scenario Mining", + "label_zh": "Waymo 场景挖掘与长尾发现", + "kind": "paper", + "tier": "B", + "topic": "data_engine", + "phase": "core", + "year": 2022, + "card": "paper_waymo_scenario_mining.md", + "summary_zh": "Waymo 通过结构化查询语言与轨迹嵌入检索从车队日志中主动挖掘罕见交互场景,被用于针对性回归测试与训练样本扩充。", + "degree": 4 + }, + { + "id": "move:design_closed_loop_metric_correlated_with_real_world_safety", + "label": "Design closed-loop metric correlated with real safety", + "label_zh": "设计与真实世界安全率强相关的闭环指标", + "kind": "move", + "tier": "move", + "topic": "evaluation_benchmark", + "phase": "frontier", + "year": 2024, + "card": "move_design_closed_loop_metric.md", + "summary_zh": "围绕真实车队事故率与脱离率反推闭环评测指标,把碰撞、近碰撞、舒适度、规则违规与进展度按事故贡献加权融合,使离线评测对决策有真正的判别力。", + "degree": 5 + }, + { + "id": "move:augment_dataset_via_offline_scenario_perturbation", + "label": "Perturb logs offline to augment dataset", + "label_zh": "用离线扰动驾驶日志扩充训练数据", + "kind": "move", + "tier": "move", + "topic": "data_engine", + "phase": "core", + "year": 2023, + "card": "move_perturb_logs.md", + "summary_zh": "在真实日志重建的 3DGS 或 NeRF 场景中对自车与他车的轨迹、速度、外观进行可控扰动,用极小的标注成本生成具有反事实意义的训练样本。", + "degree": 6 + }, + { + "id": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", + "label": "Active learning over hardest frames", + "label_zh": "用主动学习挑选最难未标注帧", + "kind": "move", + "tier": "move", + "topic": "data_engine", + "phase": "core", + "year": 2022, + "card": "move_active_learning_hard_frames.md", + "summary_zh": "在车队回传日志中按模型不确定性、罕见性嵌入与下游闭环失败贡献多准则打分,仅把得分最高的帧送人工标注或仿真扩增,使标注预算回报最大化。", + "degree": 4 + }, + { + "id": "move:distill_large_VLM_into_small_realtime_specialist", + "label": "Distill large VLM into realtime specialist", + "label_zh": "把大 VLM 蒸馏为车端实时小模型", + "kind": "move", + "tier": "move", + "topic": "efficient_computing", + "phase": "frontier", + "year": 2024, + "card": "move_distill_vlm_small.md", + "summary_zh": "用大规模 VLM 离线生成轨迹解释、风险标签与高层动作,再以教师-学生范式蒸馏到 1-3B 参数的车端模型,让大模型能力以可承担成本上车。", + "degree": 3 + }, + { + "id": "move:quantize_attention_to_int8_with_calibration", + "label": "INT8 attention quantization", + "label_zh": "用校准把注意力量化到 INT8", + "kind": "move", + "tier": "move", + "topic": "efficient_computing", + "phase": "core", + "year": 2023, + "card": "move_int8_attention.md", + "summary_zh": "针对注意力中 softmax 与残差敏感性,采用每通道激活校准与 KV cache 异步量化策略,把 transformer 推理压到 INT8 而不损失驾驶决策质量。", + "degree": 5 + }, + { + "id": "move:replace_dense_attention_with_sparse_event_driven_attention", + "label": "Sparse event-driven attention", + "label_zh": "用事件驱动稀疏注意力替换稠密注意力", + "kind": "move", + "tier": "move", + "topic": "efficient_computing", + "phase": "frontier", + "year": 2024, + "card": "move_sparse_event_attention.md", + "summary_zh": "把稠密注意力替换为只在脉冲发生时启用 QK 计算的事件驱动注意力,使能耗与场景稀疏度成线性关系,是 SNN 与神经形态硬件协同的核心算子。", + "degree": 6 + }, + { + "id": "move:use_event_camera_microsecond_latency_for_emergency_braking", + "label": "Event camera microsecond braking", + "label_zh": "用事件相机微秒延迟触发紧急制动", + "kind": "move", + "tier": "move", + "topic": "neuromorphic_hardware", + "phase": "frontier", + "year": 2023, + "card": "move_event_camera_braking.md", + "summary_zh": "在传统帧相机感知管线之外并联事件相机的低延迟运动检测模块,使切入与急刹场景的端到端反应时延从百毫秒级压缩到十毫秒级。", + "degree": 1 + }, + { + "id": "move:implement_spiking_neuron_with_surrogate_gradient_for_backprop", + "label": "Surrogate gradient for SNN backprop", + "label_zh": "用代理梯度实现脉冲神经元的反向传播", + "kind": "move", + "tier": "move", + "topic": "neuromorphic_hardware", + "phase": "core", + "year": 2018, + "card": "move_surrogate_gradient.md", + "summary_zh": "用平滑的代理函数替代脉冲激活的不可微阶跃,使脉冲神经元能在标准 PyTorch 反向传播框架中训练,是 SNN 走出实验室的关键技巧。", + "degree": 3 + }, + { + "id": "move:co_design_silicon_with_algorithm_for_minimum_energy", + "label": "Co-design silicon and algorithm", + "label_zh": "把芯片与算法联合设计以最小化能耗", + "kind": "move", + "tier": "move", + "topic": "neuromorphic_hardware", + "phase": "frontier", + "year": 2023, + "card": "move_silicon_algo_codesign.md", + "summary_zh": "把数据通路、片上存储与算法稀疏模式作为一个优化对象,让网络结构、量化精度、调度顺序与硬件 NoC 拓扑共同搜索能耗-延迟前沿。", + "degree": 5 + }, + { + "id": "move:cache_KV_state_across_frames_to_amortize_attention_cost", + "label": "Cross-frame KV cache", + "label_zh": "跨帧缓存 KV 以摊销注意力开销", + "kind": "move", + "tier": "move", + "topic": "efficient_computing", + "phase": "frontier", + "year": 2024, + "card": "move_cross_frame_kv_cache.md", + "summary_zh": "把视觉 token 的 K/V 缓存按时间窗滚动复用并只增量更新发生显著变化的部分,使长时序 BEV transformer 推理代价从 O(T) 降至接近 O(1)。", + "degree": 4 + }, + { + "id": "move:tile_attention_to_fit_SRAM_for_speedup", + "label": "Tile attention into SRAM (FlashAttention)", + "label_zh": "把注意力切块以适配 SRAM 获得加速", + "kind": "move", + "tier": "move", + "topic": "efficient_computing", + "phase": "core", + "year": 2022, + "card": "move_tile_attention_sram.md", + "summary_zh": "把 Q、K、V 沿序列维切成可放入 GPU SRAM 的小块,并通过 online softmax 在小块间累加结果,避免对 HBM 的反复读写,是精确注意力加速的核心技巧。", + "degree": 3 + }, + { + "id": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", + "label": "Linear kernel attention for long sequence", + "label_zh": "用线性核注意力处理长序列", + "kind": "move", + "tier": "move", + "topic": "efficient_computing", + "phase": "core", + "year": 2022, + "card": "move_linear_kernel_attention.md", + "summary_zh": "用核函数把 softmax 注意力近似为可交换求和顺序的形式,从而把复杂度降至 O(N),使长视频与连续驾驶日志的训练可线性扩展。", + "degree": 5 + }, + { + "id": "move:share_LiDAR_camera_calibration_via_continuous_time_optimization", + "label": "Continuous-time multi-sensor calibration", + "label_zh": "用连续时间优化联合标定 LiDAR 与相机", + "kind": "move", + "tier": "move", + "topic": "data_engine", + "phase": "core", + "year": 2021, + "card": "move_continuous_time_calibration.md", + "summary_zh": "把多传感器外参与时间偏移建模为 B 样条曲线并与 IMU 预积分共同优化,使在线运行中也能持续修正同步漂移与温度变形。", + "degree": 3 + }, + { + "id": "move:auto_label_with_offline_model_then_human_in_loop_validate", + "label": "Auto-label + human-in-loop validate", + "label_zh": "用离线大模型自动标注再人工抽样验证", + "kind": "move", + "tier": "move", + "topic": "data_engine", + "phase": "core", + "year": 2022, + "card": "move_auto_label_hitl.md", + "summary_zh": "先用大算力离线模型生成 4D 轨迹与语义标签,再让标注员仅审核置信度低或下游影响大的样本,把人工成本从线性降到对数级。", + "degree": 5 + }, + { + "id": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", + "label": "Safety constraints in signal temporal logic", + "label_zh": "用信号时序逻辑表达并验证安全约束", + "kind": "move", + "tier": "move", + "topic": "safety_standard", + "phase": "frontier", + "year": 2023, + "card": "move_stl_safety_constraint.md", + "summary_zh": "把保持车距、限速、信号灯遵守等行为约束写成可解释、可量化的信号时序逻辑(STL)公式,运行中既可作监控也可纳入轨迹优化的硬性约束。", + "degree": 5 + }, + { + "id": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", + "label": "Add shielding layer over policy", + "label_zh": "在策略输出端加屏蔽层以拒绝不安全动作", + "kind": "move", + "tier": "move", + "topic": "safety_standard", + "phase": "frontier", + "year": 2023, + "card": "move_shielding_layer.md", + "summary_zh": "在神经规划器后串接一个由可达性分析或控制屏障函数定义的屏蔽层,对违反安全包络的动作进行投影或替换为后备策略,使学习模型可在安全约束下逐步上车。", + "degree": 5 + }, + { + "id": "move:treat_corner_case_as_OOD_detection_then_route_to_human", + "label": "Treat corner case as OOD then escalate", + "label_zh": "把长尾事件视为 OOD 检测并升级到人类", + "kind": "move", + "tier": "move", + "topic": "safety_standard", + "phase": "frontier", + "year": 2024, + "card": "move_corner_case_ood.md", + "summary_zh": "用密度估计、能量函数或 VLM 描述匹配等手段检测 OOD 输入,触发后切换到保守策略、远程驾驶或安全停车,避免模型在认知盲区强行决策。", + "degree": 5 + }, + { + "id": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", + "label": "Replay sim with perturbed conditions", + "label_zh": "用扰动初始条件的回放仿真评估鲁棒性", + "kind": "move", + "tier": "move", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2023, + "card": "move_replay_perturbation.md", + "summary_zh": "在真实日志重建场景中对初始位姿、速度、他车意图施加噪声扰动,统计性地评估规划器在邻域内的稳定性,是闭环回归测试的事实方法。", + "degree": 5 + }, + { + "id": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", + "label": "Track offline-vs-closed-loop correlation", + "label_zh": "持续追踪离线与闭环指标的相关性以筛选模型", + "kind": "move", + "tier": "move", + "topic": "evaluation_benchmark", + "phase": "frontier", + "year": 2024, + "card": "move_metric_correlation_tracking.md", + "summary_zh": "定期把候选模型同时在离线集合与昂贵闭环仿真上评分,对比 Spearman 相关性以筛选出真正预测闭环性能的指标,避免开发团队被偏好性指标误导。", + "degree": 5 + }, + { + "id": "move:use_difficulty_aware_curriculum_to_accelerate_RL", + "label": "Difficulty-aware curriculum for driving RL", + "label_zh": "用难度感知课程加速驾驶强化学习", + "kind": "move", + "tier": "move", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2022, + "card": "move_difficulty_curriculum.md", + "summary_zh": "按场景难度排序逐步引入更密集的交互与异常事件,避免策略在早期被淹没在长尾,使收敛速度与最终成功率显著优于均匀采样。", + "degree": 3 + }, + { + "id": "move:add_explanation_head_to_promote_interpretability", + "label": "Add explanation head to planner", + "label_zh": "为规划器增加解释头以提升可解释性", + "kind": "move", + "tier": "move", + "topic": "safety_standard", + "phase": "frontier", + "year": 2024, + "card": "move_explanation_head.md", + "summary_zh": "在端到端模型的轨迹头之外额外训练自然语言或符号化的解释头,使每个决策都能追溯到关键场景特征,是面向监管与事故复盘的必备工程实践。", + "degree": 3 + }, + { + "id": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", + "label": "Deep ensemble / evidential uncertainty", + "label_zh": "用深度集成或证据层做不确定性量化", + "kind": "move", + "tier": "move", + "topic": "safety_standard", + "phase": "core", + "year": 2020, + "card": "move_uncertainty_quantification.md", + "summary_zh": "用深度集成、MC-Dropout 或证据深度学习层估计预测分布的偏差与方差,将其作为安全屏蔽与人机交接的触发信号,是模型化不确定性向决策回路传播的标准做法。", + "degree": 4 + }, + { + "id": "move:perform_neural_architecture_search_with_latency_constraint", + "label": "NAS with latency constraint", + "label_zh": "在延迟约束下做神经架构搜索", + "kind": "move", + "tier": "move", + "topic": "efficient_computing", + "phase": "core", + "year": 2020, + "card": "move_nas_latency.md", + "summary_zh": "把车端 ECU 上的真实延迟、能耗与可调度性作为硬约束加入 NAS 目标函数,搜索能在 30fps 实时预算下保持精度的 BEV/规划网络。", + "degree": 2 + }, + { + "id": "move:formalize_safety_case_with_claim_evidence_assumption", + "label": "Safety case with claim-evidence-assumption", + "label_zh": "用声明-证据-假设结构形式化安全论证", + "kind": "move", + "tier": "move", + "topic": "safety_standard", + "phase": "frontier", + "year": 2022, + "card": "move_safety_case.md", + "summary_zh": "按 UL 4600 的安全论证语言把每个安全主张分解为子声明、证据与显式假设,使外部审计可以逐条核查证据链与残余风险,是可审计 AD 落地的核心工程范式。", + "degree": 4 + }, + { + "id": "move:replay_buffer_prioritize_safety_critical_transitions", + "label": "Prioritize safety-critical transitions", + "label_zh": "在回放缓冲中优先采样安全关键转移", + "kind": "move", + "tier": "move", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2022, + "card": "move_priority_safety_replay.md", + "summary_zh": "对涉及碰撞、近碰撞与急刹的转移赋予更高优先级,使 RL 策略不被海量的平凡跟车样本稀释,在安全长尾上的学习效率成倍提升。", + "degree": 3 + }, + { + "id": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", + "label": "Continual learning with rehearsal buffer", + "label_zh": "用回演缓冲做持续学习对抗遗忘", + "kind": "move", + "tier": "move", + "topic": "data_engine", + "phase": "frontier", + "year": 2023, + "card": "move_continual_learning_rehearsal.md", + "summary_zh": "在模型每次 OTA 更新前重新混入过去版本的代表性样本与硬例,防止新数据驱动的微调让旧场景能力遗忘,是车队规模化运营的必备机制。", + "degree": 4 + }, + { + "id": "problem:offline_metric_does_not_predict_closed_loop_performance", + "label": "Offline metric ≠ closed-loop perf", + "label_zh": "离线指标不能预测闭环性能", + "kind": "problem", + "tier": "problem", + "topic": "evaluation_benchmark", + "phase": "frontier", + "year": 2023, + "card": "problem_offline_vs_closed_loop.md", + "summary_zh": "实证表明位移误差等开环监督指标与闭环成功率相关性很弱,导致模型选择与论文结论可能与真实驾驶安全脱节。", + "degree": 4 + }, + { + "id": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented", + "label": "Rare safety events dominate risk", + "label_zh": "罕见安全事件主导真实风险却严重欠采样", + "kind": "problem", + "tier": "problem", + "topic": "evaluation_benchmark", + "phase": "core", + "year": 2022, + "card": "problem_rare_safety_events.md", + "summary_zh": "真实事故集中在长尾交互与极端工况,但训练与评测集中绝大多数样本为低风险跟车,造成模型在最关键场景上的统计置信度不足。", + "degree": 4 + }, + { + "id": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "label": "Energy budget too small for full transformer", + "label_zh": "车端能耗预算无法支撑 30fps 全注意力 transformer", + "kind": "problem", + "tier": "problem", + "topic": "efficient_computing", + "phase": "frontier", + "year": 2024, + "card": "problem_energy_budget.md", + "summary_zh": "量产车 ECU 通常只有 30-80W 可用于 AI 推理,而原始多模态 transformer 在 30fps 闭环下功耗远超此范围,必须依赖稀疏化、量化与缓存技巧。", + "degree": 10 + }, + { + "id": "problem:sensor_calibration_drift_over_vehicle_lifetime", + "label": "Sensor calibration drift over lifetime", + "label_zh": "传感器标定在整车寿命周期内漂移", + "kind": "problem", + "tier": "problem", + "topic": "data_engine", + "phase": "core", + "year": 2021, + "card": "problem_calibration_drift.md", + "summary_zh": "振动、温度循环与碰撞会让 LiDAR-相机-IMU 的相对外参缓慢漂移,若不持续在线标定,感知与重建质量会在数月内显著退化。", + "degree": 2 + }, + { + "id": "problem:label_noise_for_3d_object_categories", + "label": "Label noise in 3D object categories", + "label_zh": "3D 目标类别标注噪声", + "kind": "problem", + "tier": "problem", + "topic": "data_engine", + "phase": "core", + "year": 2020, + "card": "problem_label_noise_3d.md", + "summary_zh": "夜间、远距离与遮挡场景下人工标注的 3D 边界框与类别一致性不足 90%,直接影响检测器的可比较性与可验证性。", + "degree": 2 + }, + { + "id": "problem:verification_of_neural_network_safety_properties_at_scale", + "label": "Scalable NN safety verification", + "label_zh": "大规模神经网络安全属性的形式化验证", + "kind": "problem", + "tier": "problem", + "topic": "safety_standard", + "phase": "frontier", + "year": 2022, + "card": "problem_nn_verification.md", + "summary_zh": "在百兆参数级网络上完成对全部安全属性的形式化证明仍超出现有 SMT 求解器与符号传播工具的可扩展范围,是 AD 形式化安全的关键瓶颈。", + "degree": 3 + }, + { + "id": "problem:realistic_other_agent_behavior_in_simulator", + "label": "Realistic other-agent behavior in sim", + "label_zh": "仿真器中其他智能体的真实行为", + "kind": "problem", + "tier": "problem", + "topic": "simulator", + "phase": "core", + "year": 2021, + "card": "problem_agent_realism.md", + "summary_zh": "现有仿真器多采用 IDM 等手工模型驱动他车,难以再现真实人类驾驶员的礼让、博弈与误判,使闭环训练存在难以察觉的 sim-to-real 偏差。", + "degree": 3 + }, + { + "id": "problem:catastrophic_forgetting_under_continual_learning", + "label": "Catastrophic forgetting under continual update", + "label_zh": "持续学习下的灾难性遗忘", + "kind": "problem", + "tier": "problem", + "topic": "data_engine", + "phase": "frontier", + "year": 2022, + "card": "problem_catastrophic_forgetting.md", + "summary_zh": "在 OTA 不断引入新数据微调模型时,老旧场景与少数群体场景的能力会被覆盖性遗忘,造成已通过验收的功能在新版本中悄悄回退。", + "degree": 2 + }, + { + "id": "problem:auditability_of_decisions_for_regulatory_compliance", + "label": "Decision auditability for regulators", + "label_zh": "决策对监管机构的可审计性", + "kind": "problem", + "tier": "problem", + "topic": "safety_standard", + "phase": "frontier", + "year": 2023, + "card": "problem_auditability.md", + "summary_zh": "黑盒端到端模型在事故复盘与监管问询中难以提供逐步可追溯的因果链,使制造商面临严苛的合规与法律风险。", + "degree": 4 + }, + { + "id": "problem:simulator_visual_gap_breaks_perception_models", + "label": "Simulator visual gap breaks perception", + "label_zh": "仿真视觉差异破坏感知模型迁移", + "kind": "problem", + "tier": "problem", + "topic": "simulator", + "phase": "core", + "year": 2022, + "card": "problem_sim_visual_gap.md", + "summary_zh": "传统游戏引擎渲染与真实相机噪声、镜头光晕与全局光照差距过大,使在仿真上预训练的感知模型在真车上性能显著下降。", + "degree": 3 + }, + { + "id": "insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners", + "label": "Closed-loop is the only ground truth", + "label_zh": "闭环评测是规划器唯一的真值", + "kind": "insight", + "tier": "insight", + "topic": "evaluation_benchmark", + "phase": "frontier", + "year": 2024, + "card": "insight_closed_loop_ground_truth.md", + "summary_zh": "由于规划器的每一步输出都会改变后续观测,离线监督指标无法捕捉滚动决策的反馈结构,只有闭环(或非反应式滚动)评测才能反映其真实驾驶能力。", + "degree": 3 + }, + { + "id": "insight:safety_emerges_from_layered_constraints_not_single_objective", + "label": "Safety as layered constraints", + "label_zh": "安全来自分层约束而非单一目标", + "kind": "insight", + "tier": "insight", + "topic": "safety_standard", + "phase": "frontier", + "year": 2023, + "card": "insight_layered_safety.md", + "summary_zh": "鲁棒的自动驾驶安全不可能通过单一损失函数达成,而是由神经规划、屏蔽层、应急 MPC、监管软件与电子电气冗余共同构成多层约束的涌现性质。", + "degree": 4 + }, + { + "id": "insight:event_driven_computation_matches_natural_sparsity_of_driving_scene", + "label": "Event computation matches scene sparsity", + "label_zh": "事件驱动计算契合驾驶场景的天然稀疏性", + "kind": "insight", + "tier": "insight", + "topic": "neuromorphic_hardware", + "phase": "frontier", + "year": 2023, + "card": "insight_event_driven_sparsity.md", + "summary_zh": "驾驶场景中大多数像素与时间步信息冗余度极高,事件驱动 SNN 与神经形态硬件以稀疏脉冲表征自然匹配这种统计结构,从而以更低能耗换取相同决策质量。", + "degree": 3 + }, + { + "id": "insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency", + "label": "HW-SW co-design unlocks 10x efficiency", + "label_zh": "软硬协同设计释放数量级能效", + "kind": "insight", + "tier": "insight", + "topic": "neuromorphic_hardware", + "phase": "frontier", + "year": 2023, + "card": "insight_hw_sw_codesign.md", + "summary_zh": "单独优化算法或单独优化硬件难以突破当前能效瓶颈,只有把架构搜索、稀疏调度与片上存储拓扑当作联合搜索目标,才能在车规算力下实现 10 倍以上的能效。", + "degree": 3 + }, + { + "id": "insight:data_engine_loop_is_more_valuable_than_static_dataset", + "label": "Data engine > static dataset", + "label_zh": "数据引擎闭环比静态数据集更有价值", + "kind": "insight", + "tier": "insight", + "topic": "data_engine", + "phase": "core", + "year": 2022, + "card": "insight_data_engine.md", + "summary_zh": "真正决定模型上限的是车队-标注-训练-评测的闭环速度,而不是某个固定数据集的规模,谁能让缺陷在数天内回到训练样本谁就拥有持续领先。", + "degree": 6 + }, + { + "id": "insight:simulator_realism_is_lower_bound_on_training_value", + "label": "Simulator realism lower-bounds training value", + "label_zh": "仿真真实度是其训练价值的下界", + "kind": "insight", + "tier": "insight", + "topic": "simulator", + "phase": "frontier", + "year": 2023, + "card": "insight_sim_realism.md", + "summary_zh": "无论行为模型还是视觉渲染,仿真器的真实度直接决定了在其上学得策略的下游可迁移性,提高真实度的边际投入往往比扩大数据规模带来更大的闭环收益。", + "degree": 4 + }, + { + "id": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation", + "label": "Uncertainty calibration enables safe delegation", + "label_zh": "不确定性校准是安全委派的前提", + "kind": "insight", + "tier": "insight", + "topic": "safety_standard", + "phase": "frontier", + "year": 2024, + "card": "insight_uncertainty_calibration.md", + "summary_zh": "只有当模型输出的概率与实际错误率高度一致,才能可靠地决定何时由 AI 决策、何时回退到 MPC 或人类,校准误差因此是任何分层安全架构的隐性瓶颈。", + "degree": 4 + }, + { + "id": "insight:offline_metrics_co_evolve_with_methods_so_must_be_re_audited", + "label": "Offline metrics co-evolve with methods", + "label_zh": "离线指标与方法共同演化故须周期性重审", + "kind": "insight", + "tier": "insight", + "topic": "evaluation_benchmark", + "phase": "frontier", + "year": 2024, + "card": "insight_metric_reaudit.md", + "summary_zh": "随着模型能力提升,旧指标可能逐步被过拟合并失去判别力,团队必须像维护代码一样周期性重审与再设计指标,使其继续与真实驾驶安全保持一致。", + "degree": 2 + }, + { + "id": "paradigm:closed_loop_data_engine_centric_development", + "label": "Closed-loop data engine centric", + "label_zh": "以闭环数据引擎为中心的开发范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "data_engine", + "phase": "frontier", + "year": 2022, + "card": "paradigm_data_engine.md", + "summary_zh": "把组织能力组织在车队回传、自动标注、定向训练与闭环回归四个环节的迭代速度上,使每条新发现的失败模式都在数天内变成训练信号与回归用例。", + "degree": 8 + }, + { + "id": "paradigm:safety_by_constraint_layered_architecture", + "label": "Layered safety-by-constraint", + "label_zh": "以分层约束实现安全的范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "safety_standard", + "phase": "frontier", + "year": 2023, + "card": "paradigm_layered_safety.md", + "summary_zh": "把神经规划、屏蔽层、应急控制、监管软件与电子电气冗余视为同一安全论证下的协同层次,每层都用可独立验证的约束承担一部分残余风险。", + "degree": 5 + }, + { + "id": "paradigm:brain_inspired_neuromorphic_co_design", + "label": "Brain-inspired neuromorphic co-design", + "label_zh": "类脑神经形态软硬协同的范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "neuromorphic_hardware", + "phase": "frontier", + "year": 2024, + "card": "paradigm_brain_inspired.md", + "summary_zh": "以事件驱动稀疏计算、脉冲表征与软硬协同搜索为主轴,把感知-决策管线从基于稠密 GPU 的范式重写为可承载未来 L4/L5 算力预算的新范式。", + "degree": 7 + }, + { + "id": "paradigm:simulator_first_synthetic_data_centric", + "label": "Simulator-first synthetic-data centric", + "label_zh": "以仿真与合成数据为先的范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "simulator", + "phase": "frontier", + "year": 2024, + "card": "paradigm_sim_first.md", + "summary_zh": "把可重建、可扰动、可控参数的合成场景作为训练与评测的一等公民,使长尾问题可以在不依赖事故数据采集的前提下被系统性研究与回归。", + "degree": 7 + }, + { + "id": "paper:lift_splat_shoot", + "label_zh": "Lift-Splat-Shoot(LSS:相机到鸟瞰图的可微提升)", + "kind": "paper", + "tier": "S", + "topic": "geometry_3d", + "phase": "core", + "year": 2020, + "summary_zh": "LSS 提出了一种把多路相机图像投射到鸟瞰图坐标系的可微方法:对每个像素同时预测一个语义特征向量和一个深度概率分布,然后用相机内外参把像素特征按深度概率撒到三维体素里,再压扁成鸟瞰特征图。它第一次把基于相机的鸟瞰感知做成端到端可学习的统一管线,成为后续几乎所有基于相机的鸟瞰感知方法的几何先验。", + "label": "Lift-Splat-Shoot", + "degree": 10 + }, + { + "id": "paper:detr3d", + "label_zh": "DETR3D(基于稀疏查询的多视角三维检测)", + "kind": "paper", + "tier": "S", + "topic": "geometry_3d", + "phase": "core", + "year": 2021, + "summary_zh": "DETR3D 把二维检测中的 DETR 范式直接搬到多视角三维检测:用一组可学习的三维参考点作为目标查询,把每个三维查询点反投影到所有相机平面上去采样图像特征,再用 Transformer 解码器迭代更新目标位置。它绕开了显式的鸟瞰特征构建,证明了稀疏查询本身就是一种隐式的三维到二维对齐机制。", + "label": "DETR3D", + "degree": 10 + }, + { + "id": "paper:petr", + "label_zh": "PETR(位置编码即三维感知)", + "kind": "paper", + "tier": "S", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "PETR 提出在二维图像特征上叠加一个由相机射线生成的三维位置编码,让每个二维像素特征隐式携带它所属相机射线的几何信息。这样目标查询只需做一次普通的注意力就能完成跨视角的三维定位,把多视角三维检测的几何建模问题转换成了纯粹的位置编码工程。", + "label": "PETR", + "degree": 7 + }, + { + "id": "paper:petrv2", + "label_zh": "PETRv2(带时序位置编码的多任务三维感知)", + "kind": "paper", + "tier": "A", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "PETRv2 在 PETR 的三维位置编码上增加了时间维度,把前一帧的相机射线按自车运动对齐到当前坐标系,使得跨时刻的图像特征共享一致的几何参考。它还首次在同一套查询机制上同时承载三维检测和鸟瞰分割两类任务,奠定了多任务共享查询的设计模板。", + "label": "PETRv2", + "degree": 3 + }, + { + "id": "paper:bevdet", + "label_zh": "BEVDet(鸟瞰图检测的标准管线)", + "kind": "paper", + "tier": "A", + "topic": "geometry_3d", + "phase": "core", + "year": 2021, + "summary_zh": "BEVDet 把 Lift-Splat-Shoot 的视角变换、鸟瞰特征编码与基于密集卷积的检测头组装成一条工程化的标准管线,并系统地研究了图像分辨率、视角变换分辨率、鸟瞰分辨率、数据增广之间的耦合。它使得基于相机的鸟瞰检测从研究原型变成可大规模复现、可工程优化的基线。", + "label": "BEVDet", + "degree": 8 + }, + { + "id": "paper:bevdet4d", + "label_zh": "BEVDet4D(时序鸟瞰特征融合)", + "kind": "paper", + "tier": "A", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "BEVDet4D 在 BEVDet 之上引入跨帧鸟瞰特征对齐,把上一帧鸟瞰特征按自车运动平移到当前帧的参考系,然后简单拼接给后续头部。它把一直困扰相机方案的速度估计问题从依赖图像光流的复杂方案简化为鸟瞰特征上的时间差分,极大改善了与点云方案的速度差距。", + "label": "BEVDet4D", + "degree": 5 + }, + { + "id": "paper:bevfusion", + "label_zh": "BEVFusion(鸟瞰特征空间的多模态融合)", + "kind": "paper", + "tier": "S", + "topic": "sensor_fusion", + "phase": "core", + "year": 2022, + "summary_zh": "BEVFusion 把相机分支的鸟瞰特征和点云分支的鸟瞰特征在同一个鸟瞰栅格上对齐后再融合,而不是在感知头处晚融合或在图像-点云层面早融合。这种统一的中间表示使得任一模态失效时另一模态仍可独立工作,并且为后续在鸟瞰空间做规划、预测打开了模块解耦的接口。", + "label": "BEVFusion", + "degree": 10 + }, + { + "id": "paper:bevformer_v2", + "label_zh": "BEVFormer v2(带透视监督的鸟瞰图 Transformer)", + "kind": "paper", + "tier": "A", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "BEVFormer v2 给 BEVFormer 增加了一个额外的透视视角检测头作为辅助监督,强迫鸟瞰特征同时携带与原始图像可对齐的语义。这种\"双视角监督\"思路有效缓解了鸟瞰特征空洞、二维表观信息流失的问题,也为引入图像域预训练大模型铺平了道路。", + "label": "BEVFormer v2", + "degree": 3 + }, + { + "id": "paper:occupancy_networks_tesla", + "label_zh": "Tesla 占用网络(占用栅格替代检测框)", + "kind": "paper", + "tier": "S", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2022, + "summary_zh": "Tesla 在 AI Day 公开的占用网络把场景表示从\"对每个已知类别画三维框\"换成\"对空间每一个体素预测是否被占用以及流速\",从而天然地处理未知类别物体和不规则形状。这一表示让感知不再依赖固定的类别集合,对应了开放世界自动驾驶的核心需求。", + "label": "Tesla Occupancy Networks", + "degree": 8 + }, + { + "id": "paper:surroundocc", + "label_zh": "SurroundOcc(多相机三维占用预测)", + "kind": "paper", + "tier": "A", + "topic": "scene_understanding", + "phase": "core", + "year": 2023, + "summary_zh": "SurroundOcc 把多相机图像融合到一个完整的三维体素空间,预测每个体素的占用与语义类别,并通过多尺度交叉注意力让低分辨率的占用预测引导高分辨率的细节。它是把 Tesla 占用网络思路开源化、学术基准化的代表性工作,让占用预测进入公开的研究比赛。", + "label": "SurroundOcc", + "degree": 7 + }, + { + "id": "paper:occ3d", + "label_zh": "Occ3D(三维占用基准)", + "kind": "paper", + "tier": "A", + "topic": "scene_understanding", + "phase": "core", + "year": 2023, + "summary_zh": "Occ3D 在 nuScenes 等数据集上构建了一致的三维占用真值生成管线,把累积多帧点云、可见性掩码、类别标签统一成体素化标准。它的真正贡献在于让\"占用预测\"作为一个独立任务有了可比较的基准,研究者从此能在同一坐标系下评估不同表示方案的优劣。", + "label": "Occ3D / Occ3D-nuScenes", + "degree": 4 + }, + { + "id": "paper:simplebev", + "label_zh": "SimpleBEV(去掉深度估计的鸟瞰感知基线)", + "kind": "paper", + "tier": "B", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "SimpleBEV 用最朴素的\"对每条相机射线均匀采样三维点然后聚合特征\"代替显式的深度概率分布,得到的鸟瞰感知性能竟然能逼近复杂的 LSS 体系。它充当了一个去伪存真的对照实验:揭示了真正起作用的是体素聚合策略而不是深度网络本身。", + "label": "SimpleBEV", + "degree": 2 + }, + { + "id": "paper:streampetr", + "label_zh": "StreamPETR(流式查询的长时序三维感知)", + "kind": "paper", + "tier": "A", + "topic": "geometry_3d", + "phase": "core", + "year": 2023, + "summary_zh": "StreamPETR 把 PETR 的目标查询做成跨帧持续传播的隐状态:每个查询不仅做当前帧的检测,还把更新后的状态传到下一帧继续推理。它把时序融合从\"对齐特征图\"上升到\"对齐对象级抽象\",让相机方案在长时间窗口下的检测稳定性接近基于点云的方案。", + "label": "StreamPETR", + "degree": 9 + }, + { + "id": "paper:mae", + "label_zh": "MAE(掩码图像建模)", + "kind": "paper", + "tier": "S", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2021, + "summary_zh": "MAE 把 BERT 的掩码语言建模迁移到图像上,遮掉百分之七十五的图像块,让编码器只看可见块,再用一个轻量解码器重建被遮像素。它最重要的发现是\"高比例遮挡是必要的\",这使得自监督预训练在视觉上第一次能像在文本上那样稳定地扩展到大模型与大数据。", + "label": "MAE", + "degree": 6 + }, + { + "id": "paper:beit", + "label_zh": "BEiT(图像版 BERT 预训练)", + "kind": "paper", + "tier": "A", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2021, + "summary_zh": "BEiT 先用一个离散视觉分词器把图像块映射成视觉词元,再用掩码视觉词元预测来训练编码器。它把语言模型的\"离散 token + 掩码预测\"范式整体复制到图像上,为后续多模态统一架构提供了\"视觉也可以离散化\"的基础工具。", + "label": "BEiT", + "degree": 3 + }, + { + "id": "paper:clip", + "label_zh": "CLIP(图文对比预训练)", + "kind": "paper", + "tier": "S", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2021, + "summary_zh": "CLIP 用四亿对网络爬取的图文对做对比学习,让图像编码器和文本编码器把语义对齐到同一空间。它最深远的影响不是分类精度,而是证明了\"用自然语言作为零样本类别接口\"这个范式可以替代固定类别表,后续所有开放词表感知和视觉-语言驱动方法都建立在这一对齐之上。", + "label": "CLIP", + "degree": 7 + }, + { + "id": "paper:blip2", + "label_zh": "BLIP-2(冻结视觉与语言基座的轻量桥接)", + "kind": "paper", + "tier": "A", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2023, + "summary_zh": "BLIP-2 提出 Q-Former 这个小型可训练桥接模块,让冻结的视觉编码器与冻结的大语言模型之间只需训练极少参数即可对齐。它确立了\"把昂贵的预训练基座当成不可改的事实,只在中间加可学习适配器\"这一极具工程价值的多模态对接范式。", + "label": "BLIP-2", + "degree": 4 + }, + { + "id": "paper:vilt", + "label_zh": "ViLT(无卷积的极简视觉语言 Transformer)", + "kind": "paper", + "tier": "B", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2021, + "summary_zh": "ViLT 去掉了图像目标检测器和卷积主干,直接把图像块和文本词元拼接送进一个 Transformer 学习视觉-语言对齐。它的极简主义证明了模态融合本身比模态特有的归纳偏置更关键,是后来多模态大模型简化结构的早期先声。", + "label": "ViLT", + "degree": 1 + }, + { + "id": "paper:nerf", + "label_zh": "NeRF(神经辐射场)", + "kind": "paper", + "tier": "S", + "topic": "scene_understanding", + "phase": "prereq", + "year": 2020, + "summary_zh": "NeRF 用一个多层感知机把三维空间坐标和观察方向映射为颜色与体密度,再用体渲染积分合成新视角图像。它把整个静态场景压缩进神经网络权重,开创了\"用隐式函数代替显式三维表示\"的全新流派,并直接催生了驾驶场景重建与仿真的神经渲染分支。", + "label": "NeRF", + "degree": 11 + }, + { + "id": "paper:3dgs", + "label_zh": "三维高斯泼溅(3D Gaussian Splatting)", + "kind": "paper", + "tier": "S", + "topic": "scene_understanding", + "phase": "core", + "year": 2023, + "summary_zh": "三维高斯泼溅用一组带位置、协方差、颜色与不透明度的各向异性高斯点来显式表示场景,再用可微的栅格化器投射到屏幕。它在保持神经辐射场可微优化能力的同时,把渲染速度提升了一两个数量级,迅速取代纯 MLP 表示成为驾驶场景重建的新基础设施。", + "label": "3D Gaussian Splatting", + "degree": 8 + }, + { + "id": "paper:emernerf", + "label_zh": "EmerNeRF(驾驶场景的自监督动静解耦神经辐射场)", + "kind": "paper", + "tier": "A", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2023, + "summary_zh": "EmerNeRF 通过把场景分解为静态背景流和时变动态流两支辐射场,并加入光流自监督,让驾驶场景的重建可以在没有人工标注动态物体的情况下自动分离动静。它把神经辐射场从\"重建一段静态片段\"推进到\"理解一段含动态对象的驾驶序列\"。", + "label": "EmerNeRF", + "degree": 8 + }, + { + "id": "paper:drivinggaussian", + "label_zh": "DrivingGaussian(动态驾驶场景的高斯重建)", + "kind": "paper", + "tier": "A", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2024, + "summary_zh": "DrivingGaussian 把三维高斯泼溅扩展到带有大量运动车辆与行人的城市驾驶场景:用静态高斯捕获背景、用以物体为中心的动态高斯捕获每个移动目标,再统一渲染。它让大规模、可编辑、可仿真的驾驶场景数字孪生成为现实。", + "label": "DrivingGaussian", + "degree": 8 + }, + { + "id": "paper:dinov1", + "label_zh": "DINO(自蒸馏视觉自监督)", + "kind": "paper", + "tier": "A", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2021, + "summary_zh": "DINO 让一个学生网络匹配教师网络对同一图像不同视图的输出分布,教师权重是学生的指数滑动平均,无需负样本即可避免坍缩。它意外地展示了 ViT 在纯自监督下会自动浮现出物体分割的注意力图,启示了\"无标签即可学到结构\"这一持续推动 SSL 路线的核心现象。", + "label": "DINO (self-distillation)", + "degree": 5 + }, + { + "id": "paper:simclr_mocov3", + "label_zh": "SimCLR / MoCo v3(对比学习视觉预训练)", + "kind": "paper", + "tier": "B", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2020, + "summary_zh": "SimCLR 与后续的 MoCo v3 用同一张图的不同数据增广作为正对、其他图作为负对做对比学习,把表示空间塑造为语义近则近、语义远则远。它们确立了对比学习作为视觉自监督的主流方案,也暴露了对负样本数与批大小的依赖,为后续无负样本的方法(如 DINO)提供了反面参照。", + "label": "SimCLR / MoCo v3", + "degree": 3 + }, + { + "id": "paper:depth_anything", + "label_zh": "Depth Anything(通用单目深度大模型)", + "kind": "paper", + "tier": "A", + "topic": "geometry_3d", + "phase": "frontier", + "year": 2024, + "summary_zh": "Depth Anything 用约六千万张未标注图像做半监督蒸馏,让一个 ViT 主干学会几乎在任意场景给出鲁棒的单目相对深度。它使\"深度\"这一原本需要昂贵设备或多视角几何的几何量,变成了一个像图像分类一样可现成调用的通用先验,显著降低了下游三维感知的入门门槛。", + "label": "Depth Anything", + "degree": 6 + }, + { + "id": "paper:vggt", + "label_zh": "VGGT(前馈三维几何 Transformer)", + "kind": "paper", + "tier": "B", + "topic": "geometry_3d", + "phase": "frontier", + "year": 2025, + "summary_zh": "VGGT 用一个 Transformer 在一次前向中同时输出相机内外参、深度图、点云和稠密对应,完全绕开传统的逐对匹配与捆绑调整。它把\"结构从运动\"这一经典计算机视觉问题彻底重写为大模型预训练问题,为基于视频的快速三维重建提供了新的工程上限。", + "label": "VGGT", + "degree": 3 + }, + { + "id": "paper:openocc_unic", + "label_zh": "OpenOccupancy 与 UniOcc(开放占用与统一占用基准)", + "kind": "paper", + "tier": "B", + "topic": "scene_understanding", + "phase": "core", + "year": 2023, + "summary_zh": "OpenOccupancy 与 UniOcc 联合提供了大规模、长尾、含未知类别的占用基准,强调评估在\"语义未知但空间存在\"情境下的鲁棒性。它们把占用预测的研究焦点从\"对已知类的分类精度\"推向\"对世界结构本身的覆盖率\",反向影响了后续模型架构的设计取向。", + "label": "OpenOccupancy / UniOcc", + "degree": 1 + }, + { + "id": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "label_zh": "把二维特征按学习到的深度分布提升到三维", + "kind": "move", + "tier": "move", + "topic": "geometry_3d", + "phase": "core", + "year": 2020, + "summary_zh": "不再尝试为每个像素回归单一深度,而是预测一个离散深度分布,把图像特征按概率\"撒\"到对应的三维体素位置。这样视角变换变得可微分,模型可以在端到端训练中自己决定特征该聚到哪个深度,绕过了深度估计不可靠时整条管线崩溃的问题。其代表性应用是 LSS、BEVDet 系列,并被 BEVFusion 借去做相机分支。", + "label": "Lift 2D features to 3D via learned depth distribution", + "degree": 8 + }, + { + "id": "move:treat_detection_as_set_prediction_with_learnable_queries", + "label_zh": "把检测任务转化为学习查询集合的预测问题", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "core", + "year": 2020, + "summary_zh": "用一组固定数量的可学习查询向量代替密集锚框,每个查询通过注意力机制竞争性地\"认领\"一个目标,最后由匈牙利匹配与真值对齐。这一移动消除了非极大值抑制等手工后处理,使得检测器变成纯可微管线,并把每个查询天然变成下游任务可继承的对象级抽象。其最早形态是 DETR,被 DETR3D 推向三维,被 UniAD 进一步当成统一规划的接口。", + "label": "Treat detection as set prediction with learnable queries", + "degree": 8 + }, + { + "id": "move:reproject_3d_query_to_2d_for_feature_sampling", + "label_zh": "把三维查询点反投影到二维图像采样特征", + "kind": "move", + "tier": "move", + "topic": "geometry_3d", + "phase": "core", + "year": 2021, + "summary_zh": "不构建显式的鸟瞰特征图,而是让每个三维查询点利用相机内外参反投影到各路相机上,直接采样原图特征。这把\"特征对齐\"从工程化的视角变换转化为简单的几何投影,省掉了对体素分辨率的依赖,也让多视角融合天然处理不同相机的几何关系。代表为 DETR3D、StreamPETR 系列。", + "label": "Reproject 3D query to 2D for feature sampling", + "degree": 3 + }, + { + "id": "move:embed_camera_geometry_into_positional_encoding", + "label_zh": "把相机几何信息直接编码进位置编码", + "kind": "move", + "tier": "move", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "把每个像素所属的相机射线起点和方向参数化后加到二维特征的位置编码中,让每个二维特征自带三维上下文。注意力机制就无需显式投影即可学到正确的几何对齐。这一移动彻底简化了多视角三维检测的结构,在 PETR 上首次工作,后被几乎所有\"几何不显式\"的方案采用。", + "label": "Embed camera geometry into positional encoding", + "degree": 4 + }, + { + "id": "move:replace_explicit_module_with_implicit_function", + "label_zh": "用隐式函数替换显式模块", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "core", + "year": 2020, + "summary_zh": "把场景、形状、占用等原本以显式离散结构存储的量,替换为以连续坐标为输入的神经函数。这样表示自然连续、可微、可任意精细化采样,并能由可微渲染监督训练。这一移动的源头是 NeRF,扩散到 SDF 表面表示、占用场表示、神经隐式定位等,几乎重塑了几何视觉的下游栈。", + "label": "Replace explicit module with implicit function", + "degree": 3 + }, + { + "id": "move:swap_implicit_for_explicit_primitives_when_compute_allows", + "label_zh": "当算力允许时,用显式基元换回隐式表示", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2023, + "summary_zh": "在保持渲染过程可微的前提下,把神经网络隐式表示换成大量显式基元(高斯点、点云、网格片)。这种反向移动牺牲一些泛化性,换来数量级的渲染加速与可编辑性。三维高斯泼溅是范式样本,提醒研究者\"隐式\"和\"显式\"是连续光谱上的两个极端而非二元对立。", + "label": "Swap implicit for explicit primitives when compute allows", + "degree": 4 + }, + { + "id": "move:add_auxiliary_perspective_supervision_to_bev", + "label_zh": "为鸟瞰特征增加透视视角的辅助监督", + "kind": "move", + "tier": "move", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "在鸟瞰特征上额外挂一个透视视角的检测/分割头作为副任务,迫使鸟瞰特征不丢失原始图像里能直接观察到的语义。它解决了纯鸟瞰监督下特征容易\"塌缩\"为只关心俯视框的问题,也是接入大规模图像预训练模型的天然入口。BEVFormer v2 是典范应用。", + "label": "Add auxiliary perspective supervision to BEV", + "degree": 2 + }, + { + "id": "move:carry_object_query_across_time_as_recurrent_state", + "label_zh": "把对象查询作为循环状态跨帧传递", + "kind": "move", + "tier": "move", + "topic": "geometry_3d", + "phase": "core", + "year": 2023, + "summary_zh": "不再在每一帧重新生成查询,而是把上一帧更新后的查询连同其隐藏状态传到当前帧作为初始状态。这种循环式查询机制把时序融合从\"对齐特征图\"升级为\"对齐对象级抽象\",天然带来 ID 一致性和长时序稳定性,典型如 StreamPETR、Sparse4D 系列。", + "label": "Carry object query across time as recurrent state", + "degree": 6 + }, + { + "id": "move:fuse_modalities_in_shared_intermediate_space", + "label_zh": "在共享中间表示空间中融合多种模态", + "kind": "move", + "tier": "move", + "topic": "sensor_fusion", + "phase": "core", + "year": 2022, + "summary_zh": "为相机、点云、雷达等设计各自的编码器,把它们各自的输出都投影到同一中间表示(如鸟瞰栅格、占用体素)后再融合。相较于早融合或晚融合,中间融合既保留模态特有的归纳偏置,又使单模态失效时其他模态可独立工作,显著提升鲁棒性。BEVFusion 是范例。", + "label": "Fuse modalities in shared intermediate space", + "degree": 4 + }, + { + "id": "move:replace_class_specific_box_with_class_agnostic_occupancy", + "label_zh": "用与类别无关的占用代替按类别画框", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2022, + "summary_zh": "放弃\"先认出类别再画框\"的范式,改为对空间每个体素预测它是否被占用以及流速。这样未知类别、不规则形状、累积的小障碍物都能被统一表示。这一移动是 Tesla 占用网络的核心思想,也被随后所有\"通用障碍物检测\"研究采用。", + "label": "Replace class-specific box with class-agnostic occupancy", + "degree": 7 + }, + { + "id": "move:scale_pretraining_then_fine_tune_with_minimal_labels", + "label_zh": "扩规模做自监督预训练,再用少量标签微调", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "core", + "year": 2021, + "summary_zh": "把大量未标注数据投喂给一个自监督目标,先得到一个通用视觉表征,再用少量任务相关标签做轻量下游微调。这一移动把\"数据标注\"从瓶颈变为可选项,使得自动驾驶研究可以受益于互联网级数据。MAE、DINOv2、DINOv3 等都是这一移动在不同自监督信号下的实例。", + "label": "Scale self-supervised pretraining, then fine-tune with minimal labels", + "degree": 6 + }, + { + "id": "move:freeze_giant_backbone_train_small_adapter", + "label_zh": "冻结大型主干,只训练小型适配器", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "core", + "year": 2022, + "summary_zh": "把昂贵的预训练大模型权重冻结,只在中间插入轻量级可训练桥接(如 Q-Former、LoRA 适配器、线性投影)。这样既保留基座知识,又把训练成本压到能在小集群上完成,使学术界也能复现工业级多模态系统。BLIP-2 是该移动的典型实现。", + "label": "Freeze giant backbone, train small adapter", + "degree": 3 + }, + { + "id": "move:tokenize_continuous_signal_to_use_transformer", + "label_zh": "把连续信号离散化以套用 Transformer", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "prereq", + "year": 2020, + "summary_zh": "把图像切块、把点云分体素、把动作分位段——总之先把连续模态人为切成有限符号集,然后整个 Transformer 训练栈(掩码预测、对比学习、自回归)都能直接复用。这一移动是 ViT、VQ-VAE、BEiT、各类世界模型共同的入场卷。", + "label": "Tokenize continuous signal to use a Transformer", + "degree": 2 + }, + { + "id": "move:use_geometry_as_self_supervision", + "label_zh": "用几何一致性作为免费的自监督信号", + "kind": "move", + "tier": "move", + "topic": "geometry_3d", + "phase": "core", + "year": 2017, + "summary_zh": "多视角一致、时序一致、立体一致、光度一致——这些几何约束在采集数据时自动成立,无需人工标注就可以作为损失项。可用于自监督深度、流、位姿、辐射场。NeRF 的体渲染监督、单目深度的光度损失、EmerNeRF 的光流监督都建立在这一移动上。", + "label": "Use geometric consistency as a free self-supervision signal", + "degree": 4 + }, + { + "id": "move:make_pipeline_differentiable_via_shared_latent", + "label_zh": "通过共享隐表示使整条管线可微", + "kind": "move", + "tier": "move", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "summary_zh": "把\"感知-预测-规划\"等本来用规则连接的模块,改用共享的隐表示(查询、鸟瞰特征、占用体素)做接口,让梯度从最后的任务损失一直反传到原始图像。这是 UniAD、VAD 等端到端方案的方法论根基,也使得\"上游模块为下游目标服务\"这件事第一次能被训练数据自动达成。", + "label": "Make pipeline differentiable via shared latent representation", + "degree": 3 + }, + { + "id": "move:rasterize_differentiable_renderer_for_inverse_problem", + "label_zh": "用可微渲染器反演成像过程", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "core", + "year": 2020, + "summary_zh": "把渲染过程写成对场景参数可微的算子,然后把\"已知图像、求场景\"的反问题变成普通梯度下降。NeRF 的体积分、三维高斯泼溅的栅格化都是这一移动的实例,使得任意场景表示形式只要配上可微渲染器就能从图像直接监督。", + "label": "Wrap a differentiable renderer to invert image formation", + "degree": 4 + }, + { + "id": "move:distill_internet_data_into_small_specialist", + "label_zh": "把网络规模数据蒸馏进领域专家模型", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "frontier", + "year": 2024, + "summary_zh": "先用海量未标注互联网数据训练一个能力宽泛的教师模型,再用伪标签把其能力蒸馏到一个针对具体任务/数据分布的较小学生。Depth Anything、SAM 的训练管线都体现了这一移动:把互联网视觉数据\"打包\"成可被下游直接使用的归纳偏置。", + "label": "Distill from web-scale data into a specialist model", + "degree": 4 + }, + { + "id": "move:make_camera_only_temporal_match_lidar", + "label_zh": "用时序聚合让纯相机方案逼近激光雷达", + "kind": "move", + "tier": "move", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "把多个时刻的相机鸟瞰特征/查询对齐叠加,等价地累积了视差与运动信息,从而在静态深度和速度估计上逼近激光雷达。这一移动让\"去激光雷达\"从口号变成可量化的工程目标,代表为 BEVDet4D、SOLOFusion、StreamPETR。", + "label": "Make camera-only with temporal aggregation match LiDAR", + "degree": 6 + }, + { + "id": "move:open_vocabulary_via_text_alignment", + "label_zh": "通过图文对齐实现开放词表识别", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "core", + "year": 2021, + "summary_zh": "把分类头换成\"和文本嵌入做内积\",于是任何能写成自然语言的概念都能即时变成新类别。这把感知的类别集从\"固定 80 类\"变成\"语言空间任意子集\",对自动驾驶里频发的长尾物体尤其重要。CLIP 是该移动的奠基者,所有开放词表检测/分割都建立在其上。", + "label": "Open vocabulary via image-text alignment", + "degree": 3 + }, + { + "id": "move:emergent_segmentation_from_self_distillation", + "label_zh": "让分割能力在自蒸馏中自然涌现", + "kind": "move", + "tier": "move", + "topic": "ssl_vision", + "phase": "core", + "year": 2021, + "summary_zh": "不显式训练任何分割损失,只让一个 ViT 学生模仿教师在不同视图下的输出分布,结果注意力图自动浮现出物体级别的分割。这一移动揭示\"结构归纳偏置 + 强自监督\"足以让显式监督本不该出现的能力涌现,改变了\"先标注后训练\"的研究次序。DINO 是源头。", + "label": "Let segmentation emerge from self-distillation", + "degree": 2 + }, + { + "id": "move:replace_handcrafted_sfm_with_feedforward_transformer", + "label_zh": "用前馈 Transformer 取代手工的多视图几何流水线", + "kind": "move", + "tier": "move", + "topic": "geometry_3d", + "phase": "frontier", + "year": 2025, + "summary_zh": "把\"特征匹配-本质矩阵-捆绑调整\"这套四十年累积的几何工程,整体替换为一个直接吃多张图像、输出相机参数和稠密三维结构的 Transformer。这一移动延续了\"苦涩教训\"的判断:大模型加规模终将赢过精雕细琢的几何方法。VGGT 是其代表。", + "label": "Replace handcrafted SfM with a feed-forward Transformer", + "degree": 2 + }, + { + "id": "move:decompose_scene_into_static_and_dynamic_streams", + "label_zh": "把场景显式分解为静态流与动态流", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "core", + "year": 2023, + "summary_zh": "用两套独立但联合渲染的表示——一套表示固定背景,一套表示时变前景——再让它们相互约束。这一移动让无需人工标注就能自动分离动态对象,是把神经辐射场扩展到真实驾驶场景的必要步骤。EmerNeRF、DrivingGaussian 都使用了它的变体。", + "label": "Decompose scene into static and dynamic streams", + "degree": 3 + }, + { + "id": "move:bridge_sim_and_real_via_neural_reconstruction", + "label_zh": "用对真实日志的神经重建桥接仿真与现实", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2023, + "summary_zh": "不再从零搭仿真世界,而是把真实路采视频重建成可重新渲染、可摆放对象、可改变天气的数字孪生,再在其中插入新场景做策略训练。这一移动用神经重建打通了\"采集-标注-训练-评估\"闭环,是当前最被工业界看重的仿真新路线。DrivingGaussian 等是入口。", + "label": "Bridge sim and real via neural reconstruction of real logs", + "degree": 5 + }, + { + "id": "move:augment_via_counterfactual_object_insertion", + "label_zh": "通过反事实物体插入扩充数据", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2024, + "summary_zh": "在已有真实序列里插入并不存在的物体(被神经渲染或图像生成模型逼真地合成),得到大量\"几乎真实但永远不会被采集到\"的边缘案例。这一移动让长尾问题第一次有了规模化的解,是当前神经重建管线最具产品价值的副产物。", + "label": "Augment data via counterfactual object insertion", + "degree": 3 + }, + { + "id": "move:share_queries_across_multiple_tasks", + "label_zh": "在多个下游任务之间共享同一组查询", + "kind": "move", + "tier": "move", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "summary_zh": "让检测、跟踪、地图、占用、规划等任务复用同一组对象级查询,把多任务联合训练变成天然的隐表示对齐器。这一移动是 UniAD\"以规划为目的统一感知\"的方法论核心,也是 PETRv2 等多任务方案的设计范式。", + "label": "Share queries across multiple downstream tasks", + "degree": 4 + }, + { + "id": "move:learn_motion_in_latent_space_then_decode", + "label_zh": "在隐空间预测运动后再解码到像素", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2023, + "summary_zh": "不直接在像素级预测下一帧,而是先把图像压成隐令牌,然后在令牌空间预测未来,最后只在需要可视化时才解码回像素。这把高维像素预测的难题转化为低维语义预测,是世界模型方案在驾驶场景可工作的关键工程基石。", + "label": "Learn motion in latent space then decode to pixels", + "degree": 3 + }, + { + "id": "move:use_visibility_mask_to_filter_supervision", + "label_zh": "用可见性掩码过滤占用学习的监督信号", + "kind": "move", + "tier": "move", + "topic": "scene_understanding", + "phase": "core", + "year": 2023, + "summary_zh": "在体素化激光雷达累积的占用真值上叠加可见性掩码,只在确实被传感器观测到的体素上计算损失,避免模型被\"遮挡背面\"的伪真值污染。这一看似工程的细节其实决定了占用学习能否泛化,是 Occ3D 等基准能稳定收敛的关键。", + "label": "Use visibility masks to filter supervision in occupancy learning", + "degree": 1 + }, + { + "id": "problem:long_tail_object_categories_in_open_world", + "label_zh": "开放世界中长尾物体类别问题", + "kind": "problem", + "tier": "problem", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2022, + "summary_zh": "真实道路上不断出现训练集没见过的物体:翻倒的椅子、散落的轮胎、奇怪形状的工程车。任何依赖固定类别表的检测器都会对其失效。开放词表识别、占用预测、世界模型蒸馏都是对该问题的不同尝试,但至今没有一个方案能在所有维度都做得好。", + "label": "Long-tail object categories in open-world driving", + "degree": 5 + }, + { + "id": "problem:sim_to_real_gap_in_camera_only_perception", + "label_zh": "纯相机感知的仿真到现实差距", + "kind": "problem", + "tier": "problem", + "topic": "geometry_3d", + "phase": "core", + "year": 2018, + "summary_zh": "由 CARLA 等仿真器渲染出的图像与真实相机图像在光照、噪声、镜头畸变、运动模糊上的分布差异,使得在仿真里训得好的纯相机感知模型一到真实路况就大幅退化。神经重建仿真、域随机化、对抗增广都是减弱这一差距的方向,但仍未根除。", + "label": "Sim-to-real gap in camera-only perception", + "degree": 1 + }, + { + "id": "problem:temporal_consistency_in_bev_segmentation", + "label_zh": "鸟瞰图分割的时序一致性问题", + "kind": "problem", + "tier": "problem", + "topic": "geometry_3d", + "phase": "core", + "year": 2021, + "summary_zh": "逐帧独立的鸟瞰分割常常出现物体边界跳动、车道线闪烁,严重影响下游规划。如何在保持低延迟的同时让鸟瞰输出在时间上平滑、ID 稳定,是一个工程上反复出现却很难有原则性解的痛点,目前主要靠循环查询、ID 追踪损失等弱方案。", + "label": "Temporal consistency in BEV segmentation", + "degree": 1 + }, + { + "id": "problem:occlusion_reasoning_without_dense_lidar", + "label_zh": "没有稠密激光雷达时的遮挡推理", + "kind": "problem", + "tier": "problem", + "topic": "scene_understanding", + "phase": "core", + "year": 2020, + "summary_zh": "纯相机方案无法直接观察被遮挡区域,但安全驾驶又必须对\"可能藏着行人的盲区\"做出合理假设。占用预测、生成式世界模型、几何先验都试图回答这一点,但缺乏一种既准确又可校准不确定性的方法。", + "label": "Occlusion reasoning without dense LiDAR", + "degree": 3 + }, + { + "id": "problem:label_efficiency_for_3d_annotation", + "label_zh": "三维标注的标签效率问题", + "kind": "problem", + "tier": "problem", + "topic": "scene_understanding", + "phase": "core", + "year": 2019, + "summary_zh": "标注三维框、占用、轨迹的人工成本远高于二维。自监督预训练、自动标注管线、神经重建辅助标注都尝试缩减成本,但目前业界仍依赖大量人工质检,这是开放数据集规模扩展和长尾覆盖的最大障碍之一。", + "label": "Label efficiency for 3D annotation", + "degree": 3 + }, + { + "id": "problem:unknown_geometry_in_distant_or_dark_regions", + "label_zh": "远距离或低光区域的几何未知问题", + "kind": "problem", + "tier": "problem", + "topic": "geometry_3d", + "phase": "frontier", + "year": 2021, + "summary_zh": "图像在远距离失去分辨率、在低光下失去对比度,传统几何方法直接失效。学习方法虽能\"猜测\"几何,但难以量化置信度。这是夜间、隧道、雨雾等场景中所有纯视觉方案共同的根本困难。", + "label": "Unknown geometry in distant or dark regions", + "degree": 1 + }, + { + "id": "problem:multi_modal_calibration_drift", + "label_zh": "多模态传感器外参漂移问题", + "kind": "problem", + "tier": "problem", + "topic": "sensor_fusion", + "phase": "core", + "year": 2018, + "summary_zh": "相机、激光雷达、毫米波雷达之间的外参在车辆使用过程中会缓慢漂移,使中间表示空间里的特征对不齐,融合性能急剧下降。在线自标定、可微外参学习都是回应,但鲁棒性仍未达到工业级要求。", + "label": "Multi-modal sensor calibration drift", + "degree": 1 + }, + { + "id": "problem:rendering_speed_vs_quality_tradeoff", + "label_zh": "神经重建中渲染速度与质量的权衡", + "kind": "problem", + "tier": "problem", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2022, + "summary_zh": "纯神经辐射场质量高但渲染慢,三维高斯泼溅渲染快但难以表达半透明、反射等复杂材质,且在动态场景中颗粒感明显。如何在保持训练-推理时间合理的前提下覆盖驾驶场景的所有材质与天气,仍是开放问题。", + "label": "Rendering speed versus quality trade-off in neural reconstruction", + "degree": 2 + }, + { + "id": "problem:catastrophic_failure_on_rare_weather", + "label_zh": "罕见天气与光照下的灾难性失效", + "kind": "problem", + "tier": "problem", + "topic": "geometry_3d", + "phase": "core", + "year": 2019, + "summary_zh": "雨、雪、大雾、夕阳逆光在自然数据集中比例极低,使得感知模型在这些条件下的表现几乎不可预测。合成数据增广、生成式数据扩充、专项数据集都被尝试,但没有任何方法已经把这一长尾压到产品可接受的水平。", + "label": "Catastrophic failure on rare weather and lighting", + "degree": 2 + }, + { + "id": "problem:annotation_inconsistency_across_datasets", + "label_zh": "跨驾驶数据集的标注不一致问题", + "kind": "problem", + "tier": "problem", + "topic": "scene_understanding", + "phase": "core", + "year": 2020, + "summary_zh": "nuScenes、Waymo Open、Argoverse、KITTI 各自的类别表、坐标系、可见性定义都不一致,使得\"联合训练\"变得困难,模型必须在不同语义之间做痛苦的折中。统一占用类、跨数据集联合学习是部分解,但根本协议仍未统一。", + "label": "Annotation inconsistency across driving datasets", + "degree": 2 + }, + { + "id": "problem:depth_ambiguity_in_low_parallax", + "label_zh": "低视差单目视图的深度歧义问题", + "kind": "problem", + "tier": "problem", + "topic": "geometry_3d", + "phase": "core", + "year": 2017, + "summary_zh": "前向单目相机的视差几乎为零,导致深度尺度无法从几何唯一确定,任何深度预测都依赖学习到的先验。这一根本歧义解释了为什么\"完全去掉激光雷达\"在前向场景中始终困难,促使了多相机环视、毫米波辅助、时间累积等多种间接解。", + "label": "Depth ambiguity in low-parallax monocular views", + "degree": 2 + }, + { + "id": "insight:multi_view_geometry_as_free_supervision", + "label_zh": "多视图几何即是一种免费的监督信号", + "kind": "insight", + "tier": "insight", + "topic": "geometry_3d", + "phase": "core", + "year": 2017, + "summary_zh": "只要采集时具备多视角或多时刻,几何一致性(光度、视差、流、深度)就免费提供大量自监督信号,无需任何人工标注。这一观察跨越深度估计、辐射场、自监督表征,是把\"采集数据\"和\"标注数据\"解耦的根本支点。", + "label": "Multi-view geometry is a free supervision signal", + "degree": 4 + }, + { + "id": "insight:foundation_features_transfer_without_finetune", + "label_zh": "基础模型的特征通常无需微调即可迁移", + "kind": "insight", + "tier": "insight", + "topic": "ssl_vision", + "phase": "core", + "year": 2023, + "summary_zh": "大规模自监督预训练得到的特征常常在下游任务上线性分类即可达到强基线,甚至冻结后即可直接使用。这一现象使\"训练通用基座 + 冻结特征 + 轻量适配\"成为成本-性能最佳折中,是当前感知系统设计的主导原则之一。", + "label": "Foundation model features often transfer without finetuning", + "degree": 5 + }, + { + "id": "insight:occupancy_unifies_static_and_dynamic_scene", + "label_zh": "占用场可以统一表示静态与动态场景", + "kind": "insight", + "tier": "insight", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2022, + "summary_zh": "把场景表示成\"每个体素的占用 + 流速\"自然涵盖了刚体、可变形体、未知类别,既能描述静态街景又能刻画动态参与者。这一统一性使得感知输出可以直接服务于规划、预测、神经渲染,是占用范式相对检测框范式的最深结构性优势。", + "label": "Occupancy fields unify static and dynamic scene representation", + "degree": 3 + }, + { + "id": "insight:open_vocabulary_via_language_anchoring", + "label_zh": "通过语言锚定实现开放词表感知", + "kind": "insight", + "tier": "insight", + "topic": "ssl_vision", + "phase": "core", + "year": 2021, + "summary_zh": "把\"类别\"这个固定离散符号替换为\"自然语言描述\",感知模型的可识别概念集合就可以随时随地扩展。这一思路统一了零样本检测、分割、属性识别,并把感知接口和大语言模型天然连通。", + "label": "Open vocabulary perception via language anchoring", + "degree": 3 + }, + { + "id": "insight:implicit_vs_explicit_is_a_continuum", + "label_zh": "隐式与显式表示是一个连续光谱", + "kind": "insight", + "tier": "insight", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2023, + "summary_zh": "纯隐式(MLP)和纯显式(点、网格)只是连续光谱的两端,中间有体素哈希、各向异性高斯、张量分解等无穷多种混合形式。研究者应根据具体任务对可微性、可编辑性、渲染速度、表达力的需求,在该光谱上选择合适的折中,而不是把两者当作二元选择。", + "label": "Implicit versus explicit representation is a continuum", + "degree": 3 + }, + { + "id": "insight:bev_is_planning_friendly_intermediate", + "label_zh": "鸟瞰图是对规划最友好的中间表示", + "kind": "insight", + "tier": "insight", + "topic": "e2e_ad", + "phase": "core", + "year": 2022, + "summary_zh": "鸟瞰图保留了道路布局、可通行区域、对象位置的真实尺度,同时去掉了透视下的纵深扭曲,使得后续路径采样、代价场计算都可以在欧氏空间中直接进行。它之所以在自动驾驶中流行不是因为感知最准,而是因为它是接口最适配规划的中间表示。", + "label": "BEV is the planning-friendly intermediate representation", + "degree": 7 + }, + { + "id": "insight:temporal_aggregation_buys_what_depth_sensor_buys", + "label_zh": "时序聚合能换取深度传感器所能换取的", + "kind": "insight", + "tier": "insight", + "topic": "geometry_3d", + "phase": "core", + "year": 2022, + "summary_zh": "运动带来的视差等价于一个虚拟的立体相机基线,因此\"看更多帧\"在很大程度上替代了\"装一个激光雷达\"。这一等价性是纯相机方案能与激光雷达方案在同一基准上比较的根本前提。", + "label": "Temporal aggregation buys what a depth sensor would have bought", + "degree": 5 + }, + { + "id": "insight:differentiable_rendering_is_universal_inverse_solver", + "label_zh": "可微渲染是一种通用的反问题求解器", + "kind": "insight", + "tier": "insight", + "topic": "scene_understanding", + "phase": "core", + "year": 2020, + "summary_zh": "凡是能写成\"场景 → 图像\"前向过程的问题,只要把这个过程做成可微,就可以通过反向传播从图像反推场景参数。这一观点把传统计算机视觉里许多孤立的反问题(深度、形状、光照、相机)统一进同一个框架,极大地拓展了可学习视觉的边界。", + "label": "Differentiable rendering is a universal inverse-problem solver", + "degree": 6 + }, + { + "id": "paradigm:camera_first_autonomy", + "label_zh": "相机优先的自动驾驶范式", + "kind": "paradigm", + "tier": "paradigm", + "topic": "geometry_3d", + "phase": "core", + "year": 2021, + "summary_zh": "相机优先范式认为:得益于深度学习,纯相机方案在感知能力上正在逼近激光雷达,且成本和可扩展性占优。它驱动了 BEV 系列、占用预测、神经重建等一连串研究浪潮,与\"必须保留激光雷达\"的传统稳健派形成长期张力。", + "label": "Camera-first autonomy paradigm", + "degree": 5 + }, + { + "id": "paradigm:neural_scene_reconstruction_as_engine", + "label_zh": "把神经场景重建当作仿真引擎", + "kind": "paradigm", + "tier": "paradigm", + "topic": "scene_understanding", + "phase": "frontier", + "year": 2023, + "summary_zh": "这一范式主张:与其手工建模仿真世界,不如把真实驾驶日志神经重建成可重新渲染、可编辑的数字孪生,然后在其中扩增、对抗、闭环。它把\"采集-训练-评估\"三件事统一进同一套表示,正成为头部公司投入最大的方向之一。", + "label": "Neural scene reconstruction as the simulation engine", + "degree": 5 + } + ], + "edges": [ + { + "source": "paper:vaswani2017", + "target": "paper:vit", + "rel": "prereq" + }, + { + "source": "paper:vaswani2017", + "target": "paper:carion2020", + "rel": "prereq" + }, + { + "source": "paper:vaswani2017", + "target": "paper:gpt3", + "rel": "prereq" + }, + { + "source": "paper:vaswani2017", + "target": "paper:2307.01694", + "rel": "prereq" + }, + { + "source": "paper:vit", + "target": "paper:dinov2", + "rel": "prereq" + }, + { + "source": "paper:vit", + "target": "paper:sam", + "rel": "prereq" + }, + { + "source": "paper:vit", + "target": "paper:li2022bevformer", + "rel": "prereq" + }, + { + "source": "paper:vit", + "target": "paper:llava", + "rel": "prereq" + }, + { + "source": "paper:vit", + "target": "paper:2307.01694", + "rel": "prereq" + }, + { + "source": "paper:dinov2", + "target": "paper:2508.10104", + "rel": "prereq" + }, + { + "source": "paper:carion2020", + "target": "paper:li2022bevformer", + "rel": "prereq" + }, + { + "source": "paper:carion2020", + "target": "paper:2212.10156", + "rel": "prereq" + }, + { + "source": "paper:carion2020", + "target": "paper:2210.14222", + "rel": "prereq" + }, + { + "source": "paper:li2022bevformer", + "target": "paper:2212.10156", + "rel": "prereq" + }, + { + "source": "paper:he2015_resnet", + "target": "paper:vit", + "rel": "prereq" + }, + { + "source": "paper:he2015_resnet", + "target": "paper:2307.01694", + "rel": "prereq" + }, + { + "source": "paper:gpt3", + "target": "paper:llava", + "rel": "prereq" + }, + { + "source": "paper:gpt3", + "target": "paper:rlhf_dpo", + "rel": "prereq" + }, + { + "source": "paper:gpt3", + "target": "paper:2309.16292", + "rel": "prereq" + }, + { + "source": "paper:gpt3", + "target": "paper:2311.10813", + "rel": "prereq" + }, + { + "source": "paper:llava", + "target": "paper:2402.12289", + "rel": "prereq" + }, + { + "source": "paper:llava", + "target": "paper:2512.24426", + "rel": "prereq" + }, + { + "source": "course:zhao_rl", + "target": "course:cs285", + "rel": "prereq" + }, + { + "source": "course:zhao_rl", + "target": "paper:mnih2015_dqn", + "rel": "covers" + }, + { + "source": "course:zhao_rl", + "target": "paper:schulman2017_ppo", + "rel": "covers" + }, + { + "source": "course:zhao_rl", + "target": "concept:mdp", + "rel": "covers" + }, + { + "source": "course:zhao_rl", + "target": "concept:bellman_eq", + "rel": "covers" + }, + { + "source": "course:zhao_rl", + "target": "concept:value_iteration", + "rel": "covers" + }, + { + "source": "course:zhao_rl", + "target": "concept:td_learning", + "rel": "covers" + }, + { + "source": "course:zhao_rl", + "target": "concept:policy_gradient", + "rel": "covers" + }, + { + "source": "course:zhao_rl", + "target": "concept:actor_critic", + "rel": "covers" + }, + { + "source": "course:cs285", + "target": "concept:imitation_learning", + "rel": "covers" + }, + { + "source": "course:cs285", + "target": "concept:covariate_shift", + "rel": "covers" + }, + { + "source": "course:cs285", + "target": "concept:dqn", + "rel": "covers" + }, + { + "source": "course:cs285", + "target": "concept:ppo", + "rel": "covers" + }, + { + "source": "course:cs285", + "target": "concept:replay_buffer", + "rel": "covers" + }, + { + "source": "course:cs285", + "target": "paper:ross2011_dagger", + "rel": "covers" + }, + { + "source": "course:cs285", + "target": "paper:rlhf_dpo", + "rel": "covers" + }, + { + "source": "course:cs285", + "target": "paper:world_models", + "rel": "covers" + }, + { + "source": "paper:sutton_barto", + "target": "course:zhao_rl", + "rel": "parallel" + }, + { + "source": "paper:sutton_barto", + "target": "essay:bitter_lesson", + "rel": "parallel" + }, + { + "source": "channel:3blue1brown", + "target": "concept:transformer", + "rel": "covers" + }, + { + "source": "channel:3blue1brown", + "target": "concept:self_attention", + "rel": "covers" + }, + { + "source": "channel:3blue1brown", + "target": "paper:vaswani2017", + "rel": "covers" + }, + { + "source": "channel:mu_li_bilibili", + "target": "paper:vaswani2017", + "rel": "covers" + }, + { + "source": "channel:mu_li_bilibili", + "target": "paper:vit", + "rel": "covers" + }, + { + "source": "channel:mu_li_bilibili", + "target": "paper:carion2020", + "rel": "covers" + }, + { + "source": "channel:mu_li_bilibili", + "target": "paper:gpt3", + "rel": "covers" + }, + { + "source": "channel:mu_li_bilibili", + "target": "paper:dinov2", + "rel": "covers" + }, + { + "source": "channel:mu_li_bilibili", + "target": "paper:he2015_resnet", + "rel": "covers" + }, + { + "source": "channel:ez_encoder_academy", + "target": "paper:gpt3", + "rel": "covers" + }, + { + "source": "channel:ez_encoder_academy", + "target": "essay:bitter_lesson", + "rel": "covers" + }, + { + "source": "essay:bitter_lesson", + "target": "paper:silver2017_alphazero", + "rel": "covers" + }, + { + "source": "essay:bitter_lesson", + "target": "paper:2309.16292", + "rel": "contrasts" + }, + { + "source": "essay:bitter_lesson", + "target": "paper:2311.10813", + "rel": "contrasts" + }, + { + "source": "essay:bitter_lesson", + "target": "paper:2210.14222", + "rel": "contrasts" + }, + { + "source": "essay:bitter_lesson", + "target": "paper:2307.01694", + "rel": "contrasts" + }, + { + "source": "essay:bitter_lesson", + "target": "paper:2508.10104", + "rel": "covers" + }, + { + "source": "essay:bitter_lesson", + "target": "paper:gpt3", + "rel": "covers" + }, + { + "source": "paper:2212.10156", + "target": "paper:vadv2", + "rel": "extends" + }, + { + "source": "paper:2212.10156", + "target": "paper:2210.14222", + "rel": "contrasts" + }, + { + "source": "paper:2212.10156", + "target": "paper:2402.12289", + "rel": "feeds" + }, + { + "source": "paper:2210.14222", + "target": "paper:transfuser", + "rel": "parallel" + }, + { + "source": "paper:2210.14222", + "target": "paper:2311.10813", + "rel": "feeds" + }, + { + "source": "paper:2402.12289", + "target": "paper:2512.24426", + "rel": "extends" + }, + { + "source": "paper:2311.10813", + "target": "paper:2402.12289", + "rel": "extends" + }, + { + "source": "paper:2311.10813", + "target": "paper:2309.16292", + "rel": "parallel" + }, + { + "source": "paper:2309.16292", + "target": "paper:2311.10813", + "rel": "parallel" + }, + { + "source": "paper:2508.10104", + "target": "paper:2212.10156", + "rel": "feeds" + }, + { + "source": "paper:2508.10104", + "target": "paper:2402.12289", + "rel": "feeds" + }, + { + "source": "paper:2307.01694", + "target": "paper:2508.10104", + "rel": "contrasts" + }, + { + "source": "paper:gaia1", + "target": "paper:2512.24426", + "rel": "parallel" + }, + { + "source": "paper:drivedreamer", + "target": "paper:gaia1", + "rel": "parallel" + }, + { + "source": "paper:world_models", + "target": "paper:gaia1", + "rel": "prereq" + }, + { + "source": "paper:world_models", + "target": "paper:drivedreamer", + "rel": "prereq" + }, + { + "source": "paper:world_models", + "target": "paper:2512.24426", + "rel": "prereq" + }, + { + "source": "paper:rlhf_dpo", + "target": "paper:2512.24426", + "rel": "prereq" + }, + { + "source": "paper:transfuser", + "target": "paper:2210.14222", + "rel": "feeds" + }, + { + "source": "paper:vadv2", + "target": "paper:2402.12289", + "rel": "parallel" + }, + { + "source": "paper:mamba", + "target": "paper:vaswani2017", + "rel": "contrasts" + }, + { + "source": "paper:diffuser", + "target": "paper:vadv2", + "rel": "parallel" + }, + { + "source": "paper:lingo2", + "target": "paper:2402.12289", + "rel": "parallel" + }, + { + "source": "paper:tesla_ai_day", + "target": "paper:2212.10156", + "rel": "parallel" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:2212.10156", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:2402.12289", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:2210.14222", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:2311.10813", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:2309.16292", + "rel": "covers" + }, + { + "source": "paper:mnih2015_dqn", + "target": "concept:dqn", + "rel": "covers" + }, + { + "source": "paper:mnih2015_dqn", + "target": "concept:replay_buffer", + "rel": "covers" + }, + { + "source": "paper:schulman2017_ppo", + "target": "concept:ppo", + "rel": "covers" + }, + { + "source": "paper:schulman2017_ppo", + "target": "concept:actor_critic", + "rel": "covers" + }, + { + "source": "paper:ross2011_dagger", + "target": "concept:imitation_learning", + "rel": "covers" + }, + { + "source": "paper:ross2011_dagger", + "target": "concept:covariate_shift", + "rel": "covers" + }, + { + "source": "paper:vaswani2017", + "target": "concept:transformer", + "rel": "covers" + }, + { + "source": "paper:vaswani2017", + "target": "concept:self_attention", + "rel": "covers" + }, + { + "source": "paper:carion2020", + "target": "concept:detr_query", + "rel": "covers" + }, + { + "source": "paper:li2022bevformer", + "target": "concept:bev", + "rel": "covers" + }, + { + "source": "paper:llava", + "target": "concept:vlm", + "rel": "covers" + }, + { + "source": "paper:2402.12289", + "target": "concept:vla", + "rel": "covers" + }, + { + "source": "paper:2402.12289", + "target": "concept:cot", + "rel": "covers" + }, + { + "source": "paper:2311.10813", + "target": "concept:tool_use", + "rel": "covers" + }, + { + "source": "paper:2309.16292", + "target": "concept:cot", + "rel": "covers" + }, + { + "source": "paper:2512.24426", + "target": "concept:counterfactual", + "rel": "covers" + }, + { + "source": "paper:2512.24426", + "target": "concept:meta_action", + "rel": "covers" + }, + { + "source": "paper:dinov2", + "target": "concept:ssl", + "rel": "covers" + }, + { + "source": "paper:2508.10104", + "target": "concept:ssl", + "rel": "covers" + }, + { + "source": "paper:2307.01694", + "target": "concept:spiking_nn", + "rel": "covers" + }, + { + "source": "paper:rlhf_dpo", + "target": "concept:rlhf", + "rel": "covers" + }, + { + "source": "essay:bitter_lesson", + "target": "concept:scaling_vs_knowledge", + "rel": "covers" + }, + { + "source": "lab:lab01", + "target": "course:zhao_rl", + "rel": "implements" + }, + { + "source": "lab:lab01", + "target": "concept:value_iteration", + "rel": "implements" + }, + { + "source": "lab:lab02", + "target": "course:cs285", + "rel": "implements" + }, + { + "source": "lab:lab02", + "target": "paper:ross2011_dagger", + "rel": "implements" + }, + { + "source": "lab:lab03", + "target": "paper:2212.10156", + "rel": "implements" + }, + { + "source": "lab:lab04", + "target": "paper:2210.14222", + "rel": "implements" + }, + { + "source": "lab:lab05", + "target": "paper:2508.10104", + "rel": "implements" + }, + { + "source": "lab:lab06", + "target": "paper:2307.01694", + "rel": "implements" + }, + { + "source": "lab:lab07", + "target": "paper:2309.16292", + "rel": "implements" + }, + { + "source": "lab:lab08", + "target": "paper:2311.10813", + "rel": "implements" + }, + { + "source": "lab:lab09", + "target": "paper:2402.12289", + "rel": "implements" + }, + { + "source": "lab:lab10", + "target": "paper:2512.24426", + "rel": "implements" + }, + { + "source": "paper:silver2017_alphazero", + "target": "paper:muzero", + "rel": "extends" + }, + { + "source": "paper:world_models", + "target": "paper:dreamer_v2", + "rel": "prereq" + }, + { + "source": "paper:dreamer_v2", + "target": "paper:dreamer_v3", + "rel": "extends" + }, + { + "source": "paper:world_models", + "target": "paper:iris_world_model", + "rel": "prereq" + }, + { + "source": "paper:vaswani2017", + "target": "paper:iris_world_model", + "rel": "prereq" + }, + { + "source": "paper:iris_world_model", + "target": "paper:dreamer_v3", + "rel": "parallel" + }, + { + "source": "paper:muzero", + "target": "paper:dreamer_v3", + "rel": "parallel" + }, + { + "source": "paper:mnih2015_dqn", + "target": "paper:sac", + "rel": "prereq" + }, + { + "source": "paper:mnih2015_dqn", + "target": "paper:td3", + "rel": "prereq" + }, + { + "source": "paper:sac", + "target": "paper:redq", + "rel": "extends" + }, + { + "source": "paper:td3", + "target": "paper:sac", + "rel": "parallel" + }, + { + "source": "paper:mnih2015_dqn", + "target": "paper:a3c_a2c", + "rel": "parallel" + }, + { + "source": "paper:a3c_a2c", + "target": "paper:impala", + "rel": "extends" + }, + { + "source": "paper:schulman2017_ppo", + "target": "paper:openai_five", + "rel": "feeds" + }, + { + "source": "paper:a3c_a2c", + "target": "paper:alphastar", + "rel": "feeds" + }, + { + "source": "paper:sac", + "target": "paper:mpo", + "rel": "parallel" + }, + { + "source": "paper:schulman2017_ppo", + "target": "paper:mpo", + "rel": "contrasts" + }, + { + "source": "paper:gpt3", + "target": "paper:decision_transformer", + "rel": "prereq" + }, + { + "source": "paper:decision_transformer", + "target": "paper:trajectory_transformer", + "rel": "parallel" + }, + { + "source": "paper:vaswani2017", + "target": "paper:trajectory_transformer", + "rel": "prereq" + }, + { + "source": "paper:diffuser", + "target": "paper:diffusion_policy_chi2023", + "rel": "parallel" + }, + { + "source": "paper:diffuser", + "target": "paper:diffusion_planner", + "rel": "extends" + }, + { + "source": "paper:cql", + "target": "paper:iql", + "rel": "extends" + }, + { + "source": "paper:cql", + "target": "paper:calql", + "rel": "extends" + }, + { + "source": "paper:iql", + "target": "paper:calql", + "rel": "parallel" + }, + { + "source": "paper:cpo_safe_rl", + "target": "paper:lagrangian_safe_rl", + "rel": "parallel" + }, + { + "source": "paper:lagrangian_safe_rl", + "target": "paper:shielded_rl", + "rel": "contrasts" + }, + { + "source": "paper:rlhf_dpo", + "target": "paper:pebble", + "rel": "parallel" + }, + { + "source": "paper:pebble", + "target": "paper:bpref", + "rel": "parallel" + }, + { + "source": "paper:rlhf_dpo", + "target": "paper:bpref", + "rel": "parallel" + }, + { + "source": "paper:lqr_classic", + "target": "paper:ilqr_classic", + "rel": "prereq" + }, + { + "source": "paper:ilqr_classic", + "target": "paper:cilqr", + "rel": "extends" + }, + { + "source": "paper:lqr_classic", + "target": "paper:mpc_book", + "rel": "prereq" + }, + { + "source": "paper:mpc_book", + "target": "paper:cilqr", + "rel": "parallel" + }, + { + "source": "paper:transfuser", + "target": "paper:interfuser", + "rel": "parallel" + }, + { + "source": "paper:transfuser", + "target": "paper:roach", + "rel": "parallel" + }, + { + "source": "paper:roach", + "target": "paper:thinktwice", + "rel": "feeds" + }, + { + "source": "paper:interfuser", + "target": "paper:thinktwice", + "rel": "parallel" + }, + { + "source": "paper:world_models", + "target": "paper:mile_driving", + "rel": "prereq" + }, + { + "source": "paper:dreamer_v2", + "target": "paper:mile_driving", + "rel": "parallel" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:nuplan_baselines", + "rel": "covers" + }, + { + "source": "paper:nuplan_baselines", + "target": "paper:interfuser", + "rel": "contrasts" + }, + { + "source": "paper:vaswani2017", + "target": "paper:trajeglish", + "rel": "prereq" + }, + { + "source": "paper:trajeglish", + "target": "paper:most_simagents", + "rel": "parallel" + }, + { + "source": "paper:gpt3", + "target": "paper:codetraj", + "rel": "prereq" + }, + { + "source": "paper:2311.10813", + "target": "paper:codetraj", + "rel": "parallel" + }, + { + "source": "paper:world_models", + "target": "paper:mbrl_pets", + "rel": "parallel" + }, + { + "source": "paper:mbrl_pets", + "target": "paper:dreamer_v2", + "rel": "prereq" + }, + { + "source": "move:learn_world_model_then_plan_in_latent_imagination", + "target": "paper:world_models", + "rel": "manifests" + }, + { + "source": "move:learn_world_model_then_plan_in_latent_imagination", + "target": "paper:dreamer_v2", + "rel": "manifests" + }, + { + "source": "move:learn_world_model_then_plan_in_latent_imagination", + "target": "paper:dreamer_v3", + "rel": "manifests" + }, + { + "source": "move:learn_world_model_then_plan_in_latent_imagination", + "target": "paper:mile_driving", + "rel": "manifests" + }, + { + "source": "move:plan_with_mcts_in_learned_model", + "target": "paper:muzero", + "rel": "manifests" + }, + { + "source": "move:plan_with_mcts_in_learned_model", + "target": "paper:silver2017_alphazero", + "rel": "extends" + }, + { + "source": "move:discrete_latent_state_for_world_model", + "target": "paper:dreamer_v2", + "rel": "manifests" + }, + { + "source": "move:discrete_latent_state_for_world_model", + "target": "paper:dreamer_v3", + "rel": "manifests" + }, + { + "source": "move:tokenize_pixel_frames_for_autoregressive_world_model", + "target": "paper:iris_world_model", + "rel": "manifests" + }, + { + "source": "move:tokenize_pixel_frames_for_autoregressive_world_model", + "target": "paper:gaia1", + "rel": "manifests" + }, + { + "source": "move:tokenize_pixel_frames_for_autoregressive_world_model", + "target": "paper:drivedreamer", + "rel": "manifests" + }, + { + "source": "move:replace_explicit_critic_with_diffusion_score", + "target": "paper:diffusion_policy_chi2023", + "rel": "manifests" + }, + { + "source": "move:replace_explicit_critic_with_diffusion_score", + "target": "paper:diffuser", + "rel": "manifests" + }, + { + "source": "move:bootstrap_target_network_to_stabilize_off_policy_learning", + "target": "paper:mnih2015_dqn", + "rel": "manifests" + }, + { + "source": "move:bootstrap_target_network_to_stabilize_off_policy_learning", + "target": "paper:sac", + "rel": "manifests" + }, + { + "source": "move:bootstrap_target_network_to_stabilize_off_policy_learning", + "target": "paper:td3", + "rel": "manifests" + }, + { + "source": "move:add_entropy_bonus_to_encourage_exploration", + "target": "paper:sac", + "rel": "manifests" + }, + { + "source": "move:add_entropy_bonus_to_encourage_exploration", + "target": "paper:schulman2017_ppo", + "rel": "manifests" + }, + { + "source": "move:add_entropy_bonus_to_encourage_exploration", + "target": "paper:a3c_a2c", + "rel": "manifests" + }, + { + "source": "move:turn_offline_dataset_into_supervised_sequence_prediction", + "target": "paper:decision_transformer", + "rel": "manifests" + }, + { + "source": "move:turn_offline_dataset_into_supervised_sequence_prediction", + "target": "paper:trajectory_transformer", + "rel": "manifests" + }, + { + "source": "move:replace_value_function_with_implicit_max_via_expectile", + "target": "paper:iql", + "rel": "manifests" + }, + { + "source": "move:use_pretrained_language_model_as_action_prior", + "target": "paper:codetraj", + "rel": "manifests" + }, + { + "source": "move:use_pretrained_language_model_as_action_prior", + "target": "paper:2311.10813", + "rel": "manifests" + }, + { + "source": "move:use_pretrained_language_model_as_action_prior", + "target": "paper:2309.16292", + "rel": "manifests" + }, + { + "source": "move:add_lagrangian_safety_constraint_to_actor_critic", + "target": "paper:lagrangian_safe_rl", + "rel": "manifests" + }, + { + "source": "move:add_lagrangian_safety_constraint_to_actor_critic", + "target": "paper:cpo_safe_rl", + "rel": "manifests" + }, + { + "source": "move:treat_planning_as_conditional_generation", + "target": "paper:diffuser", + "rel": "manifests" + }, + { + "source": "move:treat_planning_as_conditional_generation", + "target": "paper:diffusion_policy_chi2023", + "rel": "manifests" + }, + { + "source": "move:treat_planning_as_conditional_generation", + "target": "paper:diffusion_planner", + "rel": "manifests" + }, + { + "source": "move:treat_planning_as_conditional_generation", + "target": "paper:vadv2", + "rel": "manifests" + }, + { + "source": "move:cast_continuous_action_as_discretized_token_sequence", + "target": "paper:trajectory_transformer", + "rel": "manifests" + }, + { + "source": "move:cast_continuous_action_as_discretized_token_sequence", + "target": "paper:trajeglish", + "rel": "manifests" + }, + { + "source": "move:cast_continuous_action_as_discretized_token_sequence", + "target": "paper:most_simagents", + "rel": "manifests" + }, + { + "source": "move:use_n_step_returns_to_trade_bias_for_variance", + "target": "paper:mnih2015_dqn", + "rel": "enables" + }, + { + "source": "move:use_n_step_returns_to_trade_bias_for_variance", + "target": "paper:schulman2017_ppo", + "rel": "enables" + }, + { + "source": "move:add_intrinsic_motivation_via_novelty_or_curiosity", + "target": "paper:pebble", + "rel": "enables" + }, + { + "source": "move:apply_gae_to_smooth_advantage_estimation", + "target": "paper:schulman2017_ppo", + "rel": "manifests" + }, + { + "source": "move:apply_gae_to_smooth_advantage_estimation", + "target": "paper:a3c_a2c", + "rel": "manifests" + }, + { + "source": "move:apply_gae_to_smooth_advantage_estimation", + "target": "paper:openai_five", + "rel": "manifests" + }, + { + "source": "move:use_prioritized_replay_buffer", + "target": "paper:mnih2015_dqn", + "rel": "enables" + }, + { + "source": "move:cotrain_dynamics_model_with_policy_to_share_representations", + "target": "paper:mile_driving", + "rel": "manifests" + }, + { + "source": "move:cotrain_dynamics_model_with_policy_to_share_representations", + "target": "paper:dreamer_v2", + "rel": "manifests" + }, + { + "source": "move:warm_start_rl_with_imitation_then_anneal", + "target": "paper:alphastar", + "rel": "manifests" + }, + { + "source": "move:warm_start_rl_with_imitation_then_anneal", + "target": "paper:roach", + "rel": "manifests" + }, + { + "source": "move:warm_start_rl_with_imitation_then_anneal", + "target": "paper:calql", + "rel": "manifests" + }, + { + "source": "move:double_q_to_reduce_overestimation", + "target": "paper:td3", + "rel": "manifests" + }, + { + "source": "move:double_q_to_reduce_overestimation", + "target": "paper:sac", + "rel": "manifests" + }, + { + "source": "move:double_q_to_reduce_overestimation", + "target": "paper:redq", + "rel": "extends" + }, + { + "source": "move:expert_iteration_self_distillation", + "target": "paper:silver2017_alphazero", + "rel": "manifests" + }, + { + "source": "move:expert_iteration_self_distillation", + "target": "paper:muzero", + "rel": "manifests" + }, + { + "source": "move:distill_privileged_teacher_to_sensor_student", + "target": "paper:roach", + "rel": "manifests" + }, + { + "source": "move:distill_privileged_teacher_to_sensor_student", + "target": "paper:transfuser", + "rel": "manifests" + }, + { + "source": "move:trust_region_step_for_monotonic_improvement", + "target": "paper:schulman2017_ppo", + "rel": "manifests" + }, + { + "source": "move:trust_region_step_for_monotonic_improvement", + "target": "paper:cpo_safe_rl", + "rel": "manifests" + }, + { + "source": "move:trust_region_step_for_monotonic_improvement", + "target": "paper:mpo", + "rel": "manifests" + }, + { + "source": "move:expectile_or_quantile_target_for_distributional_robustness", + "target": "paper:iql", + "rel": "manifests" + }, + { + "source": "move:hindsight_experience_relabeling", + "target": "paper:sac", + "rel": "enables" + }, + { + "source": "move:safety_shield_filters_unsafe_actions", + "target": "paper:shielded_rl", + "rel": "manifests" + }, + { + "source": "move:reward_model_from_pairwise_human_preferences", + "target": "paper:rlhf_dpo", + "rel": "manifests" + }, + { + "source": "move:reward_model_from_pairwise_human_preferences", + "target": "paper:pebble", + "rel": "manifests" + }, + { + "source": "move:reward_model_from_pairwise_human_preferences", + "target": "paper:bpref", + "rel": "manifests" + }, + { + "source": "move:guided_sampling_through_classifier_gradients_at_inference", + "target": "paper:diffuser", + "rel": "manifests" + }, + { + "source": "move:guided_sampling_through_classifier_gradients_at_inference", + "target": "paper:diffusion_planner", + "rel": "manifests" + }, + { + "source": "move:plan_via_cross_entropy_method_on_dynamics_model", + "target": "paper:mbrl_pets", + "rel": "manifests" + }, + { + "source": "move:plan_via_cross_entropy_method_on_dynamics_model", + "target": "paper:muzero", + "rel": "parallel" + }, + { + "source": "move:two_stage_coarse_to_fine_trajectory", + "target": "paper:thinktwice", + "rel": "manifests" + }, + { + "source": "move:two_stage_coarse_to_fine_trajectory", + "target": "paper:vadv2", + "rel": "manifests" + }, + { + "source": "move:league_play_for_policy_diversity", + "target": "paper:alphastar", + "rel": "manifests" + }, + { + "source": "move:learn_world_model_then_plan_in_latent_imagination", + "target": "move:plan_with_mcts_in_learned_model", + "rel": "composes" + }, + { + "source": "move:discrete_latent_state_for_world_model", + "target": "move:learn_world_model_then_plan_in_latent_imagination", + "rel": "composes" + }, + { + "source": "move:tokenize_pixel_frames_for_autoregressive_world_model", + "target": "move:learn_world_model_then_plan_in_latent_imagination", + "rel": "composes" + }, + { + "source": "move:cast_continuous_action_as_discretized_token_sequence", + "target": "move:turn_offline_dataset_into_supervised_sequence_prediction", + "rel": "composes" + }, + { + "source": "move:bootstrap_target_network_to_stabilize_off_policy_learning", + "target": "move:double_q_to_reduce_overestimation", + "rel": "composes" + }, + { + "source": "move:add_entropy_bonus_to_encourage_exploration", + "target": "move:double_q_to_reduce_overestimation", + "rel": "composes" + }, + { + "source": "move:trust_region_step_for_monotonic_improvement", + "target": "move:add_lagrangian_safety_constraint_to_actor_critic", + "rel": "composes" + }, + { + "source": "move:treat_planning_as_conditional_generation", + "target": "move:guided_sampling_through_classifier_gradients_at_inference", + "rel": "composes" + }, + { + "source": "move:use_pretrained_language_model_as_action_prior", + "target": "move:treat_planning_as_conditional_generation", + "rel": "composes" + }, + { + "source": "move:reward_model_from_pairwise_human_preferences", + "target": "move:warm_start_rl_with_imitation_then_anneal", + "rel": "composes" + }, + { + "source": "move:distill_privileged_teacher_to_sensor_student", + "target": "move:warm_start_rl_with_imitation_then_anneal", + "rel": "composes" + }, + { + "source": "problem:behavior_cloning_compounds_errors_over_time", + "target": "paper:ross2011_dagger", + "rel": "motivates" + }, + { + "source": "problem:behavior_cloning_compounds_errors_over_time", + "target": "paper:roach", + "rel": "motivates" + }, + { + "source": "problem:behavior_cloning_compounds_errors_over_time", + "target": "paper:transfuser", + "rel": "motivates" + }, + { + "source": "problem:distributional_shift_between_offline_data_and_deployment", + "target": "paper:cql", + "rel": "motivates" + }, + { + "source": "problem:distributional_shift_between_offline_data_and_deployment", + "target": "paper:iql", + "rel": "motivates" + }, + { + "source": "problem:distributional_shift_between_offline_data_and_deployment", + "target": "paper:calql", + "rel": "motivates" + }, + { + "source": "problem:distributional_shift_between_offline_data_and_deployment", + "target": "paper:decision_transformer", + "rel": "motivates" + }, + { + "source": "problem:long_horizon_credit_assignment_in_driving", + "target": "paper:muzero", + "rel": "motivates" + }, + { + "source": "problem:long_horizon_credit_assignment_in_driving", + "target": "paper:diffuser", + "rel": "motivates" + }, + { + "source": "problem:reward_specification_for_safe_polite_driving", + "target": "paper:rlhf_dpo", + "rel": "motivates" + }, + { + "source": "problem:reward_specification_for_safe_polite_driving", + "target": "paper:pebble", + "rel": "motivates" + }, + { + "source": "problem:exploration_in_safety_critical_systems", + "target": "paper:cpo_safe_rl", + "rel": "motivates" + }, + { + "source": "problem:exploration_in_safety_critical_systems", + "target": "paper:lagrangian_safe_rl", + "rel": "motivates" + }, + { + "source": "problem:exploration_in_safety_critical_systems", + "target": "paper:shielded_rl", + "rel": "motivates" + }, + { + "source": "problem:closed_loop_simulation_fidelity_gap", + "target": "paper:trajeglish", + "rel": "motivates" + }, + { + "source": "problem:closed_loop_simulation_fidelity_gap", + "target": "paper:most_simagents", + "rel": "motivates" + }, + { + "source": "problem:closed_loop_simulation_fidelity_gap", + "target": "paper:gaia1", + "rel": "motivates" + }, + { + "source": "problem:closed_loop_simulation_fidelity_gap", + "target": "paper:drivedreamer", + "rel": "motivates" + }, + { + "source": "problem:multi_agent_interaction_modeling_in_dense_traffic", + "target": "paper:trajeglish", + "rel": "motivates" + }, + { + "source": "problem:multi_agent_interaction_modeling_in_dense_traffic", + "target": "paper:nuplan_baselines", + "rel": "motivates" + }, + { + "source": "problem:planning_horizon_vs_compute_budget_tradeoff", + "target": "paper:cilqr", + "rel": "motivates" + }, + { + "source": "problem:planning_horizon_vs_compute_budget_tradeoff", + "target": "paper:mpc_book", + "rel": "motivates" + }, + { + "source": "problem:rare_event_evaluation_with_no_ground_truth", + "target": "paper:nuplan_baselines", + "rel": "motivates" + }, + { + "source": "problem:rare_event_evaluation_with_no_ground_truth", + "target": "paper:ad_benchmarks", + "rel": "motivates" + }, + { + "source": "problem:reward_hacking_in_learned_objectives", + "target": "paper:bpref", + "rel": "motivates" + }, + { + "source": "problem:reward_hacking_in_learned_objectives", + "target": "paper:rlhf_dpo", + "rel": "motivates" + }, + { + "source": "problem:behavior_cloning_compounds_errors_over_time", + "target": "insight:imitation_learning_alone_cannot_recover_from_compounding_errors", + "rel": "manifests" + }, + { + "source": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", + "target": "paper:world_models", + "rel": "manifests" + }, + { + "source": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", + "target": "paper:dreamer_v3", + "rel": "manifests" + }, + { + "source": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", + "target": "paper:muzero", + "rel": "manifests" + }, + { + "source": "insight:human_demonstrations_compress_implicit_reward_function", + "target": "paper:diffusion_policy_chi2023", + "rel": "manifests" + }, + { + "source": "insight:human_demonstrations_compress_implicit_reward_function", + "target": "paper:ross2011_dagger", + "rel": "manifests" + }, + { + "source": "insight:safety_emerges_from_constraint_lagrangian_not_reward_shaping", + "target": "paper:cpo_safe_rl", + "rel": "manifests" + }, + { + "source": "insight:safety_emerges_from_constraint_lagrangian_not_reward_shaping", + "target": "paper:lagrangian_safe_rl", + "rel": "manifests" + }, + { + "source": "insight:offline_rl_is_actually_constrained_dynamic_programming", + "target": "paper:cql", + "rel": "manifests" + }, + { + "source": "insight:offline_rl_is_actually_constrained_dynamic_programming", + "target": "paper:iql", + "rel": "manifests" + }, + { + "source": "insight:offline_rl_is_actually_constrained_dynamic_programming", + "target": "paper:calql", + "rel": "manifests" + }, + { + "source": "insight:tokenized_trajectories_let_planning_borrow_from_language_modeling", + "target": "paper:decision_transformer", + "rel": "manifests" + }, + { + "source": "insight:tokenized_trajectories_let_planning_borrow_from_language_modeling", + "target": "paper:trajectory_transformer", + "rel": "manifests" + }, + { + "source": "insight:tokenized_trajectories_let_planning_borrow_from_language_modeling", + "target": "paper:trajeglish", + "rel": "manifests" + }, + { + "source": "insight:bigger_model_plus_more_data_beats_clever_priors", + "target": "paper:openai_five", + "rel": "manifests" + }, + { + "source": "insight:bigger_model_plus_more_data_beats_clever_priors", + "target": "paper:alphastar", + "rel": "manifests" + }, + { + "source": "insight:bigger_model_plus_more_data_beats_clever_priors", + "target": "essay:bitter_lesson", + "rel": "extends" + }, + { + "source": "insight:control_theory_and_rl_meet_in_optimal_control", + "target": "paper:lqr_classic", + "rel": "manifests" + }, + { + "source": "insight:control_theory_and_rl_meet_in_optimal_control", + "target": "paper:mpc_book", + "rel": "manifests" + }, + { + "source": "insight:control_theory_and_rl_meet_in_optimal_control", + "target": "concept:bellman_eq", + "rel": "manifests" + }, + { + "source": "paradigm:model_based_rl", + "target": "paper:world_models", + "rel": "manifests" + }, + { + "source": "paradigm:model_based_rl", + "target": "paper:dreamer_v2", + "rel": "manifests" + }, + { + "source": "paradigm:model_based_rl", + "target": "paper:dreamer_v3", + "rel": "manifests" + }, + { + "source": "paradigm:model_based_rl", + "target": "paper:muzero", + "rel": "manifests" + }, + { + "source": "paradigm:model_based_rl", + "target": "paper:iris_world_model", + "rel": "manifests" + }, + { + "source": "paradigm:model_based_rl", + "target": "paper:mbrl_pets", + "rel": "manifests" + }, + { + "source": "paradigm:model_free_rl", + "target": "paper:mnih2015_dqn", + "rel": "manifests" + }, + { + "source": "paradigm:model_free_rl", + "target": "paper:schulman2017_ppo", + "rel": "manifests" + }, + { + "source": "paradigm:model_free_rl", + "target": "paper:sac", + "rel": "manifests" + }, + { + "source": "paradigm:model_free_rl", + "target": "paper:td3", + "rel": "manifests" + }, + { + "source": "paradigm:model_free_rl", + "target": "paper:a3c_a2c", + "rel": "manifests" + }, + { + "source": "paradigm:model_free_rl", + "target": "paper:impala", + "rel": "manifests" + }, + { + "source": "paradigm:offline_rl", + "target": "paper:cql", + "rel": "manifests" + }, + { + "source": "paradigm:offline_rl", + "target": "paper:iql", + "rel": "manifests" + }, + { + "source": "paradigm:offline_rl", + "target": "paper:calql", + "rel": "manifests" + }, + { + "source": "paradigm:offline_rl", + "target": "paper:decision_transformer", + "rel": "manifests" + }, + { + "source": "paradigm:imitation_learning", + "target": "paper:ross2011_dagger", + "rel": "manifests" + }, + { + "source": "paradigm:imitation_learning", + "target": "paper:diffusion_policy_chi2023", + "rel": "manifests" + }, + { + "source": "paradigm:imitation_learning", + "target": "paper:transfuser", + "rel": "manifests" + }, + { + "source": "paradigm:imitation_learning", + "target": "paper:interfuser", + "rel": "manifests" + }, + { + "source": "paradigm:optimal_control", + "target": "paper:lqr_classic", + "rel": "manifests" + }, + { + "source": "paradigm:optimal_control", + "target": "paper:ilqr_classic", + "rel": "manifests" + }, + { + "source": "paradigm:optimal_control", + "target": "paper:cilqr", + "rel": "manifests" + }, + { + "source": "paradigm:optimal_control", + "target": "paper:mpc_book", + "rel": "manifests" + }, + { + "source": "paradigm:safe_rl", + "target": "paper:cpo_safe_rl", + "rel": "manifests" + }, + { + "source": "paradigm:safe_rl", + "target": "paper:lagrangian_safe_rl", + "rel": "manifests" + }, + { + "source": "paradigm:safe_rl", + "target": "paper:shielded_rl", + "rel": "manifests" + }, + { + "source": "paradigm:sequence_modeling_for_decision", + "target": "paper:decision_transformer", + "rel": "manifests" + }, + { + "source": "paradigm:sequence_modeling_for_decision", + "target": "paper:trajectory_transformer", + "rel": "manifests" + }, + { + "source": "paradigm:sequence_modeling_for_decision", + "target": "paper:trajeglish", + "rel": "manifests" + }, + { + "source": "paradigm:sequence_modeling_for_decision", + "target": "paper:most_simagents", + "rel": "manifests" + }, + { + "source": "paradigm:sequence_modeling_for_decision", + "target": "paper:codetraj", + "rel": "manifests" + }, + { + "source": "paradigm:model_free_rl", + "target": "paradigm:model_based_rl", + "rel": "contrasts" + }, + { + "source": "paradigm:imitation_learning", + "target": "paradigm:model_free_rl", + "rel": "contrasts" + }, + { + "source": "paradigm:offline_rl", + "target": "paradigm:model_free_rl", + "rel": "contrasts" + }, + { + "source": "paradigm:sequence_modeling_for_decision", + "target": "paradigm:model_free_rl", + "rel": "contrasts" + }, + { + "source": "paradigm:optimal_control", + "target": "paradigm:model_free_rl", + "rel": "parallel" + }, + { + "source": "paradigm:safe_rl", + "target": "paradigm:model_free_rl", + "rel": "extends" + }, + { + "source": "problem:reward_hacking_in_learned_objectives", + "target": "insight:safety_emerges_from_constraint_lagrangian_not_reward_shaping", + "rel": "motivates" + }, + { + "source": "problem:long_horizon_credit_assignment_in_driving", + "target": "insight:world_model_as_inner_simulator_unlocks_long_horizon_planning", + "rel": "motivates" + }, + { + "source": "problem:distributional_shift_between_offline_data_and_deployment", + "target": "insight:offline_rl_is_actually_constrained_dynamic_programming", + "rel": "motivates" + }, + { + "source": "concept:mdp", + "target": "paradigm:optimal_control", + "rel": "prereq" + }, + { + "source": "concept:bellman_eq", + "target": "paradigm:model_based_rl", + "rel": "prereq" + }, + { + "source": "concept:bellman_eq", + "target": "paradigm:model_free_rl", + "rel": "prereq" + }, + { + "source": "concept:imitation_learning", + "target": "paradigm:imitation_learning", + "rel": "prereq" + }, + { + "source": "concept:replay_buffer", + "target": "move:use_prioritized_replay_buffer", + "rel": "prereq" + }, + { + "source": "concept:actor_critic", + "target": "paper:sac", + "rel": "prereq" + }, + { + "source": "concept:actor_critic", + "target": "paper:a3c_a2c", + "rel": "prereq" + }, + { + "source": "concept:actor_critic", + "target": "paper:mpo", + "rel": "prereq" + }, + { + "source": "concept:actor_critic", + "target": "move:add_lagrangian_safety_constraint_to_actor_critic", + "rel": "prereq" + }, + { + "source": "paper:gpt3", + "target": "paper:gpt4", + "rel": "extends" + }, + { + "source": "paper:gpt4", + "target": "paper:gpt4v", + "rel": "extends" + }, + { + "source": "paper:gpt4", + "target": "paper:claude", + "rel": "parallel" + }, + { + "source": "paper:gpt4", + "target": "paper:gemini", + "rel": "parallel" + }, + { + "source": "paper:gpt3", + "target": "paper:llama", + "rel": "parallel" + }, + { + "source": "paper:llama", + "target": "paper:mistral", + "rel": "extends" + }, + { + "source": "paper:llama", + "target": "paper:qwen", + "rel": "parallel" + }, + { + "source": "paper:gpt3", + "target": "paper:instructgpt", + "rel": "extends" + }, + { + "source": "paper:instructgpt", + "target": "paper:gpt4", + "rel": "feeds" + }, + { + "source": "paper:instructgpt", + "target": "paper:rlhf_dpo", + "rel": "covers" + }, + { + "source": "paper:rlhf_dpo", + "target": "paper:constitutional_ai", + "rel": "parallel" + }, + { + "source": "paper:constitutional_ai", + "target": "paper:claude", + "rel": "feeds" + }, + { + "source": "paper:gpt3", + "target": "paper:cot_wei2022", + "rel": "extends" + }, + { + "source": "paper:cot_wei2022", + "target": "paper:self_consistency", + "rel": "extends" + }, + { + "source": "paper:cot_wei2022", + "target": "paper:react", + "rel": "extends" + }, + { + "source": "paper:cot_wei2022", + "target": "paper:tot", + "rel": "extends" + }, + { + "source": "paper:react", + "target": "paper:reflexion", + "rel": "extends" + }, + { + "source": "paper:react", + "target": "paper:toolformer", + "rel": "parallel" + }, + { + "source": "paper:react", + "target": "paper:voyager", + "rel": "feeds" + }, + { + "source": "paper:reflexion", + "target": "paper:voyager", + "rel": "feeds" + }, + { + "source": "paper:tot", + "target": "paper:verifier", + "rel": "parallel" + }, + { + "source": "paper:self_consistency", + "target": "paper:verifier", + "rel": "parallel" + }, + { + "source": "paper:debate", + "target": "paper:verifier", + "rel": "parallel" + }, + { + "source": "paper:react", + "target": "paper:swiftsage", + "rel": "feeds" + }, + { + "source": "paper:llava", + "target": "paper:flamingo", + "rel": "parallel" + }, + { + "source": "paper:flamingo", + "target": "paper:llava", + "rel": "prereq" + }, + { + "source": "paper:llava", + "target": "paper:internvl", + "rel": "extends" + }, + { + "source": "paper:llava", + "target": "paper:cambrian", + "rel": "extends" + }, + { + "source": "paper:florence", + "target": "paper:cambrian", + "rel": "parallel" + }, + { + "source": "paper:qwen", + "target": "paper:internvl", + "rel": "parallel" + }, + { + "source": "paper:llava", + "target": "paper:palme", + "rel": "parallel" + }, + { + "source": "paper:palme", + "target": "paper:rt2", + "rel": "prereq" + }, + { + "source": "paper:rt1", + "target": "paper:rt2", + "rel": "extends" + }, + { + "source": "paper:rt2", + "target": "paper:rtx", + "rel": "extends" + }, + { + "source": "paper:rtx", + "target": "paper:openvla", + "rel": "feeds" + }, + { + "source": "paper:rt2", + "target": "paper:openvla", + "rel": "extends" + }, + { + "source": "paper:rtx", + "target": "paper:octo", + "rel": "feeds" + }, + { + "source": "paper:openvla", + "target": "paper:octo", + "rel": "parallel" + }, + { + "source": "paper:vaswani2017", + "target": "paper:dit", + "rel": "prereq" + }, + { + "source": "paper:dit", + "target": "paper:sora", + "rel": "prereq" + }, + { + "source": "paper:dit", + "target": "paper:svd", + "rel": "prereq" + }, + { + "source": "paper:svd", + "target": "paper:sora", + "rel": "parallel" + }, + { + "source": "paper:sora", + "target": "paper:veo", + "rel": "parallel" + }, + { + "source": "paper:sora", + "target": "paper:cosmos", + "rel": "feeds" + }, + { + "source": "paper:svd", + "target": "paper:cosmos", + "rel": "prereq" + }, + { + "source": "paper:gaia1", + "target": "paper:cosmos", + "rel": "parallel" + }, + { + "source": "paper:drivedreamer", + "target": "paper:cosmos", + "rel": "parallel" + }, + { + "source": "paper:gaia1", + "target": "paper:prism1", + "rel": "parallel" + }, + { + "source": "paper:drivedreamer", + "target": "paper:prism1", + "rel": "parallel" + }, + { + "source": "paper:2402.12289", + "target": "paper:senna", + "rel": "parallel" + }, + { + "source": "paper:gemini", + "target": "paper:emma", + "rel": "prereq" + }, + { + "source": "paper:rt2", + "target": "paper:emma", + "rel": "parallel" + }, + { + "source": "paper:2402.12289", + "target": "paper:emma", + "rel": "parallel" + }, + { + "source": "paper:2311.10813", + "target": "paper:drivemlm", + "rel": "parallel" + }, + { + "source": "paper:2402.12289", + "target": "paper:drivemlm", + "rel": "parallel" + }, + { + "source": "paper:llava", + "target": "paper:drivelm", + "rel": "feeds" + }, + { + "source": "paper:drivelm", + "target": "paper:senna", + "rel": "feeds" + }, + { + "source": "paper:gpt_driver", + "target": "paper:lmdrive", + "rel": "parallel" + }, + { + "source": "paper:gpt3", + "target": "paper:gpt_driver", + "rel": "feeds" + }, + { + "source": "paper:llama", + "target": "paper:lmdrive", + "rel": "feeds" + }, + { + "source": "paper:lingo2", + "target": "paper:senna", + "rel": "parallel" + }, + { + "source": "paper:lingo2", + "target": "paper:prism1", + "rel": "parallel" + }, + { + "source": "paper:emma", + "target": "paper:2512.24426", + "rel": "parallel" + }, + { + "source": "paper:senna", + "target": "paper:2512.24426", + "rel": "feeds" + }, + { + "source": "paper:cosmos", + "target": "paper:2512.24426", + "rel": "feeds" + }, + { + "source": "paper:openvla", + "target": "paper:2512.24426", + "rel": "parallel" + }, + { + "source": "paper:gpt3", + "target": "move:scale_data_then_let_emergent_capabilities_appear", + "rel": "covers" + }, + { + "source": "paper:sora", + "target": "move:scale_data_then_let_emergent_capabilities_appear", + "rel": "covers" + }, + { + "source": "essay:bitter_lesson", + "target": "move:scale_data_then_let_emergent_capabilities_appear", + "rel": "covers" + }, + { + "source": "paper:llava", + "target": "move:pretrain_with_contrastive_alignment_between_modalities", + "rel": "covers" + }, + { + "source": "paper:flamingo", + "target": "move:pretrain_with_contrastive_alignment_between_modalities", + "rel": "covers" + }, + { + "source": "paper:instructgpt", + "target": "move:fine_tune_with_instruction_data_then_align_with_preferences", + "rel": "covers" + }, + { + "source": "paper:constitutional_ai", + "target": "move:fine_tune_with_instruction_data_then_align_with_preferences", + "rel": "covers" + }, + { + "source": "paper:rlhf_dpo", + "target": "move:fine_tune_with_instruction_data_then_align_with_preferences", + "rel": "covers" + }, + { + "source": "paper:llava", + "target": "move:plug_in_modality_encoder_to_frozen_language_model_via_projection", + "rel": "covers" + }, + { + "source": "paper:flamingo", + "target": "move:plug_in_modality_encoder_to_frozen_language_model_via_projection", + "rel": "covers" + }, + { + "source": "paper:palme", + "target": "move:plug_in_modality_encoder_to_frozen_language_model_via_projection", + "rel": "covers" + }, + { + "source": "paper:react", + "target": "move:wrap_language_model_with_tool_calling_loop", + "rel": "covers" + }, + { + "source": "paper:toolformer", + "target": "move:wrap_language_model_with_tool_calling_loop", + "rel": "covers" + }, + { + "source": "paper:2311.10813", + "target": "move:wrap_language_model_with_tool_calling_loop", + "rel": "covers" + }, + { + "source": "paper:reflexion", + "target": "move:add_reflection_step_so_agent_critiques_its_own_output", + "rel": "covers" + }, + { + "source": "paper:2309.16292", + "target": "move:add_reflection_step_so_agent_critiques_its_own_output", + "rel": "covers" + }, + { + "source": "paper:voyager", + "target": "move:add_reflection_step_so_agent_critiques_its_own_output", + "rel": "covers" + }, + { + "source": "paper:rt2", + "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", + "rel": "covers" + }, + { + "source": "paper:openvla", + "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", + "rel": "covers" + }, + { + "source": "paper:gpt_driver", + "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", + "rel": "covers" + }, + { + "source": "paper:emma", + "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", + "rel": "covers" + }, + { + "source": "paper:2512.24426", + "target": "move:augment_supervised_training_with_counterfactual_or_synthetic_data", + "rel": "covers" + }, + { + "source": "paper:cosmos", + "target": "move:augment_supervised_training_with_counterfactual_or_synthetic_data", + "rel": "covers" + }, + { + "source": "paper:drivedreamer", + "target": "move:augment_supervised_training_with_counterfactual_or_synthetic_data", + "rel": "covers" + }, + { + "source": "paper:gaia1", + "target": "move:condition_video_generative_model_on_control_action_for_world_model", + "rel": "covers" + }, + { + "source": "paper:drivedreamer", + "target": "move:condition_video_generative_model_on_control_action_for_world_model", + "rel": "covers" + }, + { + "source": "paper:cosmos", + "target": "move:condition_video_generative_model_on_control_action_for_world_model", + "rel": "covers" + }, + { + "source": "paper:voyager", + "target": "move:use_retrieval_augmented_memory_to_extend_context", + "rel": "covers" + }, + { + "source": "paper:2309.16292", + "target": "move:use_retrieval_augmented_memory_to_extend_context", + "rel": "covers" + }, + { + "source": "paper:tot", + "target": "move:cast_reasoning_as_search_over_thought_tree", + "rel": "covers" + }, + { + "source": "paper:silver2017_alphazero", + "target": "move:cast_reasoning_as_search_over_thought_tree", + "rel": "covers" + }, + { + "source": "paper:verifier", + "target": "move:cast_reasoning_as_search_over_thought_tree", + "rel": "covers" + }, + { + "source": "paper:rt2", + "target": "move:co_finetune_language_model_with_action_data_jointly", + "rel": "covers" + }, + { + "source": "paper:openvla", + "target": "move:co_finetune_language_model_with_action_data_jointly", + "rel": "covers" + }, + { + "source": "paper:emma", + "target": "move:co_finetune_language_model_with_action_data_jointly", + "rel": "covers" + }, + { + "source": "paper:silver2017_alphazero", + "target": "move:use_self_play_to_generate_unlimited_training_signal", + "rel": "covers" + }, + { + "source": "paper:debate", + "target": "move:use_self_play_to_generate_unlimited_training_signal", + "rel": "covers" + }, + { + "source": "paper:constitutional_ai", + "target": "move:use_self_play_to_generate_unlimited_training_signal", + "rel": "covers" + }, + { + "source": "paper:gpt4", + "target": "move:distill_large_model_into_specialist_for_deployment", + "rel": "covers" + }, + { + "source": "paper:2402.12289", + "target": "move:distill_large_model_into_specialist_for_deployment", + "rel": "covers" + }, + { + "source": "paper:senna", + "target": "move:distill_large_model_into_specialist_for_deployment", + "rel": "covers" + }, + { + "source": "paper:gaia1", + "target": "move:rewrite_continuous_video_as_token_sequence_for_transformer_world_model", + "rel": "covers" + }, + { + "source": "paper:cosmos", + "target": "move:rewrite_continuous_video_as_token_sequence_for_transformer_world_model", + "rel": "covers" + }, + { + "source": "paper:senna", + "target": "move:condition_on_language_meta_action_then_emit_low_level_action", + "rel": "covers" + }, + { + "source": "paper:2402.12289", + "target": "move:condition_on_language_meta_action_then_emit_low_level_action", + "rel": "covers" + }, + { + "source": "paper:2512.24426", + "target": "move:condition_on_language_meta_action_then_emit_low_level_action", + "rel": "covers" + }, + { + "source": "paper:gpt4", + "target": "move:cache_kv_state_to_amortize_long_context", + "rel": "covers" + }, + { + "source": "paper:mistral", + "target": "move:cache_kv_state_to_amortize_long_context", + "rel": "covers" + }, + { + "source": "paper:mistral", + "target": "move:speculative_decoding_with_draft_model", + "rel": "covers" + }, + { + "source": "paper:gemini", + "target": "move:speculative_decoding_with_draft_model", + "rel": "covers" + }, + { + "source": "paper:llava", + "target": "move:freeze_visual_encoder_and_only_train_connector", + "rel": "covers" + }, + { + "source": "paper:flamingo", + "target": "move:freeze_visual_encoder_and_only_train_connector", + "rel": "covers" + }, + { + "source": "paper:cambrian", + "target": "move:freeze_visual_encoder_and_only_train_connector", + "rel": "covers" + }, + { + "source": "paper:octo", + "target": "move:use_diffusion_head_for_continuous_action", + "rel": "covers" + }, + { + "source": "paper:diffuser", + "target": "move:use_diffusion_head_for_continuous_action", + "rel": "covers" + }, + { + "source": "paper:gpt_driver", + "target": "move:treat_planning_as_autoregressive_trajectory_generation", + "rel": "covers" + }, + { + "source": "paper:emma", + "target": "move:treat_planning_as_autoregressive_trajectory_generation", + "rel": "covers" + }, + { + "source": "paper:vadv2", + "target": "move:treat_planning_as_autoregressive_trajectory_generation", + "rel": "parallel" + }, + { + "source": "paper:2512.24426", + "target": "move:use_world_model_rollout_as_critic_for_policy", + "rel": "covers" + }, + { + "source": "paper:world_models", + "target": "move:use_world_model_rollout_as_critic_for_policy", + "rel": "covers" + }, + { + "source": "paper:cosmos", + "target": "move:use_world_model_rollout_as_critic_for_policy", + "rel": "covers" + }, + { + "source": "paper:voyager", + "target": "move:long_horizon_via_hierarchical_subgoal", + "rel": "covers" + }, + { + "source": "paper:swiftsage", + "target": "move:long_horizon_via_hierarchical_subgoal", + "rel": "covers" + }, + { + "source": "paper:2311.10813", + "target": "move:long_horizon_via_hierarchical_subgoal", + "rel": "covers" + }, + { + "source": "paper:debate", + "target": "move:prompt_chain_with_explicit_persona_roles", + "rel": "covers" + }, + { + "source": "paper:swiftsage", + "target": "move:prompt_chain_with_explicit_persona_roles", + "rel": "covers" + }, + { + "source": "paper:2311.10813", + "target": "move:prompt_chain_with_explicit_persona_roles", + "rel": "covers" + }, + { + "source": "paper:2512.24426", + "target": "move:contrast_corner_case_against_normal_case_in_training", + "rel": "covers" + }, + { + "source": "paper:cosmos", + "target": "move:contrast_corner_case_against_normal_case_in_training", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "move:evaluate_open_loop_then_close_loop_for_realism", + "rel": "covers" + }, + { + "source": "paper:drivemlm", + "target": "move:evaluate_open_loop_then_close_loop_for_realism", + "rel": "covers" + }, + { + "source": "paper:lmdrive", + "target": "move:evaluate_open_loop_then_close_loop_for_realism", + "rel": "covers" + }, + { + "source": "paper:drivelm", + "target": "move:use_language_explanation_as_auxiliary_supervision", + "rel": "covers" + }, + { + "source": "paper:lingo2", + "target": "move:use_language_explanation_as_auxiliary_supervision", + "rel": "covers" + }, + { + "source": "paper:emma", + "target": "move:use_language_explanation_as_auxiliary_supervision", + "rel": "covers" + }, + { + "source": "problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop", + "target": "paper:2512.24426", + "rel": "motivates" + }, + { + "source": "problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop", + "target": "paper:senna", + "rel": "motivates" + }, + { + "source": "problem:hallucinated_action_from_vision_language_model_in_safety_critical_loop", + "target": "move:add_reflection_step_so_agent_critiques_its_own_output", + "rel": "motivates" + }, + { + "source": "problem:grounding_language_token_to_continuous_physical_world", + "target": "paper:rt2", + "rel": "motivates" + }, + { + "source": "problem:grounding_language_token_to_continuous_physical_world", + "target": "paper:palme", + "rel": "motivates" + }, + { + "source": "problem:grounding_language_token_to_continuous_physical_world", + "target": "move:replace_explicit_action_head_with_tokenized_action_sequence", + "rel": "motivates" + }, + { + "source": "problem:latency_budget_for_large_model_in_realtime_control", + "target": "paper:2402.12289", + "rel": "motivates" + }, + { + "source": "problem:latency_budget_for_large_model_in_realtime_control", + "target": "move:distill_large_model_into_specialist_for_deployment", + "rel": "motivates" + }, + { + "source": "problem:latency_budget_for_large_model_in_realtime_control", + "target": "move:speculative_decoding_with_draft_model", + "rel": "motivates" + }, + { + "source": "problem:latency_budget_for_large_model_in_realtime_control", + "target": "move:cache_kv_state_to_amortize_long_context", + "rel": "motivates" + }, + { + "source": "problem:long_horizon_reasoning_with_finite_context_window", + "target": "paper:gemini", + "rel": "motivates" + }, + { + "source": "problem:long_horizon_reasoning_with_finite_context_window", + "target": "move:use_retrieval_augmented_memory_to_extend_context", + "rel": "motivates" + }, + { + "source": "problem:long_horizon_reasoning_with_finite_context_window", + "target": "move:long_horizon_via_hierarchical_subgoal", + "rel": "motivates" + }, + { + "source": "problem:zero_shot_generalization_to_unseen_driving_scenes", + "target": "paper:gpt4v", + "rel": "motivates" + }, + { + "source": "problem:zero_shot_generalization_to_unseen_driving_scenes", + "target": "paper:emma", + "rel": "motivates" + }, + { + "source": "problem:fine_grained_spatial_understanding_in_vision_language_model", + "target": "paper:cambrian", + "rel": "motivates" + }, + { + "source": "problem:fine_grained_spatial_understanding_in_vision_language_model", + "target": "paper:florence", + "rel": "motivates" + }, + { + "source": "problem:counterfactual_reasoning_about_other_agents_intent", + "target": "paper:2512.24426", + "rel": "motivates" + }, + { + "source": "problem:counterfactual_reasoning_about_other_agents_intent", + "target": "paper:gaia1", + "rel": "motivates" + }, + { + "source": "problem:counterfactual_reasoning_about_other_agents_intent", + "target": "move:use_world_model_rollout_as_critic_for_policy", + "rel": "motivates" + }, + { + "source": "problem:open_world_corner_case_synthesis_for_training", + "target": "paper:cosmos", + "rel": "motivates" + }, + { + "source": "problem:open_world_corner_case_synthesis_for_training", + "target": "paper:prism1", + "rel": "motivates" + }, + { + "source": "problem:open_world_corner_case_synthesis_for_training", + "target": "move:contrast_corner_case_against_normal_case_in_training", + "rel": "motivates" + }, + { + "source": "problem:evaluation_gap_between_offline_benchmark_and_closed_loop", + "target": "paper:drivemlm", + "rel": "motivates" + }, + { + "source": "problem:evaluation_gap_between_offline_benchmark_and_closed_loop", + "target": "paper:lmdrive", + "rel": "motivates" + }, + { + "source": "problem:evaluation_gap_between_offline_benchmark_and_closed_loop", + "target": "move:evaluate_open_loop_then_close_loop_for_realism", + "rel": "motivates" + }, + { + "source": "problem:catastrophic_forgetting_after_action_finetuning", + "target": "paper:rt2", + "rel": "motivates" + }, + { + "source": "problem:catastrophic_forgetting_after_action_finetuning", + "target": "paper:openvla", + "rel": "motivates" + }, + { + "source": "problem:catastrophic_forgetting_after_action_finetuning", + "target": "move:co_finetune_language_model_with_action_data_jointly", + "rel": "motivates" + }, + { + "source": "insight:language_is_compressed_world_model_for_human_priors", + "target": "paper:gpt3", + "rel": "covers" + }, + { + "source": "insight:language_is_compressed_world_model_for_human_priors", + "target": "paper:2309.16292", + "rel": "covers" + }, + { + "source": "insight:language_is_compressed_world_model_for_human_priors", + "target": "paper:palme", + "rel": "covers" + }, + { + "source": "insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models", + "target": "paper:gpt4", + "rel": "covers" + }, + { + "source": "insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models", + "target": "essay:bitter_lesson", + "rel": "covers" + }, + { + "source": "insight:scaling_data_unlocks_capabilities_not_present_in_smaller_models", + "target": "move:scale_data_then_let_emergent_capabilities_appear", + "rel": "covers" + }, + { + "source": "insight:world_model_video_diffusion_is_implicit_physics_engine", + "target": "paper:sora", + "rel": "covers" + }, + { + "source": "insight:world_model_video_diffusion_is_implicit_physics_engine", + "target": "paper:gaia1", + "rel": "covers" + }, + { + "source": "insight:world_model_video_diffusion_is_implicit_physics_engine", + "target": "paper:cosmos", + "rel": "covers" + }, + { + "source": "insight:agent_loop_is_just_iterated_conditional_generation", + "target": "paper:react", + "rel": "covers" + }, + { + "source": "insight:agent_loop_is_just_iterated_conditional_generation", + "target": "paper:reflexion", + "rel": "covers" + }, + { + "source": "insight:agent_loop_is_just_iterated_conditional_generation", + "target": "paper:2311.10813", + "rel": "covers" + }, + { + "source": "insight:tool_use_extends_language_model_into_environment_grounded_actor", + "target": "paper:toolformer", + "rel": "covers" + }, + { + "source": "insight:tool_use_extends_language_model_into_environment_grounded_actor", + "target": "paper:react", + "rel": "covers" + }, + { + "source": "insight:tool_use_extends_language_model_into_environment_grounded_actor", + "target": "paper:2311.10813", + "rel": "covers" + }, + { + "source": "insight:counterfactual_replanning_separates_intent_from_execution", + "target": "paper:2512.24426", + "rel": "covers" + }, + { + "source": "insight:counterfactual_replanning_separates_intent_from_execution", + "target": "move:use_world_model_rollout_as_critic_for_policy", + "rel": "covers" + }, + { + "source": "insight:foundation_model_decouples_perception_from_task_specific_training", + "target": "paper:dinov2", + "rel": "covers" + }, + { + "source": "insight:foundation_model_decouples_perception_from_task_specific_training", + "target": "paper:sam", + "rel": "covers" + }, + { + "source": "insight:foundation_model_decouples_perception_from_task_specific_training", + "target": "paper:florence", + "rel": "covers" + }, + { + "source": "insight:dual_system_fast_slow_loop_marries_reactive_and_deliberative_control", + "target": "paper:swiftsage", + "rel": "covers" + }, + { + "source": "insight:dual_system_fast_slow_loop_marries_reactive_and_deliberative_control", + "target": "paper:2402.12289", + "rel": "covers" + }, + { + "source": "insight:emergent_planning_from_next_token_prediction_alone", + "target": "paper:gpt4", + "rel": "covers" + }, + { + "source": "insight:emergent_planning_from_next_token_prediction_alone", + "target": "paper:cot_wei2022", + "rel": "covers" + }, + { + "source": "insight:alignment_is_constraint_satisfaction_over_generation", + "target": "paper:instructgpt", + "rel": "covers" + }, + { + "source": "insight:alignment_is_constraint_satisfaction_over_generation", + "target": "paper:rlhf_dpo", + "rel": "covers" + }, + { + "source": "insight:alignment_is_constraint_satisfaction_over_generation", + "target": "paper:constitutional_ai", + "rel": "covers" + }, + { + "source": "insight:open_weight_release_compounds_research_velocity", + "target": "paper:llama", + "rel": "covers" + }, + { + "source": "insight:open_weight_release_compounds_research_velocity", + "target": "paper:openvla", + "rel": "covers" + }, + { + "source": "insight:open_weight_release_compounds_research_velocity", + "target": "paper:qwen", + "rel": "covers" + }, + { + "source": "insight:open_weight_release_compounds_research_velocity", + "target": "paper:mistral", + "rel": "covers" + }, + { + "source": "paradigm:foundation_model_axis", + "target": "paper:gpt4", + "rel": "covers" + }, + { + "source": "paradigm:foundation_model_axis", + "target": "paper:llama", + "rel": "covers" + }, + { + "source": "paradigm:foundation_model_axis", + "target": "paper:llava", + "rel": "covers" + }, + { + "source": "paradigm:foundation_model_axis", + "target": "paradigm:vla_paradigm", + "rel": "parallel" + }, + { + "source": "paradigm:foundation_model_axis", + "target": "paradigm:world_model_paradigm", + "rel": "parallel" + }, + { + "source": "paradigm:foundation_model_axis", + "target": "paradigm:llm_agent_paradigm", + "rel": "parallel" + }, + { + "source": "paradigm:world_model_paradigm", + "target": "paper:world_models", + "rel": "covers" + }, + { + "source": "paradigm:world_model_paradigm", + "target": "paper:gaia1", + "rel": "covers" + }, + { + "source": "paradigm:world_model_paradigm", + "target": "paper:cosmos", + "rel": "covers" + }, + { + "source": "paradigm:world_model_paradigm", + "target": "paper:sora", + "rel": "covers" + }, + { + "source": "paradigm:llm_agent_paradigm", + "target": "paper:react", + "rel": "covers" + }, + { + "source": "paradigm:llm_agent_paradigm", + "target": "paper:2311.10813", + "rel": "covers" + }, + { + "source": "paradigm:llm_agent_paradigm", + "target": "paper:2309.16292", + "rel": "covers" + }, + { + "source": "paradigm:llm_agent_paradigm", + "target": "paper:voyager", + "rel": "covers" + }, + { + "source": "paradigm:vla_paradigm", + "target": "paper:rt2", + "rel": "covers" + }, + { + "source": "paradigm:vla_paradigm", + "target": "paper:openvla", + "rel": "covers" + }, + { + "source": "paradigm:vla_paradigm", + "target": "paper:emma", + "rel": "covers" + }, + { + "source": "paradigm:vla_paradigm", + "target": "paper:2512.24426", + "rel": "covers" + }, + { + "source": "paradigm:vla_paradigm", + "target": "paper:2402.12289", + "rel": "covers" + }, + { + "source": "move:residual_connection", + "target": "insight:residual_learning_unlocks_arbitrary_depth", + "rel": "composes" + }, + { + "source": "paper:he2015_resnet", + "target": "insight:residual_learning_unlocks_arbitrary_depth", + "rel": "composes" + }, + { + "source": "move:masking_for_pretext", + "target": "insight:masked_prediction_yields_self_supervised_signal", + "rel": "composes" + }, + { + "source": "concept:ssl", + "target": "insight:masked_prediction_yields_self_supervised_signal", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "insight:attention_is_typed_entity_communication", + "rel": "composes" + }, + { + "source": "concept:self_attention", + "target": "insight:attention_is_typed_entity_communication", + "rel": "composes" + }, + { + "source": "concept:detr_query", + "target": "insight:attention_is_typed_entity_communication", + "rel": "composes" + }, + { + "source": "move:contrastive_alignment", + "target": "insight:contrastive_alignment_creates_zero_shot_transfer", + "rel": "composes" + }, + { + "source": "concept:vlm", + "target": "insight:contrastive_alignment_creates_zero_shot_transfer", + "rel": "composes" + }, + { + "source": "move:diffusion_denoise_sampling", + "target": "insight:diffusion_unifies_generation_and_decision", + "rel": "composes" + }, + { + "source": "paper:diffuser", + "target": "insight:diffusion_unifies_generation_and_decision", + "rel": "composes" + }, + { + "source": "paper:2212.10156", + "target": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", + "rel": "composes" + }, + { + "source": "essay:bitter_lesson", + "target": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", + "rel": "composes" + }, + { + "source": "move:dual_system_fast_slow", + "target": "insight:dual_system_handles_latency_quality_tradeoff", + "rel": "composes" + }, + { + "source": "paper:2402.12289", + "target": "insight:dual_system_handles_latency_quality_tradeoff", + "rel": "composes" + }, + { + "source": "concept:cot", + "target": "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "rel": "composes" + }, + { + "source": "paper:2309.16292", + "target": "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "rel": "composes" + }, + { + "source": "move:counterfactual_replan", + "target": "insight:long_tail_solved_by_synthesis_not_data_alone", + "rel": "composes" + }, + { + "source": "paper:2512.24426", + "target": "insight:long_tail_solved_by_synthesis_not_data_alone", + "rel": "composes" + }, + { + "source": "paper:drivedreamer", + "target": "insight:long_tail_solved_by_synthesis_not_data_alone", + "rel": "composes" + }, + { + "source": "essay:bitter_lesson", + "target": "insight:scaling_laws_predict_capability_emergence", + "rel": "composes" + }, + { + "source": "concept:scaling_vs_knowledge", + "target": "insight:scaling_laws_predict_capability_emergence", + "rel": "composes" + }, + { + "source": "paper:gpt3", + "target": "insight:scaling_laws_predict_capability_emergence", + "rel": "composes" + }, + { + "source": "paper:dinov2", + "target": "insight:foundation_pretraining_decouples_data_from_task", + "rel": "composes" + }, + { + "source": "paper:2508.10104", + "target": "insight:foundation_pretraining_decouples_data_from_task", + "rel": "composes" + }, + { + "source": "concept:ssl", + "target": "insight:foundation_pretraining_decouples_data_from_task", + "rel": "composes" + }, + { + "source": "move:self_play_with_search", + "target": "insight:test_time_compute_substitutes_train_time_via_search", + "rel": "composes" + }, + { + "source": "paper:silver2017_alphazero", + "target": "insight:test_time_compute_substitutes_train_time_via_search", + "rel": "composes" + }, + { + "source": "concept:imitation_learning", + "target": "insight:imitation_data_compresses_unspecified_reward", + "rel": "composes" + }, + { + "source": "paper:ross2011_dagger", + "target": "insight:imitation_data_compresses_unspecified_reward", + "rel": "composes" + }, + { + "source": "move:latent_imagination_rollout", + "target": "insight:world_models_let_planning_be_done_in_imagination", + "rel": "composes" + }, + { + "source": "paper:world_models", + "target": "insight:world_models_let_planning_be_done_in_imagination", + "rel": "composes" + }, + { + "source": "paper:gaia1", + "target": "insight:world_models_let_planning_be_done_in_imagination", + "rel": "composes" + }, + { + "source": "move:tokenize_modalities", + "target": "insight:tokenization_collapses_modality_gap", + "rel": "composes" + }, + { + "source": "move:patchify_tokenization", + "target": "insight:tokenization_collapses_modality_gap", + "rel": "composes" + }, + { + "source": "move:set_prediction_with_hungarian", + "target": "insight:set_prediction_eliminates_postprocessing_heuristics", + "rel": "composes" + }, + { + "source": "paper:carion2020", + "target": "insight:set_prediction_eliminates_postprocessing_heuristics", + "rel": "composes" + }, + { + "source": "paper:gpt3", + "target": "insight:in_context_learning_emerges_at_scale", + "rel": "composes" + }, + { + "source": "concept:cot", + "target": "insight:in_context_learning_emerges_at_scale", + "rel": "composes" + }, + { + "source": "concept:policy_gradient", + "target": "insight:safety_constraints_via_lagrangian_dual", + "rel": "composes" + }, + { + "source": "concept:actor_critic", + "target": "insight:safety_constraints_via_lagrangian_dual", + "rel": "composes" + }, + { + "source": "move:spike_event_compute", + "target": "insight:event_sparse_compute_matches_energy_budget", + "rel": "composes" + }, + { + "source": "concept:spiking_nn", + "target": "insight:event_sparse_compute_matches_energy_budget", + "rel": "composes" + }, + { + "source": "paper:2307.01694", + "target": "insight:event_sparse_compute_matches_energy_budget", + "rel": "composes" + }, + { + "source": "paper:li2022bevformer", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "paper:carion2020", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "concept:bev", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "concept:detr_query", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "move:set_prediction_with_hungarian", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "concept:imitation_learning", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "insight:attention_is_typed_entity_communication", + "target": "validation:trace_unified_planning_oriented_e2e_driving", + "rel": "composes" + }, + { + "source": "validation:trace_unified_planning_oriented_e2e_driving", + "target": "paper:2212.10156", + "rel": "validates" + }, + { + "source": "paper:vaswani2017", + "target": "validation:trace_object_level_planning_transformer", + "rel": "composes" + }, + { + "source": "paper:carion2020", + "target": "validation:trace_object_level_planning_transformer", + "rel": "composes" + }, + { + "source": "concept:transformer", + "target": "validation:trace_object_level_planning_transformer", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "validation:trace_object_level_planning_transformer", + "rel": "composes" + }, + { + "source": "move:set_prediction_with_hungarian", + "target": "validation:trace_object_level_planning_transformer", + "rel": "composes" + }, + { + "source": "concept:imitation_learning", + "target": "validation:trace_object_level_planning_transformer", + "rel": "composes" + }, + { + "source": "paper:ad_benchmarks", + "target": "validation:trace_object_level_planning_transformer", + "rel": "composes" + }, + { + "source": "paper:transfuser", + "target": "validation:trace_object_level_planning_transformer", + "rel": "composes" + }, + { + "source": "validation:trace_object_level_planning_transformer", + "target": "paper:2210.14222", + "rel": "validates" + }, + { + "source": "paper:llava", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "concept:vlm", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "concept:vla", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "concept:cot", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "move:dual_system_fast_slow", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "insight:dual_system_handles_latency_quality_tradeoff", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "paper:li2022bevformer", + "target": "validation:trace_vision_language_action_dual_loop", + "rel": "composes" + }, + { + "source": "validation:trace_vision_language_action_dual_loop", + "target": "paper:2402.12289", + "rel": "validates" + }, + { + "source": "paper:gpt3", + "target": "validation:trace_llm_decision_agent_for_driving", + "rel": "composes" + }, + { + "source": "concept:vlm", + "target": "validation:trace_llm_decision_agent_for_driving", + "rel": "composes" + }, + { + "source": "concept:cot", + "target": "validation:trace_llm_decision_agent_for_driving", + "rel": "composes" + }, + { + "source": "concept:tool_use", + "target": "validation:trace_llm_decision_agent_for_driving", + "rel": "composes" + }, + { + "source": "move:tool_use_function_calling", + "target": "validation:trace_llm_decision_agent_for_driving", + "rel": "composes" + }, + { + "source": "paper:2210.14222", + "target": "validation:trace_llm_decision_agent_for_driving", + "rel": "composes" + }, + { + "source": "insight:in_context_learning_emerges_at_scale", + "target": "validation:trace_llm_decision_agent_for_driving", + "rel": "composes" + }, + { + "source": "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "target": "validation:trace_llm_decision_agent_for_driving", + "rel": "composes" + }, + { + "source": "validation:trace_llm_decision_agent_for_driving", + "target": "paper:2311.10813", + "rel": "validates" + }, + { + "source": "paper:gpt3", + "target": "validation:trace_knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "concept:vlm", + "target": "validation:trace_knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "concept:cot", + "target": "validation:trace_knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "concept:tool_use", + "target": "validation:trace_knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "essay:bitter_lesson", + "target": "validation:trace_knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "paper:ad_benchmarks", + "target": "validation:trace_knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "target": "validation:trace_knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "insight:in_context_learning_emerges_at_scale", + "target": "validation:trace_knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "validation:trace_knowledge_driven_reflective_agent", + "target": "paper:2309.16292", + "rel": "validates" + }, + { + "source": "concept:spiking_nn", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "paper:vaswani2017", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "paper:vit", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "paper:he2015_resnet", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "concept:self_attention", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "move:residual_connection", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "move:spike_event_compute", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "insight:event_sparse_compute_matches_energy_budget", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "essay:bitter_lesson", + "target": "validation:trace_brain_inspired_spike_attention", + "rel": "composes" + }, + { + "source": "validation:trace_brain_inspired_spike_attention", + "target": "paper:2307.01694", + "rel": "validates" + }, + { + "source": "paper:dinov2", + "target": "validation:trace_scalable_self_supervised_vision_backbone", + "rel": "composes" + }, + { + "source": "paper:vit", + "target": "validation:trace_scalable_self_supervised_vision_backbone", + "rel": "composes" + }, + { + "source": "concept:ssl", + "target": "validation:trace_scalable_self_supervised_vision_backbone", + "rel": "composes" + }, + { + "source": "move:masking_for_pretext", + "target": "validation:trace_scalable_self_supervised_vision_backbone", + "rel": "composes" + }, + { + "source": "essay:bitter_lesson", + "target": "validation:trace_scalable_self_supervised_vision_backbone", + "rel": "composes" + }, + { + "source": "insight:foundation_pretraining_decouples_data_from_task", + "target": "validation:trace_scalable_self_supervised_vision_backbone", + "rel": "composes" + }, + { + "source": "insight:scaling_laws_predict_capability_emergence", + "target": "validation:trace_scalable_self_supervised_vision_backbone", + "rel": "composes" + }, + { + "source": "insight:masked_prediction_yields_self_supervised_signal", + "target": "validation:trace_scalable_self_supervised_vision_backbone", + "rel": "composes" + }, + { + "source": "validation:trace_scalable_self_supervised_vision_backbone", + "target": "paper:2508.10104", + "rel": "validates" + }, + { + "source": "paper:2402.12289", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "paper:llava", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "paper:world_models", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "paper:gaia1", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "paper:drivedreamer", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "paper:rlhf_dpo", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "concept:vla", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "concept:counterfactual", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "concept:meta_action", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "move:counterfactual_replan", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "move:latent_imagination_rollout", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "insight:long_tail_solved_by_synthesis_not_data_alone", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "insight:world_models_let_planning_be_done_in_imagination", + "target": "validation:trace_counterfactual_vla_replanner", + "rel": "composes" + }, + { + "source": "validation:trace_counterfactual_vla_replanner", + "target": "paper:2512.24426", + "rel": "validates" + }, + { + "source": "paper:vaswani2017", + "target": "validation:trace_set_prediction_with_object_queries", + "rel": "composes" + }, + { + "source": "concept:transformer", + "target": "validation:trace_set_prediction_with_object_queries", + "rel": "composes" + }, + { + "source": "concept:self_attention", + "target": "validation:trace_set_prediction_with_object_queries", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "validation:trace_set_prediction_with_object_queries", + "rel": "composes" + }, + { + "source": "move:set_prediction_with_hungarian", + "target": "validation:trace_set_prediction_with_object_queries", + "rel": "composes" + }, + { + "source": "insight:attention_is_typed_entity_communication", + "target": "validation:trace_set_prediction_with_object_queries", + "rel": "composes" + }, + { + "source": "insight:set_prediction_eliminates_postprocessing_heuristics", + "target": "validation:trace_set_prediction_with_object_queries", + "rel": "composes" + }, + { + "source": "validation:trace_set_prediction_with_object_queries", + "target": "paper:carion2020", + "rel": "validates" + }, + { + "source": "paper:he2015_resnet", + "target": "validation:trace_self_attention_replaces_recurrence", + "rel": "composes" + }, + { + "source": "concept:self_attention", + "target": "validation:trace_self_attention_replaces_recurrence", + "rel": "composes" + }, + { + "source": "concept:transformer", + "target": "validation:trace_self_attention_replaces_recurrence", + "rel": "composes" + }, + { + "source": "move:residual_connection", + "target": "validation:trace_self_attention_replaces_recurrence", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "validation:trace_self_attention_replaces_recurrence", + "rel": "composes" + }, + { + "source": "insight:residual_learning_unlocks_arbitrary_depth", + "target": "validation:trace_self_attention_replaces_recurrence", + "rel": "composes" + }, + { + "source": "insight:attention_is_typed_entity_communication", + "target": "validation:trace_self_attention_replaces_recurrence", + "rel": "composes" + }, + { + "source": "validation:trace_self_attention_replaces_recurrence", + "target": "paper:vaswani2017", + "rel": "validates" + }, + { + "source": "paper:vaswani2017", + "target": "validation:trace_image_transformer_via_patch_tokenization", + "rel": "composes" + }, + { + "source": "concept:transformer", + "target": "validation:trace_image_transformer_via_patch_tokenization", + "rel": "composes" + }, + { + "source": "concept:self_attention", + "target": "validation:trace_image_transformer_via_patch_tokenization", + "rel": "composes" + }, + { + "source": "move:patchify_tokenization", + "target": "validation:trace_image_transformer_via_patch_tokenization", + "rel": "composes" + }, + { + "source": "insight:tokenization_collapses_modality_gap", + "target": "validation:trace_image_transformer_via_patch_tokenization", + "rel": "composes" + }, + { + "source": "insight:scaling_laws_predict_capability_emergence", + "target": "validation:trace_image_transformer_via_patch_tokenization", + "rel": "composes" + }, + { + "source": "validation:trace_image_transformer_via_patch_tokenization", + "target": "paper:vit", + "rel": "validates" + }, + { + "source": "paper:vaswani2017", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "paper:vit", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "paper:carion2020", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "concept:transformer", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "concept:detr_query", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "concept:bev", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "move:lift_2d_to_3d", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "move:patchify_tokenization", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "insight:attention_is_typed_entity_communication", + "target": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "rel": "composes" + }, + { + "source": "validation:trace_bird_eye_view_transformer_with_temporal_aggregation", + "target": "paper:li2022bevformer", + "rel": "validates" + }, + { + "source": "paper:vaswani2017", + "target": "validation:trace_few_shot_in_context_learning_at_scale", + "rel": "composes" + }, + { + "source": "concept:transformer", + "target": "validation:trace_few_shot_in_context_learning_at_scale", + "rel": "composes" + }, + { + "source": "essay:bitter_lesson", + "target": "validation:trace_few_shot_in_context_learning_at_scale", + "rel": "composes" + }, + { + "source": "concept:scaling_vs_knowledge", + "target": "validation:trace_few_shot_in_context_learning_at_scale", + "rel": "composes" + }, + { + "source": "insight:scaling_laws_predict_capability_emergence", + "target": "validation:trace_few_shot_in_context_learning_at_scale", + "rel": "composes" + }, + { + "source": "insight:in_context_learning_emerges_at_scale", + "target": "validation:trace_few_shot_in_context_learning_at_scale", + "rel": "composes" + }, + { + "source": "validation:trace_few_shot_in_context_learning_at_scale", + "target": "paper:gpt3", + "rel": "validates" + }, + { + "source": "concept:policy_gradient", + "target": "validation:trace_clipped_policy_gradient_surrogate", + "rel": "composes" + }, + { + "source": "concept:actor_critic", + "target": "validation:trace_clipped_policy_gradient_surrogate", + "rel": "composes" + }, + { + "source": "concept:ppo", + "target": "validation:trace_clipped_policy_gradient_surrogate", + "rel": "composes" + }, + { + "source": "move:clipped_surrogate_objective", + "target": "validation:trace_clipped_policy_gradient_surrogate", + "rel": "composes" + }, + { + "source": "course:zhao_rl", + "target": "validation:trace_clipped_policy_gradient_surrogate", + "rel": "composes" + }, + { + "source": "paper:sutton_barto", + "target": "validation:trace_clipped_policy_gradient_surrogate", + "rel": "composes" + }, + { + "source": "validation:trace_clipped_policy_gradient_surrogate", + "target": "paper:schulman2017_ppo", + "rel": "validates" + }, + { + "source": "concept:mdp", + "target": "validation:trace_deep_q_network_with_replay_and_target", + "rel": "composes" + }, + { + "source": "concept:bellman_eq", + "target": "validation:trace_deep_q_network_with_replay_and_target", + "rel": "composes" + }, + { + "source": "concept:td_learning", + "target": "validation:trace_deep_q_network_with_replay_and_target", + "rel": "composes" + }, + { + "source": "concept:dqn", + "target": "validation:trace_deep_q_network_with_replay_and_target", + "rel": "composes" + }, + { + "source": "concept:replay_buffer", + "target": "validation:trace_deep_q_network_with_replay_and_target", + "rel": "composes" + }, + { + "source": "move:replay_and_target_net", + "target": "validation:trace_deep_q_network_with_replay_and_target", + "rel": "composes" + }, + { + "source": "course:zhao_rl", + "target": "validation:trace_deep_q_network_with_replay_and_target", + "rel": "composes" + }, + { + "source": "paper:sutton_barto", + "target": "validation:trace_deep_q_network_with_replay_and_target", + "rel": "composes" + }, + { + "source": "validation:trace_deep_q_network_with_replay_and_target", + "target": "paper:mnih2015_dqn", + "rel": "validates" + }, + { + "source": "concept:mdp", + "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", + "rel": "composes" + }, + { + "source": "concept:policy_gradient", + "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", + "rel": "composes" + }, + { + "source": "concept:actor_critic", + "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", + "rel": "composes" + }, + { + "source": "essay:bitter_lesson", + "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", + "rel": "composes" + }, + { + "source": "move:self_play_with_search", + "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", + "rel": "composes" + }, + { + "source": "insight:test_time_compute_substitutes_train_time_via_search", + "target": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", + "rel": "composes" + }, + { + "source": "validation:trace_alpha_zero_self_play_with_mcts_guided_policy", + "target": "paper:silver2017_alphazero", + "rel": "validates" + }, + { + "source": "concept:imitation_learning", + "target": "validation:trace_dataset_aggregation_for_imitation", + "rel": "composes" + }, + { + "source": "concept:covariate_shift", + "target": "validation:trace_dataset_aggregation_for_imitation", + "rel": "composes" + }, + { + "source": "move:dataset_aggregation", + "target": "validation:trace_dataset_aggregation_for_imitation", + "rel": "composes" + }, + { + "source": "course:cs285", + "target": "validation:trace_dataset_aggregation_for_imitation", + "rel": "composes" + }, + { + "source": "insight:imitation_data_compresses_unspecified_reward", + "target": "validation:trace_dataset_aggregation_for_imitation", + "rel": "composes" + }, + { + "source": "validation:trace_dataset_aggregation_for_imitation", + "target": "paper:ross2011_dagger", + "rel": "validates" + }, + { + "source": "concept:mdp", + "target": "validation:trace_world_model_in_latent_imagination", + "rel": "composes" + }, + { + "source": "concept:policy_gradient", + "target": "validation:trace_world_model_in_latent_imagination", + "rel": "composes" + }, + { + "source": "concept:actor_critic", + "target": "validation:trace_world_model_in_latent_imagination", + "rel": "composes" + }, + { + "source": "move:latent_imagination_rollout", + "target": "validation:trace_world_model_in_latent_imagination", + "rel": "composes" + }, + { + "source": "insight:world_models_let_planning_be_done_in_imagination", + "target": "validation:trace_world_model_in_latent_imagination", + "rel": "composes" + }, + { + "source": "paper:world_models", + "target": "validation:trace_world_model_in_latent_imagination", + "rel": "composes" + }, + { + "source": "validation:trace_world_model_in_latent_imagination", + "target": "paper:world_models", + "rel": "validates" + }, + { + "source": "paper:vit", + "target": "validation:trace_vision_language_pretrained_dual_encoder", + "rel": "composes" + }, + { + "source": "paper:vaswani2017", + "target": "validation:trace_vision_language_pretrained_dual_encoder", + "rel": "composes" + }, + { + "source": "concept:vlm", + "target": "validation:trace_vision_language_pretrained_dual_encoder", + "rel": "composes" + }, + { + "source": "move:contrastive_alignment", + "target": "validation:trace_vision_language_pretrained_dual_encoder", + "rel": "composes" + }, + { + "source": "concept:ssl", + "target": "validation:trace_vision_language_pretrained_dual_encoder", + "rel": "composes" + }, + { + "source": "insight:contrastive_alignment_creates_zero_shot_transfer", + "target": "validation:trace_vision_language_pretrained_dual_encoder", + "rel": "composes" + }, + { + "source": "insight:foundation_pretraining_decouples_data_from_task", + "target": "validation:trace_vision_language_pretrained_dual_encoder", + "rel": "composes" + }, + { + "source": "validation:trace_vision_language_pretrained_dual_encoder", + "target": "paper:llava", + "rel": "validates" + }, + { + "source": "concept:imitation_learning", + "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", + "rel": "composes" + }, + { + "source": "move:diffusion_denoise_sampling", + "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", + "rel": "composes" + }, + { + "source": "paper:diffuser", + "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", + "rel": "composes" + }, + { + "source": "insight:diffusion_unifies_generation_and_decision", + "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", + "rel": "composes" + }, + { + "source": "insight:imitation_data_compresses_unspecified_reward", + "target": "validation:trace_diffusion_policy_as_score_based_action_sampler", + "rel": "composes" + }, + { + "source": "validation:trace_diffusion_policy_as_score_based_action_sampler", + "target": "paper:diffuser", + "rel": "validates" + }, + { + "source": "paper:vit", + "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "rel": "composes" + }, + { + "source": "paper:carion2020", + "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "rel": "composes" + }, + { + "source": "paper:li2022bevformer", + "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "rel": "composes" + }, + { + "source": "concept:bev", + "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "rel": "composes" + }, + { + "source": "concept:detr_query", + "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "rel": "composes" + }, + { + "source": "move:lift_2d_to_3d", + "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "rel": "composes" + }, + { + "source": "paper:tesla_ai_day", + "target": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "rel": "composes" + }, + { + "source": "validation:trace_modular_perception_pipeline_with_bev_fusion", + "target": "paper:li2022bevformer", + "rel": "validates" + }, + { + "source": "paper:vit", + "target": "validation:trace_neural_field_for_dynamic_driving_scene", + "rel": "composes" + }, + { + "source": "paper:gaia1", + "target": "validation:trace_neural_field_for_dynamic_driving_scene", + "rel": "composes" + }, + { + "source": "paper:drivedreamer", + "target": "validation:trace_neural_field_for_dynamic_driving_scene", + "rel": "composes" + }, + { + "source": "paper:world_models", + "target": "validation:trace_neural_field_for_dynamic_driving_scene", + "rel": "composes" + }, + { + "source": "move:latent_imagination_rollout", + "target": "validation:trace_neural_field_for_dynamic_driving_scene", + "rel": "composes" + }, + { + "source": "insight:long_tail_solved_by_synthesis_not_data_alone", + "target": "validation:trace_neural_field_for_dynamic_driving_scene", + "rel": "composes" + }, + { + "source": "insight:world_models_let_planning_be_done_in_imagination", + "target": "validation:trace_neural_field_for_dynamic_driving_scene", + "rel": "composes" + }, + { + "source": "validation:trace_neural_field_for_dynamic_driving_scene", + "target": "paper:gaia1", + "rel": "validates" + }, + { + "source": "paper:vaswani2017", + "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "rel": "composes" + }, + { + "source": "paper:gpt3", + "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "rel": "composes" + }, + { + "source": "concept:transformer", + "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "rel": "composes" + }, + { + "source": "concept:imitation_learning", + "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "rel": "composes" + }, + { + "source": "move:tokenize_modalities", + "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "rel": "composes" + }, + { + "source": "insight:tokenization_collapses_modality_gap", + "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "rel": "composes" + }, + { + "source": "insight:imitation_data_compresses_unspecified_reward", + "target": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "rel": "composes" + }, + { + "source": "validation:trace_decision_transformer_offline_rl_via_sequence_modeling", + "target": "paper:diffuser", + "rel": "validates" + }, + { + "source": "concept:mdp", + "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", + "rel": "composes" + }, + { + "source": "concept:policy_gradient", + "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", + "rel": "composes" + }, + { + "source": "concept:actor_critic", + "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", + "rel": "composes" + }, + { + "source": "concept:ppo", + "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", + "rel": "composes" + }, + { + "source": "move:clipped_surrogate_objective", + "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", + "rel": "composes" + }, + { + "source": "insight:safety_constraints_via_lagrangian_dual", + "target": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", + "rel": "composes" + }, + { + "source": "validation:trace_safe_rl_via_lagrangian_constrained_optimization", + "target": "paper:schulman2017_ppo", + "rel": "validates" + }, + { + "source": "paper:tesla_ai_day", + "target": "paradigm:modular_perception_to_planning_pipeline", + "rel": "manifests" + }, + { + "source": "paper:li2022bevformer", + "target": "paradigm:modular_perception_to_planning_pipeline", + "rel": "manifests" + }, + { + "source": "paper:transfuser", + "target": "paradigm:modular_perception_to_planning_pipeline", + "rel": "manifests" + }, + { + "source": "concept:bev", + "target": "paradigm:modular_perception_to_planning_pipeline", + "rel": "composes" + }, + { + "source": "move:lift_2d_to_3d", + "target": "paradigm:modular_perception_to_planning_pipeline", + "rel": "composes" + }, + { + "source": "insight:set_prediction_eliminates_postprocessing_heuristics", + "target": "paradigm:modular_perception_to_planning_pipeline", + "rel": "composes" + }, + { + "source": "paper:2212.10156", + "target": "paradigm:differentiable_end_to_end_imitation", + "rel": "manifests" + }, + { + "source": "paper:2210.14222", + "target": "paradigm:differentiable_end_to_end_imitation", + "rel": "manifests" + }, + { + "source": "paper:vadv2", + "target": "paradigm:differentiable_end_to_end_imitation", + "rel": "manifests" + }, + { + "source": "paper:transfuser", + "target": "paradigm:differentiable_end_to_end_imitation", + "rel": "manifests" + }, + { + "source": "concept:imitation_learning", + "target": "paradigm:differentiable_end_to_end_imitation", + "rel": "composes" + }, + { + "source": "move:cross_attention_query", + "target": "paradigm:differentiable_end_to_end_imitation", + "rel": "composes" + }, + { + "source": "insight:end_to_end_differentiable_beats_handcraft_when_signal_strong", + "target": "paradigm:differentiable_end_to_end_imitation", + "rel": "composes" + }, + { + "source": "paper:world_models", + "target": "paradigm:model_based_world_imagination_planning", + "rel": "manifests" + }, + { + "source": "paper:gaia1", + "target": "paradigm:model_based_world_imagination_planning", + "rel": "manifests" + }, + { + "source": "paper:drivedreamer", + "target": "paradigm:model_based_world_imagination_planning", + "rel": "manifests" + }, + { + "source": "move:latent_imagination_rollout", + "target": "paradigm:model_based_world_imagination_planning", + "rel": "composes" + }, + { + "source": "move:diffusion_denoise_sampling", + "target": "paradigm:model_based_world_imagination_planning", + "rel": "composes" + }, + { + "source": "insight:world_models_let_planning_be_done_in_imagination", + "target": "paradigm:model_based_world_imagination_planning", + "rel": "composes" + }, + { + "source": "insight:long_tail_solved_by_synthesis_not_data_alone", + "target": "paradigm:model_based_world_imagination_planning", + "rel": "composes" + }, + { + "source": "paper:gpt3", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "manifests" + }, + { + "source": "paper:llava", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "manifests" + }, + { + "source": "paper:2402.12289", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "manifests" + }, + { + "source": "paper:2311.10813", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "manifests" + }, + { + "source": "paper:lingo2", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "manifests" + }, + { + "source": "concept:vla", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "composes" + }, + { + "source": "concept:tool_use", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "composes" + }, + { + "source": "move:dual_system_fast_slow", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "composes" + }, + { + "source": "insight:dual_system_handles_latency_quality_tradeoff", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "composes" + }, + { + "source": "insight:in_context_learning_emerges_at_scale", + "target": "paradigm:foundation_model_zero_shot_driving_agent", + "rel": "composes" + }, + { + "source": "concept:spiking_nn", + "target": "paradigm:brain_inspired_event_sparse_compute", + "rel": "composes" + }, + { + "source": "paper:2307.01694", + "target": "paradigm:brain_inspired_event_sparse_compute", + "rel": "manifests" + }, + { + "source": "move:spike_event_compute", + "target": "paradigm:brain_inspired_event_sparse_compute", + "rel": "composes" + }, + { + "source": "insight:event_sparse_compute_matches_energy_budget", + "target": "paradigm:brain_inspired_event_sparse_compute", + "rel": "composes" + }, + { + "source": "paper:2512.24426", + "target": "paradigm:counterfactual_data_centric_safety", + "rel": "manifests" + }, + { + "source": "paper:gaia1", + "target": "paradigm:counterfactual_data_centric_safety", + "rel": "manifests" + }, + { + "source": "paper:drivedreamer", + "target": "paradigm:counterfactual_data_centric_safety", + "rel": "manifests" + }, + { + "source": "concept:counterfactual", + "target": "paradigm:counterfactual_data_centric_safety", + "rel": "composes" + }, + { + "source": "concept:meta_action", + "target": "paradigm:counterfactual_data_centric_safety", + "rel": "composes" + }, + { + "source": "move:counterfactual_replan", + "target": "paradigm:counterfactual_data_centric_safety", + "rel": "composes" + }, + { + "source": "insight:long_tail_solved_by_synthesis_not_data_alone", + "target": "paradigm:counterfactual_data_centric_safety", + "rel": "composes" + }, + { + "source": "paper:gpt3", + "target": "paradigm:knowledge_driven_reflective_agent", + "rel": "manifests" + }, + { + "source": "paper:2309.16292", + "target": "paradigm:knowledge_driven_reflective_agent", + "rel": "manifests" + }, + { + "source": "paper:2311.10813", + "target": "paradigm:knowledge_driven_reflective_agent", + "rel": "manifests" + }, + { + "source": "concept:cot", + "target": "paradigm:knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "concept:tool_use", + "target": "paradigm:knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "essay:bitter_lesson", + "target": "paradigm:knowledge_driven_reflective_agent", + "rel": "contrasts" + }, + { + "source": "insight:symbolic_intermediate_enables_interpretability_and_transfer", + "target": "paradigm:knowledge_driven_reflective_agent", + "rel": "composes" + }, + { + "source": "paper:dinov2", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "manifests" + }, + { + "source": "paper:2508.10104", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "manifests" + }, + { + "source": "paper:sam", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "manifests" + }, + { + "source": "concept:ssl", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "composes" + }, + { + "source": "move:masking_for_pretext", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "composes" + }, + { + "source": "move:contrastive_alignment", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "composes" + }, + { + "source": "insight:masked_prediction_yields_self_supervised_signal", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "composes" + }, + { + "source": "insight:foundation_pretraining_decouples_data_from_task", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "composes" + }, + { + "source": "insight:scaling_laws_predict_capability_emergence", + "target": "paradigm:scaling_data_with_self_supervision", + "rel": "composes" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:nuplan", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:waymo_motion", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:argoverse2", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:navsim", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:bench2drive", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:carla_lb2", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:womd_pred", + "rel": "covers" + }, + { + "source": "paper:ad_benchmarks", + "target": "paper:interaction_dataset", + "rel": "covers" + }, + { + "source": "paper:nuplan", + "target": "paper:navsim", + "rel": "feeds" + }, + { + "source": "paper:nuplan", + "target": "paper:bench2drive", + "rel": "parallel" + }, + { + "source": "paper:navsim", + "target": "paper:bench2drive", + "rel": "parallel" + }, + { + "source": "paper:carla_lb2", + "target": "paper:bench2drive", + "rel": "feeds" + }, + { + "source": "paper:waymo_motion", + "target": "paper:womd_pred", + "rel": "feeds" + }, + { + "source": "paper:argoverse2", + "target": "paper:womd_pred", + "rel": "parallel" + }, + { + "source": "paper:interaction_dataset", + "target": "paper:waymo_motion", + "rel": "parallel" + }, + { + "source": "paper:bdd100k", + "target": "paper:argoverse2", + "rel": "parallel" + }, + { + "source": "paper:lyft_l5", + "target": "paper:waymo_motion", + "rel": "parallel" + }, + { + "source": "paper:pandaset", + "target": "paper:argoverse2", + "rel": "parallel" + }, + { + "source": "paper:apolloscape", + "target": "paper:bdd100k", + "rel": "parallel" + }, + { + "source": "paper:highway_env", + "target": "paper:metadrive", + "rel": "prereq" + }, + { + "source": "paper:metadrive", + "target": "paper:smarts", + "rel": "parallel" + }, + { + "source": "paper:smarts", + "target": "paper:commonroad", + "rel": "parallel" + }, + { + "source": "paper:commonroad", + "target": "paper:carla_lb2", + "rel": "parallel" + }, + { + "source": "paper:highway_env", + "target": "paper:smarts", + "rel": "prereq" + }, + { + "source": "paper:tesla_ai_day", + "target": "paper:tesla_autolabel", + "rel": "covers" + }, + { + "source": "paper:tesla_autolabel", + "target": "paper:waymo_scenario_mining", + "rel": "parallel" + }, + { + "source": "paper:tesla_autolabel", + "target": "paper:gs_for_ad", + "rel": "feeds" + }, + { + "source": "paper:gs_for_ad", + "target": "paper:shift_dataset", + "rel": "parallel" + }, + { + "source": "paper:shift_dataset", + "target": "paper:v2x_sim", + "rel": "parallel" + }, + { + "source": "paper:vaswani2017", + "target": "paper:flashattention", + "rel": "prereq" + }, + { + "source": "paper:vaswani2017", + "target": "paper:performer", + "rel": "prereq" + }, + { + "source": "paper:vaswani2017", + "target": "paper:linear_attention", + "rel": "prereq" + }, + { + "source": "paper:performer", + "target": "paper:linear_attention", + "rel": "parallel" + }, + { + "source": "paper:flashattention", + "target": "paper:performer", + "rel": "contrasts" + }, + { + "source": "paper:flashattention", + "target": "paper:mamba", + "rel": "contrasts" + }, + { + "source": "paper:linear_attention", + "target": "paper:mamba", + "rel": "prereq" + }, + { + "source": "paper:llava", + "target": "paper:distill_vlm", + "rel": "feeds" + }, + { + "source": "paper:gptq", + "target": "paper:awq", + "rel": "parallel" + }, + { + "source": "paper:gpt3", + "target": "paper:gptq", + "rel": "feeds" + }, + { + "source": "paper:llava", + "target": "paper:gptq", + "rel": "feeds" + }, + { + "source": "paper:llava", + "target": "paper:awq", + "rel": "feeds" + }, + { + "source": "paper:distill_vlm", + "target": "paper:2402.12289", + "rel": "feeds" + }, + { + "source": "paper:2307.01694", + "target": "paper:loihi2", + "rel": "feeds" + }, + { + "source": "paper:loihi2", + "target": "paper:truenorth", + "rel": "extends" + }, + { + "source": "paper:truenorth", + "target": "paper:tianjic", + "rel": "parallel" + }, + { + "source": "paper:loihi2", + "target": "paper:tianjic", + "rel": "parallel" + }, + { + "source": "paper:loihi2", + "target": "paper:grai", + "rel": "parallel" + }, + { + "source": "paper:dvs_event_camera", + "target": "paper:loihi2", + "rel": "feeds" + }, + { + "source": "paper:dvs_event_camera", + "target": "paper:2307.01694", + "rel": "feeds" + }, + { + "source": "paper:iso26262", + "target": "paper:sotif_21448", + "rel": "prereq" + }, + { + "source": "paper:sotif_21448", + "target": "paper:ul4600", + "rel": "extends" + }, + { + "source": "paper:iso26262", + "target": "paper:ul4600", + "rel": "prereq" + }, + { + "source": "paper:nuplan", + "target": "move:design_closed_loop_metric_correlated_with_real_world_safety", + "rel": "feeds" + }, + { + "source": "paper:navsim", + "target": "move:design_closed_loop_metric_correlated_with_real_world_safety", + "rel": "feeds" + }, + { + "source": "paper:bench2drive", + "target": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", + "rel": "feeds" + }, + { + "source": "paper:navsim", + "target": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", + "rel": "feeds" + }, + { + "source": "paper:nuplan", + "target": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", + "rel": "feeds" + }, + { + "source": "paper:waymo_scenario_mining", + "target": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", + "rel": "feeds" + }, + { + "source": "paper:tesla_autolabel", + "target": "move:auto_label_with_offline_model_then_human_in_loop_validate", + "rel": "feeds" + }, + { + "source": "paper:tesla_autolabel", + "target": "move:augment_dataset_via_offline_scenario_perturbation", + "rel": "feeds" + }, + { + "source": "paper:gs_for_ad", + "target": "move:augment_dataset_via_offline_scenario_perturbation", + "rel": "feeds" + }, + { + "source": "paper:gs_for_ad", + "target": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", + "rel": "feeds" + }, + { + "source": "paper:lidar_cam_calib", + "target": "move:share_LiDAR_camera_calibration_via_continuous_time_optimization", + "rel": "feeds" + }, + { + "source": "paper:flashattention", + "target": "move:tile_attention_to_fit_SRAM_for_speedup", + "rel": "covers" + }, + { + "source": "paper:performer", + "target": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", + "rel": "covers" + }, + { + "source": "paper:linear_attention", + "target": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", + "rel": "covers" + }, + { + "source": "paper:gptq", + "target": "move:quantize_attention_to_int8_with_calibration", + "rel": "feeds" + }, + { + "source": "paper:awq", + "target": "move:quantize_attention_to_int8_with_calibration", + "rel": "feeds" + }, + { + "source": "paper:distill_vlm", + "target": "move:distill_large_VLM_into_small_realtime_specialist", + "rel": "covers" + }, + { + "source": "paper:flashattention", + "target": "move:cache_KV_state_across_frames_to_amortize_attention_cost", + "rel": "feeds" + }, + { + "source": "paper:mamba", + "target": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", + "rel": "parallel" + }, + { + "source": "paper:2307.01694", + "target": "move:replace_dense_attention_with_sparse_event_driven_attention", + "rel": "covers" + }, + { + "source": "paper:2307.01694", + "target": "move:implement_spiking_neuron_with_surrogate_gradient_for_backprop", + "rel": "feeds" + }, + { + "source": "paper:loihi2", + "target": "move:co_design_silicon_with_algorithm_for_minimum_energy", + "rel": "feeds" + }, + { + "source": "paper:tianjic", + "target": "move:co_design_silicon_with_algorithm_for_minimum_energy", + "rel": "feeds" + }, + { + "source": "paper:grai", + "target": "move:co_design_silicon_with_algorithm_for_minimum_energy", + "rel": "feeds" + }, + { + "source": "paper:dvs_event_camera", + "target": "move:use_event_camera_microsecond_latency_for_emergency_braking", + "rel": "feeds" + }, + { + "source": "paper:dvs_event_camera", + "target": "move:replace_dense_attention_with_sparse_event_driven_attention", + "rel": "feeds" + }, + { + "source": "paper:iso26262", + "target": "move:formalize_safety_case_with_claim_evidence_assumption", + "rel": "feeds" + }, + { + "source": "paper:sotif_21448", + "target": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", + "rel": "feeds" + }, + { + "source": "paper:sotif_21448", + "target": "move:treat_corner_case_as_OOD_detection_then_route_to_human", + "rel": "feeds" + }, + { + "source": "paper:ul4600", + "target": "move:formalize_safety_case_with_claim_evidence_assumption", + "rel": "covers" + }, + { + "source": "paper:sotif_21448", + "target": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", + "rel": "feeds" + }, + { + "source": "paper:sotif_21448", + "target": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", + "rel": "feeds" + }, + { + "source": "paper:ul4600", + "target": "move:add_explanation_head_to_promote_interpretability", + "rel": "feeds" + }, + { + "source": "paper:metadrive", + "target": "move:use_difficulty_aware_curriculum_to_accelerate_RL", + "rel": "feeds" + }, + { + "source": "paper:smarts", + "target": "move:use_difficulty_aware_curriculum_to_accelerate_RL", + "rel": "feeds" + }, + { + "source": "paper:metadrive", + "target": "move:replay_buffer_prioritize_safety_critical_transitions", + "rel": "feeds" + }, + { + "source": "paper:highway_env", + "target": "move:replay_buffer_prioritize_safety_critical_transitions", + "rel": "feeds" + }, + { + "source": "paper:commonroad", + "target": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", + "rel": "feeds" + }, + { + "source": "paper:waymo_scenario_mining", + "target": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", + "rel": "feeds" + }, + { + "source": "paper:shift_dataset", + "target": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", + "rel": "feeds" + }, + { + "source": "move:design_closed_loop_metric_correlated_with_real_world_safety", + "target": "problem:offline_metric_does_not_predict_closed_loop_performance", + "rel": "motivates" + }, + { + "source": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", + "target": "problem:offline_metric_does_not_predict_closed_loop_performance", + "rel": "motivates" + }, + { + "source": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", + "target": "problem:offline_metric_does_not_predict_closed_loop_performance", + "rel": "motivates" + }, + { + "source": "move:augment_dataset_via_offline_scenario_perturbation", + "target": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented", + "rel": "motivates" + }, + { + "source": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", + "target": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented", + "rel": "motivates" + }, + { + "source": "move:replay_buffer_prioritize_safety_critical_transitions", + "target": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented", + "rel": "motivates" + }, + { + "source": "move:distill_large_VLM_into_small_realtime_specialist", + "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "rel": "motivates" + }, + { + "source": "move:quantize_attention_to_int8_with_calibration", + "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "rel": "motivates" + }, + { + "source": "move:tile_attention_to_fit_SRAM_for_speedup", + "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "rel": "motivates" + }, + { + "source": "move:cache_KV_state_across_frames_to_amortize_attention_cost", + "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "rel": "motivates" + }, + { + "source": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", + "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "rel": "motivates" + }, + { + "source": "move:replace_dense_attention_with_sparse_event_driven_attention", + "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "rel": "motivates" + }, + { + "source": "move:perform_neural_architecture_search_with_latency_constraint", + "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "rel": "motivates" + }, + { + "source": "move:co_design_silicon_with_algorithm_for_minimum_energy", + "target": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "rel": "motivates" + }, + { + "source": "move:share_LiDAR_camera_calibration_via_continuous_time_optimization", + "target": "problem:sensor_calibration_drift_over_vehicle_lifetime", + "rel": "motivates" + }, + { + "source": "move:auto_label_with_offline_model_then_human_in_loop_validate", + "target": "problem:label_noise_for_3d_object_categories", + "rel": "motivates" + }, + { + "source": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", + "target": "problem:label_noise_for_3d_object_categories", + "rel": "motivates" + }, + { + "source": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", + "target": "problem:verification_of_neural_network_safety_properties_at_scale", + "rel": "motivates" + }, + { + "source": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", + "target": "problem:verification_of_neural_network_safety_properties_at_scale", + "rel": "motivates" + }, + { + "source": "move:augment_dataset_via_offline_scenario_perturbation", + "target": "problem:realistic_other_agent_behavior_in_simulator", + "rel": "motivates" + }, + { + "source": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", + "target": "problem:catastrophic_forgetting_under_continual_learning", + "rel": "motivates" + }, + { + "source": "move:add_explanation_head_to_promote_interpretability", + "target": "problem:auditability_of_decisions_for_regulatory_compliance", + "rel": "motivates" + }, + { + "source": "move:formalize_safety_case_with_claim_evidence_assumption", + "target": "problem:auditability_of_decisions_for_regulatory_compliance", + "rel": "motivates" + }, + { + "source": "move:treat_corner_case_as_OOD_detection_then_route_to_human", + "target": "problem:auditability_of_decisions_for_regulatory_compliance", + "rel": "motivates" + }, + { + "source": "paper:gs_for_ad", + "target": "problem:simulator_visual_gap_breaks_perception_models", + "rel": "motivates" + }, + { + "source": "paper:shift_dataset", + "target": "problem:simulator_visual_gap_breaks_perception_models", + "rel": "motivates" + }, + { + "source": "paper:v2x_sim", + "target": "problem:realistic_other_agent_behavior_in_simulator", + "rel": "motivates" + }, + { + "source": "problem:offline_metric_does_not_predict_closed_loop_performance", + "target": "insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners", + "rel": "composes" + }, + { + "source": "problem:rare_safety_critical_events_dominate_real_risk_but_are_under_represented", + "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset", + "rel": "composes" + }, + { + "source": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "target": "insight:event_driven_computation_matches_natural_sparsity_of_driving_scene", + "rel": "composes" + }, + { + "source": "problem:energy_budget_too_small_for_full_transformer_at_30fps", + "target": "insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency", + "rel": "composes" + }, + { + "source": "problem:verification_of_neural_network_safety_properties_at_scale", + "target": "insight:safety_emerges_from_layered_constraints_not_single_objective", + "rel": "composes" + }, + { + "source": "problem:auditability_of_decisions_for_regulatory_compliance", + "target": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation", + "rel": "composes" + }, + { + "source": "problem:realistic_other_agent_behavior_in_simulator", + "target": "insight:simulator_realism_is_lower_bound_on_training_value", + "rel": "composes" + }, + { + "source": "problem:simulator_visual_gap_breaks_perception_models", + "target": "insight:simulator_realism_is_lower_bound_on_training_value", + "rel": "composes" + }, + { + "source": "problem:catastrophic_forgetting_under_continual_learning", + "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset", + "rel": "composes" + }, + { + "source": "problem:sensor_calibration_drift_over_vehicle_lifetime", + "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset", + "rel": "composes" + }, + { + "source": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", + "target": "insight:offline_metrics_co_evolve_with_methods_so_must_be_re_audited", + "rel": "composes" + }, + { + "source": "move:design_closed_loop_metric_correlated_with_real_world_safety", + "target": "insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners", + "rel": "composes" + }, + { + "source": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", + "target": "insight:safety_emerges_from_layered_constraints_not_single_objective", + "rel": "composes" + }, + { + "source": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", + "target": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation", + "rel": "composes" + }, + { + "source": "move:treat_corner_case_as_OOD_detection_then_route_to_human", + "target": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation", + "rel": "composes" + }, + { + "source": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", + "target": "insight:safety_emerges_from_layered_constraints_not_single_objective", + "rel": "composes" + }, + { + "source": "move:co_design_silicon_with_algorithm_for_minimum_energy", + "target": "insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency", + "rel": "composes" + }, + { + "source": "move:replace_dense_attention_with_sparse_event_driven_attention", + "target": "insight:event_driven_computation_matches_natural_sparsity_of_driving_scene", + "rel": "composes" + }, + { + "source": "move:auto_label_with_offline_model_then_human_in_loop_validate", + "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset", + "rel": "composes" + }, + { + "source": "move:augment_dataset_via_offline_scenario_perturbation", + "target": "insight:simulator_realism_is_lower_bound_on_training_value", + "rel": "composes" + }, + { + "source": "insight:closed_loop_evaluation_is_the_only_ground_truth_for_planners", + "target": "paradigm:closed_loop_data_engine_centric_development", + "rel": "composes" + }, + { + "source": "insight:data_engine_loop_is_more_valuable_than_static_dataset", + "target": "paradigm:closed_loop_data_engine_centric_development", + "rel": "composes" + }, + { + "source": "insight:offline_metrics_co_evolve_with_methods_so_must_be_re_audited", + "target": "paradigm:closed_loop_data_engine_centric_development", + "rel": "composes" + }, + { + "source": "insight:safety_emerges_from_layered_constraints_not_single_objective", + "target": "paradigm:safety_by_constraint_layered_architecture", + "rel": "composes" + }, + { + "source": "insight:uncertainty_calibration_is_prerequisite_for_safe_delegation", + "target": "paradigm:safety_by_constraint_layered_architecture", + "rel": "composes" + }, + { + "source": "insight:event_driven_computation_matches_natural_sparsity_of_driving_scene", + "target": "paradigm:brain_inspired_neuromorphic_co_design", + "rel": "composes" + }, + { + "source": "insight:hardware_software_co_design_unlocks_orders_of_magnitude_efficiency", + "target": "paradigm:brain_inspired_neuromorphic_co_design", + "rel": "composes" + }, + { + "source": "insight:simulator_realism_is_lower_bound_on_training_value", + "target": "paradigm:simulator_first_synthetic_data_centric", + "rel": "composes" + }, + { + "source": "paradigm:closed_loop_data_engine_centric_development", + "target": "paper:tesla_autolabel", + "rel": "covers" + }, + { + "source": "paradigm:closed_loop_data_engine_centric_development", + "target": "paper:waymo_scenario_mining", + "rel": "covers" + }, + { + "source": "paradigm:closed_loop_data_engine_centric_development", + "target": "paper:nuplan", + "rel": "covers" + }, + { + "source": "paradigm:closed_loop_data_engine_centric_development", + "target": "paper:bench2drive", + "rel": "covers" + }, + { + "source": "paradigm:safety_by_constraint_layered_architecture", + "target": "paper:iso26262", + "rel": "covers" + }, + { + "source": "paradigm:safety_by_constraint_layered_architecture", + "target": "paper:sotif_21448", + "rel": "covers" + }, + { + "source": "paradigm:safety_by_constraint_layered_architecture", + "target": "paper:ul4600", + "rel": "covers" + }, + { + "source": "paradigm:brain_inspired_neuromorphic_co_design", + "target": "paper:2307.01694", + "rel": "covers" + }, + { + "source": "paradigm:brain_inspired_neuromorphic_co_design", + "target": "paper:loihi2", + "rel": "covers" + }, + { + "source": "paradigm:brain_inspired_neuromorphic_co_design", + "target": "paper:tianjic", + "rel": "covers" + }, + { + "source": "paradigm:brain_inspired_neuromorphic_co_design", + "target": "paper:dvs_event_camera", + "rel": "covers" + }, + { + "source": "paradigm:simulator_first_synthetic_data_centric", + "target": "paper:carla_lb2", + "rel": "covers" + }, + { + "source": "paradigm:simulator_first_synthetic_data_centric", + "target": "paper:shift_dataset", + "rel": "covers" + }, + { + "source": "paradigm:simulator_first_synthetic_data_centric", + "target": "paper:v2x_sim", + "rel": "covers" + }, + { + "source": "paradigm:simulator_first_synthetic_data_centric", + "target": "paper:gs_for_ad", + "rel": "covers" + }, + { + "source": "paradigm:simulator_first_synthetic_data_centric", + "target": "paper:metadrive", + "rel": "covers" + }, + { + "source": "essay:bitter_lesson", + "target": "paradigm:closed_loop_data_engine_centric_development", + "rel": "parallel" + }, + { + "source": "essay:bitter_lesson", + "target": "paradigm:simulator_first_synthetic_data_centric", + "rel": "parallel" + }, + { + "source": "essay:bitter_lesson", + "target": "paradigm:brain_inspired_neuromorphic_co_design", + "rel": "contrasts" + }, + { + "source": "essay:bitter_lesson", + "target": "insight:data_engine_loop_is_more_valuable_than_static_dataset", + "rel": "parallel" + }, + { + "source": "paper:2212.10156", + "target": "paper:bench2drive", + "rel": "covers" + }, + { + "source": "paper:vadv2", + "target": "paper:nuplan", + "rel": "covers" + }, + { + "source": "paper:transfuser", + "target": "paper:carla_lb2", + "rel": "covers" + }, + { + "source": "paper:2402.12289", + "target": "paper:distill_vlm", + "rel": "feeds" + }, + { + "source": "paper:diffuser", + "target": "paper:nuplan", + "rel": "parallel" + }, + { + "source": "concept:spiking_nn", + "target": "paper:loihi2", + "rel": "covers" + }, + { + "source": "concept:spiking_nn", + "target": "paper:tianjic", + "rel": "covers" + }, + { + "source": "concept:spiking_nn", + "target": "paper:truenorth", + "rel": "covers" + }, + { + "source": "concept:spiking_nn", + "target": "move:implement_spiking_neuron_with_surrogate_gradient_for_backprop", + "rel": "covers" + }, + { + "source": "concept:self_attention", + "target": "paper:flashattention", + "rel": "covers" + }, + { + "source": "concept:self_attention", + "target": "paper:performer", + "rel": "covers" + }, + { + "source": "concept:transformer", + "target": "paper:flashattention", + "rel": "covers" + }, + { + "source": "lab:lab06", + "target": "move:implement_spiking_neuron_with_surrogate_gradient_for_backprop", + "rel": "implements" + }, + { + "source": "lab:lab06", + "target": "move:replace_dense_attention_with_sparse_event_driven_attention", + "rel": "implements" + }, + { + "source": "lab:lab02", + "target": "move:use_difficulty_aware_curriculum_to_accelerate_RL", + "rel": "parallel" + }, + { + "source": "lab:lab03", + "target": "move:cache_KV_state_across_frames_to_amortize_attention_cost", + "rel": "parallel" + }, + { + "source": "lab:lab04", + "target": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", + "rel": "parallel" + }, + { + "source": "move:specify_safety_constraint_as_signal_temporal_logic_then_verify", + "target": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", + "rel": "feeds" + }, + { + "source": "move:add_shield_layer_that_rejects_unsafe_actions_at_inference", + "target": "move:treat_corner_case_as_OOD_detection_then_route_to_human", + "rel": "parallel" + }, + { + "source": "move:apply_uncertainty_quantification_via_deep_ensemble_or_evidential_layer", + "target": "move:treat_corner_case_as_OOD_detection_then_route_to_human", + "rel": "feeds" + }, + { + "source": "move:tile_attention_to_fit_SRAM_for_speedup", + "target": "move:cache_KV_state_across_frames_to_amortize_attention_cost", + "rel": "parallel" + }, + { + "source": "move:replace_softmax_attention_with_linear_kernel_for_long_sequence", + "target": "move:replace_dense_attention_with_sparse_event_driven_attention", + "rel": "parallel" + }, + { + "source": "move:quantize_attention_to_int8_with_calibration", + "target": "move:perform_neural_architecture_search_with_latency_constraint", + "rel": "parallel" + }, + { + "source": "move:distill_large_VLM_into_small_realtime_specialist", + "target": "move:quantize_attention_to_int8_with_calibration", + "rel": "feeds" + }, + { + "source": "move:augment_dataset_via_offline_scenario_perturbation", + "target": "move:run_replay_simulation_with_perturbed_initial_conditions_for_robustness", + "rel": "feeds" + }, + { + "source": "move:auto_label_with_offline_model_then_human_in_loop_validate", + "target": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", + "rel": "feeds" + }, + { + "source": "move:run_active_learning_loop_to_query_hardest_unlabeled_frames", + "target": "move:run_continual_learning_with_rehearsal_buffer_against_forgetting", + "rel": "feeds" + }, + { + "source": "move:share_LiDAR_camera_calibration_via_continuous_time_optimization", + "target": "move:auto_label_with_offline_model_then_human_in_loop_validate", + "rel": "prereq" + }, + { + "source": "move:design_closed_loop_metric_correlated_with_real_world_safety", + "target": "move:track_metric_correlation_offline_vs_closed_loop_to_select_models", + "rel": "feeds" + }, + { + "source": "move:formalize_safety_case_with_claim_evidence_assumption", + "target": "move:add_explanation_head_to_promote_interpretability", + "rel": "feeds" + }, + { + "source": "paper:lift_splat_shoot", + "target": "paper:bevdet", + "rel": "prereq" + }, + { + "source": "paper:bevdet", + "target": "paper:bevdet4d", + "rel": "extends" + }, + { + "source": "paper:lift_splat_shoot", + "target": "paper:li2022bevformer", + "rel": "parallel" + }, + { + "source": "paper:li2022bevformer", + "target": "paper:bevformer_v2", + "rel": "extends" + }, + { + "source": "paper:carion2020", + "target": "paper:detr3d", + "rel": "prereq" + }, + { + "source": "paper:detr3d", + "target": "paper:petr", + "rel": "parallel" + }, + { + "source": "paper:petr", + "target": "paper:petrv2", + "rel": "extends" + }, + { + "source": "paper:petr", + "target": "paper:streampetr", + "rel": "extends" + }, + { + "source": "paper:detr3d", + "target": "paper:streampetr", + "rel": "prereq" + }, + { + "source": "paper:bevdet", + "target": "paper:bevfusion", + "rel": "feeds" + }, + { + "source": "paper:lift_splat_shoot", + "target": "paper:bevfusion", + "rel": "prereq" + }, + { + "source": "paper:lift_splat_shoot", + "target": "paper:simplebev", + "rel": "contrasts" + }, + { + "source": "paper:simplebev", + "target": "paper:bevdet", + "rel": "contrasts" + }, + { + "source": "paper:occupancy_networks_tesla", + "target": "paper:surroundocc", + "rel": "feeds" + }, + { + "source": "paper:tesla_ai_day", + "target": "paper:occupancy_networks_tesla", + "rel": "covers" + }, + { + "source": "paper:surroundocc", + "target": "paper:occ3d", + "rel": "parallel" + }, + { + "source": "paper:occ3d", + "target": "paper:openocc_unic", + "rel": "parallel" + }, + { + "source": "paper:li2022bevformer", + "target": "paper:surroundocc", + "rel": "prereq" + }, + { + "source": "paper:vit", + "target": "paper:mae", + "rel": "prereq" + }, + { + "source": "paper:vit", + "target": "paper:beit", + "rel": "prereq" + }, + { + "source": "paper:mae", + "target": "paper:dinov2", + "rel": "parallel" + }, + { + "source": "paper:beit", + "target": "paper:mae", + "rel": "parallel" + }, + { + "source": "paper:vit", + "target": "paper:clip", + "rel": "prereq" + }, + { + "source": "paper:clip", + "target": "paper:blip2", + "rel": "prereq" + }, + { + "source": "paper:clip", + "target": "paper:vilt", + "rel": "parallel" + }, + { + "source": "paper:blip2", + "target": "paper:llava", + "rel": "prereq" + }, + { + "source": "paper:vit", + "target": "paper:dinov1", + "rel": "prereq" + }, + { + "source": "paper:dinov1", + "target": "paper:dinov2", + "rel": "prereq" + }, + { + "source": "paper:simclr_mocov3", + "target": "paper:dinov1", + "rel": "contrasts" + }, + { + "source": "paper:simclr_mocov3", + "target": "paper:mae", + "rel": "contrasts" + }, + { + "source": "paper:nerf", + "target": "paper:3dgs", + "rel": "contrasts" + }, + { + "source": "paper:nerf", + "target": "paper:emernerf", + "rel": "prereq" + }, + { + "source": "paper:3dgs", + "target": "paper:drivinggaussian", + "rel": "prereq" + }, + { + "source": "paper:emernerf", + "target": "paper:drivinggaussian", + "rel": "parallel" + }, + { + "source": "paper:dinov2", + "target": "paper:depth_anything", + "rel": "prereq" + }, + { + "source": "paper:depth_anything", + "target": "paper:lift_splat_shoot", + "rel": "feeds" + }, + { + "source": "paper:depth_anything", + "target": "paper:bevdet", + "rel": "feeds" + }, + { + "source": "paper:vit", + "target": "paper:vggt", + "rel": "prereq" + }, + { + "source": "paper:vggt", + "target": "paper:3dgs", + "rel": "feeds" + }, + { + "source": "paper:li2022bevformer", + "target": "paper:2212.10156", + "rel": "feeds" + }, + { + "source": "paper:bevfusion", + "target": "paper:2212.10156", + "rel": "feeds" + }, + { + "source": "paper:streampetr", + "target": "paper:2212.10156", + "rel": "feeds" + }, + { + "source": "paper:occupancy_networks_tesla", + "target": "paper:2212.10156", + "rel": "feeds" + }, + { + "source": "paper:occupancy_networks_tesla", + "target": "paper:vadv2", + "rel": "feeds" + }, + { + "source": "paper:dinov2", + "target": "paper:surroundocc", + "rel": "feeds" + }, + { + "source": "paper:2508.10104", + "target": "paper:surroundocc", + "rel": "feeds" + }, + { + "source": "paper:sam", + "target": "paper:depth_anything", + "rel": "parallel" + }, + { + "source": "paper:clip", + "target": "paper:llava", + "rel": "prereq" + }, + { + "source": "paper:lift_splat_shoot", + "target": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "rel": "covers" + }, + { + "source": "paper:bevdet", + "target": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "rel": "covers" + }, + { + "source": "paper:bevfusion", + "target": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "rel": "covers" + }, + { + "source": "paper:carion2020", + "target": "move:treat_detection_as_set_prediction_with_learnable_queries", + "rel": "covers" + }, + { + "source": "paper:detr3d", + "target": "move:treat_detection_as_set_prediction_with_learnable_queries", + "rel": "covers" + }, + { + "source": "paper:2212.10156", + "target": "move:treat_detection_as_set_prediction_with_learnable_queries", + "rel": "covers" + }, + { + "source": "paper:detr3d", + "target": "move:reproject_3d_query_to_2d_for_feature_sampling", + "rel": "covers" + }, + { + "source": "paper:streampetr", + "target": "move:reproject_3d_query_to_2d_for_feature_sampling", + "rel": "covers" + }, + { + "source": "paper:petr", + "target": "move:embed_camera_geometry_into_positional_encoding", + "rel": "covers" + }, + { + "source": "paper:petrv2", + "target": "move:embed_camera_geometry_into_positional_encoding", + "rel": "covers" + }, + { + "source": "paper:nerf", + "target": "move:replace_explicit_module_with_implicit_function", + "rel": "covers" + }, + { + "source": "paper:3dgs", + "target": "move:swap_implicit_for_explicit_primitives_when_compute_allows", + "rel": "covers" + }, + { + "source": "paper:bevformer_v2", + "target": "move:add_auxiliary_perspective_supervision_to_bev", + "rel": "covers" + }, + { + "source": "paper:streampetr", + "target": "move:carry_object_query_across_time_as_recurrent_state", + "rel": "covers" + }, + { + "source": "paper:bevdet4d", + "target": "move:carry_object_query_across_time_as_recurrent_state", + "rel": "covers" + }, + { + "source": "paper:bevfusion", + "target": "move:fuse_modalities_in_shared_intermediate_space", + "rel": "covers" + }, + { + "source": "paper:occupancy_networks_tesla", + "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", + "rel": "covers" + }, + { + "source": "paper:surroundocc", + "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", + "rel": "covers" + }, + { + "source": "paper:mae", + "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", + "rel": "covers" + }, + { + "source": "paper:dinov2", + "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", + "rel": "covers" + }, + { + "source": "paper:2508.10104", + "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", + "rel": "covers" + }, + { + "source": "paper:blip2", + "target": "move:freeze_giant_backbone_train_small_adapter", + "rel": "covers" + }, + { + "source": "paper:llava", + "target": "move:freeze_giant_backbone_train_small_adapter", + "rel": "covers" + }, + { + "source": "paper:vit", + "target": "move:tokenize_continuous_signal_to_use_transformer", + "rel": "covers" + }, + { + "source": "paper:beit", + "target": "move:tokenize_continuous_signal_to_use_transformer", + "rel": "covers" + }, + { + "source": "paper:nerf", + "target": "move:use_geometry_as_self_supervision", + "rel": "covers" + }, + { + "source": "paper:emernerf", + "target": "move:use_geometry_as_self_supervision", + "rel": "covers" + }, + { + "source": "paper:2212.10156", + "target": "move:make_pipeline_differentiable_via_shared_latent", + "rel": "covers" + }, + { + "source": "paper:vadv2", + "target": "move:make_pipeline_differentiable_via_shared_latent", + "rel": "covers" + }, + { + "source": "paper:nerf", + "target": "move:rasterize_differentiable_renderer_for_inverse_problem", + "rel": "covers" + }, + { + "source": "paper:3dgs", + "target": "move:rasterize_differentiable_renderer_for_inverse_problem", + "rel": "covers" + }, + { + "source": "paper:depth_anything", + "target": "move:distill_internet_data_into_small_specialist", + "rel": "covers" + }, + { + "source": "paper:sam", + "target": "move:distill_internet_data_into_small_specialist", + "rel": "covers" + }, + { + "source": "paper:bevdet4d", + "target": "move:make_camera_only_temporal_match_lidar", + "rel": "covers" + }, + { + "source": "paper:streampetr", + "target": "move:make_camera_only_temporal_match_lidar", + "rel": "covers" + }, + { + "source": "paper:clip", + "target": "move:open_vocabulary_via_text_alignment", + "rel": "covers" + }, + { + "source": "paper:dinov1", + "target": "move:emergent_segmentation_from_self_distillation", + "rel": "covers" + }, + { + "source": "paper:dinov2", + "target": "move:emergent_segmentation_from_self_distillation", + "rel": "covers" + }, + { + "source": "paper:vggt", + "target": "move:replace_handcrafted_sfm_with_feedforward_transformer", + "rel": "covers" + }, + { + "source": "paper:emernerf", + "target": "move:decompose_scene_into_static_and_dynamic_streams", + "rel": "covers" + }, + { + "source": "paper:drivinggaussian", + "target": "move:decompose_scene_into_static_and_dynamic_streams", + "rel": "covers" + }, + { + "source": "paper:drivinggaussian", + "target": "move:bridge_sim_and_real_via_neural_reconstruction", + "rel": "covers" + }, + { + "source": "paper:emernerf", + "target": "move:bridge_sim_and_real_via_neural_reconstruction", + "rel": "covers" + }, + { + "source": "paper:drivinggaussian", + "target": "move:augment_via_counterfactual_object_insertion", + "rel": "covers" + }, + { + "source": "paper:2212.10156", + "target": "move:share_queries_across_multiple_tasks", + "rel": "covers" + }, + { + "source": "paper:petrv2", + "target": "move:share_queries_across_multiple_tasks", + "rel": "covers" + }, + { + "source": "paper:gaia1", + "target": "move:learn_motion_in_latent_space_then_decode", + "rel": "covers" + }, + { + "source": "paper:drivedreamer", + "target": "move:learn_motion_in_latent_space_then_decode", + "rel": "covers" + }, + { + "source": "paper:occ3d", + "target": "move:use_visibility_mask_to_filter_supervision", + "rel": "covers" + }, + { + "source": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "target": "paper:lift_splat_shoot", + "rel": "manifests" + }, + { + "source": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "target": "paper:bevdet", + "rel": "manifests" + }, + { + "source": "move:embed_camera_geometry_into_positional_encoding", + "target": "paper:petr", + "rel": "manifests" + }, + { + "source": "move:replace_class_specific_box_with_class_agnostic_occupancy", + "target": "paper:occupancy_networks_tesla", + "rel": "manifests" + }, + { + "source": "move:carry_object_query_across_time_as_recurrent_state", + "target": "paper:streampetr", + "rel": "manifests" + }, + { + "source": "move:share_queries_across_multiple_tasks", + "target": "paper:2212.10156", + "rel": "manifests" + }, + { + "source": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "target": "paper:bevfusion", + "rel": "composes" + }, + { + "source": "move:fuse_modalities_in_shared_intermediate_space", + "target": "paper:bevfusion", + "rel": "composes" + }, + { + "source": "move:treat_detection_as_set_prediction_with_learnable_queries", + "target": "paper:detr3d", + "rel": "composes" + }, + { + "source": "move:reproject_3d_query_to_2d_for_feature_sampling", + "target": "paper:detr3d", + "rel": "composes" + }, + { + "source": "move:treat_detection_as_set_prediction_with_learnable_queries", + "target": "paper:petr", + "rel": "composes" + }, + { + "source": "move:embed_camera_geometry_into_positional_encoding", + "target": "paper:petr", + "rel": "composes" + }, + { + "source": "move:treat_detection_as_set_prediction_with_learnable_queries", + "target": "paper:2212.10156", + "rel": "composes" + }, + { + "source": "move:share_queries_across_multiple_tasks", + "target": "paper:2212.10156", + "rel": "composes" + }, + { + "source": "move:make_pipeline_differentiable_via_shared_latent", + "target": "paper:2212.10156", + "rel": "composes" + }, + { + "source": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "target": "paper:li2022bevformer", + "rel": "composes" + }, + { + "source": "move:treat_detection_as_set_prediction_with_learnable_queries", + "target": "paper:li2022bevformer", + "rel": "composes" + }, + { + "source": "move:add_auxiliary_perspective_supervision_to_bev", + "target": "paper:bevformer_v2", + "rel": "composes" + }, + { + "source": "move:replace_explicit_module_with_implicit_function", + "target": "paper:nerf", + "rel": "composes" + }, + { + "source": "move:rasterize_differentiable_renderer_for_inverse_problem", + "target": "paper:nerf", + "rel": "composes" + }, + { + "source": "move:swap_implicit_for_explicit_primitives_when_compute_allows", + "target": "paper:3dgs", + "rel": "composes" + }, + { + "source": "move:decompose_scene_into_static_and_dynamic_streams", + "target": "paper:emernerf", + "rel": "composes" + }, + { + "source": "move:bridge_sim_and_real_via_neural_reconstruction", + "target": "paper:drivinggaussian", + "rel": "composes" + }, + { + "source": "move:carry_object_query_across_time_as_recurrent_state", + "target": "paper:streampetr", + "rel": "composes" + }, + { + "source": "move:make_camera_only_temporal_match_lidar", + "target": "paper:bevdet4d", + "rel": "composes" + }, + { + "source": "problem:long_tail_object_categories_in_open_world", + "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", + "rel": "motivates" + }, + { + "source": "problem:long_tail_object_categories_in_open_world", + "target": "move:open_vocabulary_via_text_alignment", + "rel": "motivates" + }, + { + "source": "problem:long_tail_object_categories_in_open_world", + "target": "move:augment_via_counterfactual_object_insertion", + "rel": "motivates" + }, + { + "source": "problem:sim_to_real_gap_in_camera_only_perception", + "target": "move:bridge_sim_and_real_via_neural_reconstruction", + "rel": "motivates" + }, + { + "source": "problem:temporal_consistency_in_bev_segmentation", + "target": "move:carry_object_query_across_time_as_recurrent_state", + "rel": "motivates" + }, + { + "source": "problem:occlusion_reasoning_without_dense_lidar", + "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", + "rel": "motivates" + }, + { + "source": "problem:occlusion_reasoning_without_dense_lidar", + "target": "move:learn_motion_in_latent_space_then_decode", + "rel": "motivates" + }, + { + "source": "problem:label_efficiency_for_3d_annotation", + "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", + "rel": "motivates" + }, + { + "source": "problem:label_efficiency_for_3d_annotation", + "target": "move:use_geometry_as_self_supervision", + "rel": "motivates" + }, + { + "source": "problem:label_efficiency_for_3d_annotation", + "target": "move:distill_internet_data_into_small_specialist", + "rel": "motivates" + }, + { + "source": "problem:unknown_geometry_in_distant_or_dark_regions", + "target": "move:make_camera_only_temporal_match_lidar", + "rel": "motivates" + }, + { + "source": "problem:multi_modal_calibration_drift", + "target": "move:fuse_modalities_in_shared_intermediate_space", + "rel": "motivates" + }, + { + "source": "problem:rendering_speed_vs_quality_tradeoff", + "target": "move:swap_implicit_for_explicit_primitives_when_compute_allows", + "rel": "motivates" + }, + { + "source": "problem:catastrophic_failure_on_rare_weather", + "target": "move:augment_via_counterfactual_object_insertion", + "rel": "motivates" + }, + { + "source": "problem:annotation_inconsistency_across_datasets", + "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", + "rel": "motivates" + }, + { + "source": "problem:depth_ambiguity_in_low_parallax", + "target": "move:make_camera_only_temporal_match_lidar", + "rel": "motivates" + }, + { + "source": "problem:long_tail_object_categories_in_open_world", + "target": "paper:detr3d", + "rel": "unsolved_by" + }, + { + "source": "problem:long_tail_object_categories_in_open_world", + "target": "paper:bevdet", + "rel": "unsolved_by" + }, + { + "source": "problem:occlusion_reasoning_without_dense_lidar", + "target": "paper:detr3d", + "rel": "unsolved_by" + }, + { + "source": "problem:catastrophic_failure_on_rare_weather", + "target": "paper:bevfusion", + "rel": "unsolved_by" + }, + { + "source": "problem:depth_ambiguity_in_low_parallax", + "target": "paper:lift_splat_shoot", + "rel": "unsolved_by" + }, + { + "source": "problem:rendering_speed_vs_quality_tradeoff", + "target": "paper:nerf", + "rel": "unsolved_by" + }, + { + "source": "problem:annotation_inconsistency_across_datasets", + "target": "paper:occ3d", + "rel": "unsolved_by" + }, + { + "source": "insight:multi_view_geometry_as_free_supervision", + "target": "paper:nerf", + "rel": "manifests" + }, + { + "source": "insight:multi_view_geometry_as_free_supervision", + "target": "paper:emernerf", + "rel": "manifests" + }, + { + "source": "insight:multi_view_geometry_as_free_supervision", + "target": "paper:depth_anything", + "rel": "manifests" + }, + { + "source": "insight:foundation_features_transfer_without_finetune", + "target": "paper:dinov2", + "rel": "manifests" + }, + { + "source": "insight:foundation_features_transfer_without_finetune", + "target": "paper:2508.10104", + "rel": "manifests" + }, + { + "source": "insight:foundation_features_transfer_without_finetune", + "target": "paper:blip2", + "rel": "manifests" + }, + { + "source": "insight:occupancy_unifies_static_and_dynamic_scene", + "target": "paper:occupancy_networks_tesla", + "rel": "manifests" + }, + { + "source": "insight:occupancy_unifies_static_and_dynamic_scene", + "target": "paper:surroundocc", + "rel": "manifests" + }, + { + "source": "insight:open_vocabulary_via_language_anchoring", + "target": "paper:clip", + "rel": "manifests" + }, + { + "source": "insight:open_vocabulary_via_language_anchoring", + "target": "paper:llava", + "rel": "manifests" + }, + { + "source": "insight:implicit_vs_explicit_is_a_continuum", + "target": "paper:nerf", + "rel": "manifests" + }, + { + "source": "insight:implicit_vs_explicit_is_a_continuum", + "target": "paper:3dgs", + "rel": "manifests" + }, + { + "source": "insight:bev_is_planning_friendly_intermediate", + "target": "paper:li2022bevformer", + "rel": "manifests" + }, + { + "source": "insight:bev_is_planning_friendly_intermediate", + "target": "paper:bevfusion", + "rel": "manifests" + }, + { + "source": "insight:bev_is_planning_friendly_intermediate", + "target": "paper:2212.10156", + "rel": "manifests" + }, + { + "source": "insight:temporal_aggregation_buys_what_depth_sensor_buys", + "target": "paper:bevdet4d", + "rel": "manifests" + }, + { + "source": "insight:temporal_aggregation_buys_what_depth_sensor_buys", + "target": "paper:streampetr", + "rel": "manifests" + }, + { + "source": "insight:differentiable_rendering_is_universal_inverse_solver", + "target": "paper:nerf", + "rel": "manifests" + }, + { + "source": "insight:differentiable_rendering_is_universal_inverse_solver", + "target": "paper:3dgs", + "rel": "manifests" + }, + { + "source": "insight:differentiable_rendering_is_universal_inverse_solver", + "target": "paper:drivinggaussian", + "rel": "manifests" + }, + { + "source": "insight:multi_view_geometry_as_free_supervision", + "target": "move:use_geometry_as_self_supervision", + "rel": "enables" + }, + { + "source": "insight:foundation_features_transfer_without_finetune", + "target": "move:freeze_giant_backbone_train_small_adapter", + "rel": "enables" + }, + { + "source": "insight:foundation_features_transfer_without_finetune", + "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", + "rel": "enables" + }, + { + "source": "insight:occupancy_unifies_static_and_dynamic_scene", + "target": "move:replace_class_specific_box_with_class_agnostic_occupancy", + "rel": "enables" + }, + { + "source": "insight:open_vocabulary_via_language_anchoring", + "target": "move:open_vocabulary_via_text_alignment", + "rel": "enables" + }, + { + "source": "insight:implicit_vs_explicit_is_a_continuum", + "target": "move:swap_implicit_for_explicit_primitives_when_compute_allows", + "rel": "enables" + }, + { + "source": "insight:bev_is_planning_friendly_intermediate", + "target": "move:lift_2d_features_to_3d_via_learned_depth_distribution", + "rel": "enables" + }, + { + "source": "insight:bev_is_planning_friendly_intermediate", + "target": "move:fuse_modalities_in_shared_intermediate_space", + "rel": "enables" + }, + { + "source": "insight:temporal_aggregation_buys_what_depth_sensor_buys", + "target": "move:make_camera_only_temporal_match_lidar", + "rel": "enables" + }, + { + "source": "insight:temporal_aggregation_buys_what_depth_sensor_buys", + "target": "move:carry_object_query_across_time_as_recurrent_state", + "rel": "enables" + }, + { + "source": "insight:differentiable_rendering_is_universal_inverse_solver", + "target": "move:rasterize_differentiable_renderer_for_inverse_problem", + "rel": "enables" + }, + { + "source": "insight:differentiable_rendering_is_universal_inverse_solver", + "target": "move:replace_explicit_module_with_implicit_function", + "rel": "enables" + }, + { + "source": "paradigm:camera_first_autonomy", + "target": "insight:temporal_aggregation_buys_what_depth_sensor_buys", + "rel": "composes" + }, + { + "source": "paradigm:camera_first_autonomy", + "target": "insight:bev_is_planning_friendly_intermediate", + "rel": "composes" + }, + { + "source": "paradigm:camera_first_autonomy", + "target": "paper:lift_splat_shoot", + "rel": "manifests" + }, + { + "source": "paradigm:camera_first_autonomy", + "target": "paper:occupancy_networks_tesla", + "rel": "manifests" + }, + { + "source": "paradigm:camera_first_autonomy", + "target": "paper:tesla_ai_day", + "rel": "manifests" + }, + { + "source": "paradigm:neural_scene_reconstruction_as_engine", + "target": "insight:differentiable_rendering_is_universal_inverse_solver", + "rel": "composes" + }, + { + "source": "paradigm:neural_scene_reconstruction_as_engine", + "target": "move:bridge_sim_and_real_via_neural_reconstruction", + "rel": "composes" + }, + { + "source": "paradigm:neural_scene_reconstruction_as_engine", + "target": "paper:drivinggaussian", + "rel": "manifests" + }, + { + "source": "paradigm:neural_scene_reconstruction_as_engine", + "target": "paper:emernerf", + "rel": "manifests" + }, + { + "source": "paradigm:neural_scene_reconstruction_as_engine", + "target": "paper:gaia1", + "rel": "parallel" + }, + { + "source": "essay:bitter_lesson", + "target": "move:replace_handcrafted_sfm_with_feedforward_transformer", + "rel": "covers" + }, + { + "source": "essay:bitter_lesson", + "target": "move:distill_internet_data_into_small_specialist", + "rel": "covers" + }, + { + "source": "essay:bitter_lesson", + "target": "move:scale_pretraining_then_fine_tune_with_minimal_labels", + "rel": "covers" + }, + { + "source": "concept:bev", + "target": "paper:lift_splat_shoot", + "rel": "covers" + }, + { + "source": "concept:bev", + "target": "paper:bevfusion", + "rel": "covers" + }, + { + "source": "concept:bev", + "target": "insight:bev_is_planning_friendly_intermediate", + "rel": "covers" + }, + { + "source": "concept:detr_query", + "target": "paper:detr3d", + "rel": "covers" + }, + { + "source": "concept:detr_query", + "target": "move:treat_detection_as_set_prediction_with_learnable_queries", + "rel": "covers" + }, + { + "source": "concept:ssl", + "target": "paper:mae", + "rel": "covers" + }, + { + "source": "concept:ssl", + "target": "paper:dinov1", + "rel": "covers" + }, + { + "source": "concept:ssl", + "target": "paper:simclr_mocov3", + "rel": "covers" + }, + { + "source": "concept:ssl", + "target": "paper:clip", + "rel": "covers" + } + ], + "playbooks": { + "A": { + "title": "Path A — 强化学习从零 / RL from scratch", + "nodes": [ + "course:zhao_rl", + "essay:bitter_lesson", + "course:cs285", + "paper:ross2011_dagger", + "paper:mnih2015_dqn", + "paper:schulman2017_ppo", + "paper:rlhf_dpo", + "paper:2309.16292", + "paper:2311.10813", + "lab:lab01", + "lab:lab02", + "lab:lab07", + "concept:bellman_eq", + "concept:value_iteration", + "concept:td_learning", + "concept:policy_gradient", + "concept:actor_critic", + "concept:dqn", + "concept:ppo", + "concept:imitation_learning", + "concept:covariate_shift" + ] + }, + "B": { + "title": "Path B — 感知到端到端 AD / Perception → E2E", + "nodes": [ + "channel:3blue1brown", + "channel:mu_li_bilibili", + "paper:vaswani2017", + "paper:vit", + "paper:he2015_resnet", + "paper:carion2020", + "paper:li2022bevformer", + "paper:dinov2", + "paper:2508.10104", + "paper:2212.10156", + "paper:2210.14222", + "paper:vadv2", + "paper:transfuser", + "paper:2402.12289", + "paper:2512.24426", + "lab:lab03", + "lab:lab04", + "lab:lab05", + "lab:lab09", + "lab:lab10", + "concept:transformer", + "concept:detr_query", + "concept:bev", + "concept:vla", + "concept:cot" + ] + }, + "C": { + "title": "Path C — 类脑 / 高效 AD / Brain-inspired & efficient", + "nodes": [ + "paper:vit", + "paper:2307.01694", + "paper:2508.10104", + "paper:2212.10156", + "essay:bitter_lesson", + "paper:mamba", + "lab:lab06", + "concept:spiking_nn", + "concept:self_attention", + "concept:scaling_vs_knowledge" + ] + }, + "D": { + "title": "Path D — LLM·VLM·VLA 范式", + "nodes": [ + "channel:3blue1brown", + "channel:mu_li_bilibili", + "paper:vaswani2017", + "paper:gpt3", + "paper:llava", + "paper:rlhf_dpo", + "paper:2309.16292", + "paper:2311.10813", + "paper:2402.12289", + "paper:2512.24426", + "paper:gaia1", + "paper:drivedreamer", + "paper:world_models", + "paper:lingo2", + "lab:lab07", + "lab:lab08", + "lab:lab09", + "lab:lab10", + "concept:vlm", + "concept:vla", + "concept:cot", + "concept:tool_use", + "concept:counterfactual", + "concept:meta_action", + "concept:rlhf" + ] + } + }, + "topic_palette": { + "math_foundations": "#7c4dff", + "rl_foundations": "#1565c0", + "deep_rl": "#0d47a1", + "ssl_vision": "#2e7d32", + "e2e_ad": "#ef6c00", + "vlm_vla": "#c62828", + "brain_inspired": "#6a1b9a", + "meta_philosophy": "#424242", + "companion_media": "#00838f", + "scene_understanding": "#388e3c", + "geometry_3d": "#00897b", + "sensor_fusion": "#558b2f", + "planning": "#1976d2", + "control": "#01579b", + "safety": "#d84315", + "world_models": "#5d4037", + "llm_agent": "#b71c1c", + "foundation_models": "#ad1457", + "alignment": "#4a148c", + "reasoning": "#283593", + "evaluation": "#e65100", + "data_engineering": "#f57f17", + "hardware": "#37474f", + "neuromorphic": "#7b1fa2", + "interpretability": "#827717", + "simulation": "#006064", + "cross_domain": "#bf360c", + "methodology": "#3e2723", + "evaluation_benchmark": "#e65100", + "simulator": "#006064", + "dataset": "#f57f17", + "efficient_computing": "#37474f", + "neuromorphic_hardware": "#7b1fa2", + "data_engine": "#f57f17", + "safety_standard": "#d84315" + }, + "tier_border_width": { + "spine": 8, + "S": 5, + "A": 3, + "B": 2, + "concept": 1, + "lab": 2, + "move": 2, + "problem": 3, + "insight": 4, + "validation": 4, + "paradigm": 6 + } +} \ No newline at end of file diff --git a/docs/data/graph_extended.stats.json b/docs/data/graph_extended.stats.json new file mode 100644 index 0000000..edd83d0 --- /dev/null +++ b/docs/data/graph_extended.stats.json @@ -0,0 +1,82 @@ +{ + "node_count": 489, + "edge_count": 1266, + "by_kind": { + "paper": 174, + "channel": 3, + "course": 2, + "essay": 1, + "concept": 25, + "lab": 11, + "move": 129, + "problem": 41, + "insight": 54, + "paradigm": 25, + "validation": 24 + }, + "by_tier": { + "spine": 14, + "S": 36, + "A": 71, + "B": 59, + "concept": 25, + "lab": 11, + "move": 129, + "problem": 41, + "insight": 54, + "paradigm": 25, + "validation": 24 + }, + "by_topic": { + "e2e_ad": 23, + "vlm_vla": 71, + "brain_inspired": 7, + "ssl_vision": 37, + "math_foundations": 14, + "companion_media": 2, + "rl_foundations": 8, + "deep_rl": 78, + "meta_philosophy": 3, + "world_models": 32, + "planning": 15, + "control": 7, + "safety": 11, + "foundation_models": 15, + "alignment": 4, + "llm_agent": 14, + "reasoning": 8, + "evaluation_benchmark": 17, + "simulator": 8, + "dataset": 6, + "efficient_computing": 14, + "neuromorphic_hardware": 11, + "data_engine": 14, + "safety_standard": 14, + "geometry_3d": 27, + "sensor_fusion": 3, + "scene_understanding": 26 + }, + "by_rel": { + "prereq": 82, + "covers": 283, + "parallel": 108, + "contrasts": 23, + "extends": 35, + "feeds": 88, + "implements": 14, + "manifests": 168, + "enables": 17, + "composes": 316, + "motivates": 101, + "validates": 24, + "unsolved_by": 7 + }, + "by_source": { + "seed": 75, + "decision_axis": 93, + "foundation_axis": 93, + "insights_and_validations": 69, + "methodology_axis": 84, + "perception_axis": 75 + } +} \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index 8de3be7..5a28bb4 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3,137 +3,174 @@ -Autonomous-Driving Learning Atlas - - +自动驾驶研究洞察星图 / Autonomous-Driving Research Insight Atlas + + + - - + + + - - - - -
-
- - AD Learning Atlas - 从数学直觉到自动驾驶前沿 · ML · RL · VLA -
- -
+ -