Add protocol-native trajectory histories and tokenization

docs/features/additional-histories.mdx

@@ -30,7 +30,7 @@ Some models, like Qwen 3, use chat templates that remove special tokens (such as
 By splitting each turn into a separate history, you can preserve these tokens for training:
 
 ```python
-from art.trajectories import Trajectory, History
+from art.trajectories import LegacyHistory, Trajectory
 
 # Instead of a single multi-turn conversation that loses <think> tokens
 # Train as separate histories to preserve them
@@ -41,7 +41,7 @@ trajectory = Trajectory(
         {"role": "assistant", "content": "<think>I need to add 2 and 2</think>4"}
     ],
     additional_histories=[
-        History(
+        LegacyHistory(
             messages_and_choices=[
                 # The Qwen 3 chat template removes <think> tokens from previous turns
                 {"role": "user", "content": "What is 2+2?"},
@@ -82,15 +82,15 @@ trajectory = Trajectory(
     ],
     additional_histories=[
         # Sub-agent 1: Code analysis
-        History(
+        LegacyHistory(
             messages_and_choices=[
                 {"role": "system", "content": "You are a code analysis expert"},
                 {"role": "user", "content": "Find potential bugs in main.py"},
                 {"role": "assistant", "content": "Found 3 potential issues..."}
             ]
         ),
         # Sub-agent 2: Bug fixing
-        History(
+        LegacyHistory(
             messages_and_choices=[
                 {"role": "system", "content": "You are a bug fixing expert"},
                 {"role": "user", "content": "Fix the null pointer issue on line 42"},
@@ -116,7 +116,7 @@ trajectory = Trajectory(
     ],
     additional_histories=[
         # Previous conversation segment before compaction
-        History(
+        LegacyHistory(
             messages_and_choices=[
                 {"role": "system", "content": "You are a helpful assistant."},
                 {"role": "user", "content": "Compacted conversation history: the user asked about quantum entanglement, and the assistant explained..."},
@@ -152,11 +152,11 @@ for history in histories:
 
 ### Data Structure
 
-The `History` class structure:
+The legacy `LegacyHistory` payload structure:
 
 ```python
 @dataclass
-class History:
+class LegacyHistory:
     messages_and_choices: list[dict[str, Any]]
     tools: list[Tool] | None = None
 ```
@@ -168,7 +168,7 @@ The `Trajectory` class with additional histories:
 class Trajectory:
     messages_and_choices: list[dict[str, Any]]
     tools: list[Tool] | None = None
-    additional_histories: list[History] = field(default_factory=list)
+    additional_histories: list[LegacyHistory] = field(default_factory=list)
     reward: float | None = None
     metrics: dict[str, Any] = field(default_factory=dict)
 ```
@@ -178,7 +178,7 @@ class Trajectory:
 ### Creating a Trajectory with Additional Histories
 
 ```python
-from art.trajectories import Trajectory, History
+from art.trajectories import LegacyHistory, Trajectory
 
 # Create the main conversation
 main_messages = [
@@ -188,14 +188,14 @@ main_messages = [
 ]
 
 # Create additional histories
-history1 = History(
+history1 = LegacyHistory(
     messages_and_choices=[
         {"role": "user", "content": "First subtask"},
         {"role": "assistant", "content": "Completing first subtask..."}
     ]
 )
 
-history2 = History(
+history2 = LegacyHistory(
     messages_and_choices=[
         {"role": "user", "content": "Second subtask"},
         {"role": "assistant", "content": "Completing second subtask..."}

pyproject.toml

@@ -26,7 +26,7 @@ plotting = ["matplotlib>=3.10.1", "seaborn>=0.13.2"]
 backend = [
     "peft>=0.14.0",
     "hf-xet>=1.1.0",
-    "bitsandbytes>=0.45.2",
+    "bitsandbytes>=0.45.2,!=0.50.0",
     "unsloth==2026.3.3",
     "unsloth-zoo==2026.3.1",
     "torch==2.11.0",

src/art/__init__.py

@@ -79,17 +79,28 @@
 from .serverless import ServerlessBackend
 from .trajectories import (
     AnthropicMessagesHistory,
+    AnthropicMessageSource,
     ChatCompletionsExchange,
     ChatCompletionsHistory,
+    ChatCompletionsMessageSource,
     CompletionsExchange,
-    CompletionsHistory,
+    CompletionsSource,
+    CompletionsStringHistory,
+    CompletionsStringSourceSpan,
+    CompletionsTokenHistory,
+    CompletionsTokenSourceSpan,
     History,
+    LegacyHistory,
     MessagesExchange,
     ResponsesExchange,
     ResponsesHistory,
+    ResponsesItemSource,
     TokenFlag,
+    TokenizedHistory,
+    TokenizedMultiHistoryTrajectory,
     TokenizedTrajectory,
     TokenizedTrajectoryGroup,
+    Tokenizer,
     Trajectory,
     TrajectoryExchanges,
     TrajectoryGroup,
@@ -98,10 +109,6 @@
     capture_auto_trajectory,  # ty: ignore[deprecated]
     current_trajectory,
     current_trajectory_group,
-    tokenize_trajectories,
-    tokenize_trajectory,
-    tokenize_trajectory_group,
-    tokenize_trajectory_groups,
     trajectory,
     trajectory_group,
 )
@@ -158,24 +165,31 @@
     "TrajectoryExchanges",
     "TrajectoryGroup",
     "History",
+    "LegacyHistory",
     "TrajectoryHistory",
     "ChatCompletionsExchange",
     "ChatCompletionsHistory",
+    "ChatCompletionsMessageSource",
     "CompletionsExchange",
-    "CompletionsHistory",
+    "CompletionsSource",
+    "CompletionsTokenHistory",
+    "CompletionsStringHistory",
+    "CompletionsTokenSourceSpan",
+    "CompletionsStringSourceSpan",
     "ResponsesExchange",
     "ResponsesHistory",
+    "ResponsesItemSource",
     "MessagesExchange",
     "AnthropicMessagesHistory",
+    "AnthropicMessageSource",
     "TokenFlag",
+    "Tokenizer",
+    "TokenizedHistory",
+    "TokenizedMultiHistoryTrajectory",
     "TokenizedTrajectory",
     "TokenizedTrajectoryGroup",
     "trajectory",
     "trajectory_group",
-    "tokenize_trajectory",
-    "tokenize_trajectories",
-    "tokenize_trajectory_group",
-    "tokenize_trajectory_groups",
     "capture_yielded_trajectory",
     "yield_trajectory",
 ]

src/art/langgraph/llm_wrapper.py

@@ -14,7 +14,7 @@
 from langchain_core.utils.function_calling import convert_to_openai_tool
 from langchain_openai import ChatOpenAI
 
-from art.trajectories import History, Trajectory
+from art.trajectories import LegacyHistory, Trajectory
 
 from .logging import FileLogger
 from .message_utils import convert_langgraph_messages
@@ -89,7 +89,7 @@ def create_messages_from_logs(logger: FileLogger, trajectory: Trajectory):
                 trajectory.tools = tools[idx]
             else:
                 trajectory.additional_histories.append(
-                    History(messages_and_choices=converted, tools=tools[idx])
+                    LegacyHistory(messages_and_choices=converted, tools=tools[idx])
                 )
         except Exception:
             pass

src/art/local/backend.py

@@ -91,6 +91,7 @@
 )
 from ..serving_capabilities import ServingCapabilities
 from ..trajectories import Trajectory, TrajectoryGroup
+from ..trajectories._selection import automatic_training_model_selector
 from ..types import (
     Choice,
     LocalTrainResult,
@@ -949,6 +950,12 @@ def _get_packed_tensors(
         chat_template_tool_schema_format = self._chat_template_tool_schema_format(
             internal_config
         )
+        model_max_sequence_length = self._model_max_sequence_length(model)
+        training_max_sequence_length = (
+            min(model_max_sequence_length, packed_sequence_length)
+            if packed_sequence_length is not None
+            else model_max_sequence_length
+        )
         tokenized_results = list(
             tokenize_trajectory_groups(
                 tokenizer,
@@ -958,11 +965,14 @@ def _get_packed_tensors(
                 image_processor=self._image_processors[model.base_model],
                 chat_template_kwargs=chat_template_kwargs,
                 chat_template_tool_schema_format=chat_template_tool_schema_format,
+                model=automatic_training_model_selector(
+                    self._model_inference_name(model)
+                ),
+                _max_sequence_length=training_max_sequence_length,
             )
         )
         if not tokenized_results:
             return None
-        model_max_sequence_length = self._model_max_sequence_length(model)
         too_long_for_model = [
             result
             for result in tokenized_results

src/art/openai.py

@@ -163,14 +163,14 @@ def update_chat_completion(
                         tool_call.function.arguments += (
                             tool_call_delta.function.arguments
                         )
-        if getattr(chunk_choice.delta, "reasoning", None):
-            if not hasattr(choice.message, "reasoning"):
-                setattr(choice.message, "reasoning", "")
+        for field in ("reasoning", "reasoning_content"):
+            value = getattr(chunk_choice.delta, field, None)
+            if not value:
+                continue
             setattr(
                 choice.message,
-                "reasoning",
-                getattr(choice.message, "reasoning")
-                + getattr(chunk_choice.delta, "reasoning"),
+                field,
+                (getattr(choice.message, field, None) or "") + value,
             )
     chat_completion.service_tier = chunk.service_tier
     chat_completion.system_fingerprint = chunk.system_fingerprint