From 72e648c3f643285ffef057916a83711fd7355834 Mon Sep 17 00:00:00 2001
From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com>
Date: Thu, 21 May 2026 23:36:00 -0400
Subject: [PATCH] chat-format: ignore HuggingFace's {% generation %}
 chat-template tag

HuggingFace's transformers chat-template extension adds {% generation %}
and {% endgeneration %} tags so trainers can mark generation spans for
loss masking. The tags ship in GGUF tokenizer.chat_template metadata
(SmolLM3 et al), but jinja2's default environment doesn't recognize
them, so Llama() raises TemplateSyntaxError at init for any affected
GGUF, even when the caller passes an explicit chat_format override.

Register a minimal Jinja extension that treats both tags as inert
wrappers: the body between them renders as-is, the markers themselves
emit nothing. No behavioral change for templates that don't use the
tags.

Prior art: PR #2082 attempted the same approach but referenced an
unimported 'nodes' module and didn't consume the body or closing tag.
---
 llama_cpp/llama_chat_format.py  | 13 +++++++++++++
 tests/test_llama_chat_format.py | 23 +++++++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py
index 1024fb85b..ac8c02513 100644
--- a/llama_cpp/llama_chat_format.py
+++ b/llama_cpp/llama_chat_format.py
@@ -24,6 +24,8 @@
 )
 
 import jinja2
+from jinja2 import nodes
+from jinja2.ext import Extension
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 import numpy as np
@@ -191,6 +193,16 @@ def __call__(
     ) -> ChatFormatterResponse: ...
 
 
+class _GenerationTagIgnore(Extension):
+    """Pass-through for HuggingFace's ``{% generation %}`` chat-template tag."""
+
+    tags = {"generation"}
+
+    def parse(self, parser: jinja2.parser.Parser) -> List[nodes.Node]:
+        parser.stream.skip(1)  # discard the 'generation' tag-name token
+        return parser.parse_statements(("name:endgeneration",), drop_needle=True)
+
+
 class Jinja2ChatFormatter(ChatFormatter):
     def __init__(
         self,
@@ -213,6 +225,7 @@ def __init__(
             loader=jinja2.BaseLoader(),
             trim_blocks=True,
             lstrip_blocks=True,
+            extensions=[_GenerationTagIgnore],
         ).from_string(self.template)
 
     @staticmethod
diff --git a/tests/test_llama_chat_format.py b/tests/test_llama_chat_format.py
index 18c7279cf..16852a472 100644
--- a/tests/test_llama_chat_format.py
+++ b/tests/test_llama_chat_format.py
@@ -92,3 +92,26 @@ def test_hf_tokenizer_config_str_to_chat_formatter():
     )
 
     assert chat_formatter_respoonse.prompt == ("<s>[INST] Hello, world! [/INST]</s>")
+
+
+def test_generation_tag_is_ignored() -> None:
+    """HuggingFace chat templates use {% generation %}/{% endgeneration %} to
+    mark training-time loss spans. At inference the tags must be no-ops or
+    affected GGUFs (SmolLM3 and similar) fail to load with TemplateSyntaxError.
+    """
+    template = (
+        "{% for message in messages %}"
+        "{% generation %}{{ message['role'] }}: {{ message['content'] }}{% endgeneration %}"
+        "{% endfor %}"
+    )
+    chat_formatter = llama_chat_format.Jinja2ChatFormatter(
+        template=template,
+        eos_token="</s>",
+        bos_token="<s>",
+    )
+    response = chat_formatter(
+        messages=[
+            ChatCompletionRequestUserMessage(role="user", content="hi"),
+        ]
+    )
+    assert "user: hi" in response.prompt