From 53885de682a703b67e2067df71f4bde2b00cf5fb Mon Sep 17 00:00:00 2001 From: cscs <35064549+zpczpc@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:52:27 +0800 Subject: [PATCH 1/2] Document that BpeOptions.ByteLevel requires a PreTokenizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The documentation for `BpeOptions.ByteLevel` only described the byte <-> unicode mapping (for example `Space -> 'Ġ'`) and did not mention that a pre-tokenizer is required. Without a pre-tokenizer, byte-level encoding maps the space character to `'Ġ'` but does not keep the whitespace attached to the following token, so spaces and newlines are dropped during encoding and cannot be recovered by decoding. This adds a `` block stating the requirement and points at `RegexPreTokenizer` with the GPT-2 pattern as the configuration that keeps the round trip lossless. Relates to #7715. --- src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs b/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs index 8eee50ac66..1c6edd7eac 100644 --- a/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs +++ b/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs @@ -143,6 +143,14 @@ public BpeOptions(string vocabFile, string? mergesFile = null) /// if true, the input text will be converted to UTF-8 bytes before encoding it. /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character). /// + /// + /// Byte-level encoding is normally paired with a byte-level pre-tokenizer. When this property is set to + /// , must be set as well. + /// On its own, byte-level encoding maps the space character to 'Ġ' but does not keep the whitespace attached + /// to the following token, so spaces and newlines are dropped during encoding and cannot be recovered by decoding. + /// Configuring a byte-level pre-tokenizer, for example built from the GPT-2 + /// pattern, makes the round trip lossless. + /// public bool ByteLevel { get; set; } /// From 9d393fca65402d841ea9f66cae8505c7f24ccb73 Mon Sep 17 00:00:00 2001 From: cscs <35064549+zpczpc@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:10:47 +0800 Subject: [PATCH 2/2] Update ByteLevel remarks per review feedback Updated documentation for ByteLevel property to clarify behavior with pre-tokenizers. --- src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs b/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs index 1c6edd7eac..1e276435fe 100644 --- a/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs +++ b/src/Microsoft.ML.Tokenizers/Model/BpeOptions.cs @@ -144,12 +144,12 @@ public BpeOptions(string vocabFile, string? mergesFile = null) /// Additionally, some ASCII characters will be transformed to different characters (e.g Space character will be transformed to 'Ġ' character). /// /// - /// Byte-level encoding is normally paired with a byte-level pre-tokenizer. When this property is set to - /// , must be set as well. - /// On its own, byte-level encoding maps the space character to 'Ġ' but does not keep the whitespace attached - /// to the following token, so spaces and newlines are dropped during encoding and cannot be recovered by decoding. - /// Configuring a byte-level pre-tokenizer, for example built from the GPT-2 - /// pattern, makes the round trip lossless. + /// When this property is set to and no pre-tokenizer is specified, the tokenizer falls + /// back to PreTokenizer.CreateWordOrNonWord, whose pattern does not cover whitespace characters. As + /// only the segments returned by the pre-tokenizer are encoded, whitespace is not preserved during + /// pre-tokenization and cannot be recovered by decoding. + /// To keep whitespace, set to a pre-tokenizer whose pattern covers it, for example + /// a built from the GPT-2 pattern. /// public bool ByteLevel { get; set; }