Replace <|endoftext|> with constant (#186)

alvarobartt · web-flow · commit 52fceb8fa1d2 · 2023-09-11T21:18:00.000-07:00
diff --git a/tiktoken_ext/openai_public.py b/tiktoken_ext/openai_public.py
@@ -17,7 +17,7 @@ def gpt2():
         "explicit_n_vocab": 50257,
         "pat_str": r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""",
         "mergeable_ranks": mergeable_ranks,
-        "special_tokens": {"<|endoftext|>": 50256},
+        "special_tokens": {ENDOFTEXT: 50256},
     }
 
 

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ def gpt2():`
`17`	`17`	`"explicit_n_vocab": 50257,`
`18`	`18`	`"pat_str": r"""'s\|'t\|'re\|'ve\|'m\|'ll\|'d\| ?\p{L}+\| ?\p{N}+\| ?[^\s\p{L}\p{N}]+\|\s+(?!\S)\|\s+""",`
`19`	`19`	`"mergeable_ranks": mergeable_ranks,`
`20`		`- "special_tokens": {"<\|endoftext\|>": 50256},`
	`20`	`+ "special_tokens": {ENDOFTEXT: 50256},`
`21`	`21`	`}`
`22`	`22`
`23`	`23`