Xenova
/

dbrx-instruct-tokenizer

Transformers.js

Model card Files Files and versions

Xenova HF Staff commited on Mar 27, 2024

Commit

4ec0196

·

verified ·

1 Parent(s): 39221ee

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +3 -2

tokenizer.json CHANGED Viewed

@@ -100373,8 +100373,9 @@
       "<|fim_middle|>": 100259,
       "<|fim_suffix|>": 100260,
       "<|endofprompt|>": 100276,
-      "<|im_start|>": 100264,
-      "<|im_end|>": 100265
     },
     "merges": [
       "Ġ Ġ",

       "<|fim_middle|>": 100259,
       "<|fim_suffix|>": 100260,
       "<|endofprompt|>": 100276,
+	  "<|pad|>": 100277,
+      "<|im_start|>": 100278,
+      "<|im_end|>": 100279
     },
     "merges": [
       "Ġ Ġ",