sql-ocr / tokenizer.json
JohnnyZeppelin's picture
Upload tokenizer.json with huggingface_hub
0fe9e15 verified
{
"version": "1.0",
"truncation": null,
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 0,
"pad_type_id": 0,
"pad_token": "[PAD]"
},
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[BOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[EOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"[PAD]": 0,
"[UNK]": 1,
"[BOS]": 2,
"[EOS]": 3,
"the": 4,
"|>": 5,
"'": 6,
"(": 7,
"Keys": 8,
"database": 9,
"end": 10,
",": 11,
")": 12,
"',": 13,
"above": 14,
"and": 15,
"for": 16,
"that": 17,
").": 18,
".": 19,
":": 20,
"<|": 21,
"FK": 22,
"FROM": 23,
"Foreign": 24,
"Given": 25,
"PK": 26,
"Primary": 27,
"SELECT": 28,
"SQL": 29,
"Then": 30,
"analyze": 31,
"answers": 32,
"based": 33,
"between": 34,
"displayed": 35,
"on": 36,
"query": 37,
"question": 38,
"relationships": 39,
"schema": 40,
"system": 41,
"tables": 42,
"user": 43,
"write": 44,
"|><|": 45,
"WHERE": 46,
";<|": 47,
".'<|": 48,
"=": 49,
"books": 50,
"?'<|": 51,
"BY": 52,
"GROUP": 53,
"loans": 54,
"priority": 55,
"users": 56,
"(*)": 57,
"-": 58,
"01": 59,
"15": 60,
"2024": 61,
"3": 62,
"5": 63,
">": 64,
"AS": 65,
"COUNT": 66,
"JOIN": 67,
"List": 68,
"ON": 69,
"SUM": 70,
"amount": 71,
"customer_id": 72,
"date": 73,
"employees": 74,
"id": 75,
"month": 76,
"name": 77,
"open": 78,
"products": 79,
"revenue": 80,
"students": 81,
"');<|": 82,
"';<|": 83,
"('": 84,
"('%": 85,
"-%'": 86,
"0": 87,
"2025": 88,
"<": 89,
">=": 90,
"Count": 91,
"DATE": 92,
"Find": 93,
"GPA": 94,
"How": 95,
"LIKE": 96,
"Show": 97,
"What": 98,
"Which": 99,
"are": 100,
"book_id": 101,
"borrowers": 102,
"by": 103,
"customer": 104,
"delay_minutes": 105,
"delays": 106,
"due_date": 107,
"expense": 108,
"expenses": 109,
"gpa": 110,
"have": 111,
"hire_date": 112,
"is": 113,
"joined": 114,
"m": 115,
"many": 116,
"minutes": 117,
"monthly": 118,
"now": 119,
"of": 120,
"orders": 121,
"out": 122,
"over": 123,
"overdue": 124,
"per": 125,
"product_name": 126,
"quantity": 127,
"route_id": 128,
"routes": 129,
"status": 130,
"stock": 131,
"strftime": 132,
"support_tickets": 133,
"their": 134,
"this": 135,
"tickets": 136,
"title": 137,
"total": 138,
"totals": 139,
"trips": 140,
"user_id": 141,
"with": 142,
"year": 143,
"education_11": 144,
"education_19": 145,
"education_27": 146,
"education_3": 147,
"education_35": 148,
"education_43": 149,
"education_51": 150,
"education_59": 151,
"finance_12": 152,
"finance_20": 153,
"finance_28": 154,
"finance_36": 155,
"finance_4": 156,
"finance_44": 157,
"finance_52": 158,
"finance_60": 159,
"hr_10": 160,
"hr_18": 161,
"hr_2": 162,
"hr_26": 163,
"hr_34": 164,
"hr_42": 165,
"hr_50": 166,
"hr_58": 167,
"inventory_1": 168,
"inventory_17": 169,
"inventory_25": 170,
"inventory_33": 171,
"inventory_41": 172,
"inventory_49": 173,
"inventory_57": 174,
"inventory_9": 175,
"library_14": 176,
"library_22": 177,
"library_30": 178,
"library_38": 179,
"library_46": 180,
"library_54": 181,
"library_6": 182,
"library_62": 183,
"sales_0": 184,
"sales_16": 185,
"sales_24": 186,
"sales_32": 187,
"sales_40": 188,
"sales_48": 189,
"sales_56": 190,
"sales_8": 191,
"tickets_15": 192,
"tickets_23": 193,
"tickets_31": 194,
"tickets_39": 195,
"tickets_47": 196,
"tickets_55": 197,
"tickets_63": 198,
"tickets_7": 199,
"transport_13": 200,
"transport_21": 201,
"transport_29": 202,
"transport_37": 203,
"transport_45": 204,
"transport_5": 205,
"transport_53": 206,
"transport_61": 207
},
"unk_token": "[UNK]"
}
}