Update tokenizer_config.json (#3)
Browse files- Update tokenizer_config.json (d913439da3a3e7a8c3047e97b025da79b60b9ee8)
Co-authored-by: Arthur Zucker <ArthurZ@users.noreply.huggingface.co>
- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
|
@@ -491,7 +491,7 @@
|
|
| 491 |
"eos_token": "[e~[",
|
| 492 |
"extra_special_tokens": {},
|
| 493 |
"model_max_length": 40960000,
|
| 494 |
-
"tokenizer_class": "
|
| 495 |
"unk_token": "]!d~[",
|
| 496 |
"tool_parser_type": "minimax_m2"
|
| 497 |
}
|
|
|
|
| 491 |
"eos_token": "[e~[",
|
| 492 |
"extra_special_tokens": {},
|
| 493 |
"model_max_length": 40960000,
|
| 494 |
+
"tokenizer_class": "TokenizersBackend",
|
| 495 |
"unk_token": "]!d~[",
|
| 496 |
"tool_parser_type": "minimax_m2"
|
| 497 |
}
|