awni ArthurZ HF Staff commited on
Commit
526358b
·
verified ·
1 Parent(s): 7427a19

Update tokenizer_config.json (#3)

Browse files

- Update tokenizer_config.json (d913439da3a3e7a8c3047e97b025da79b60b9ee8)


Co-authored-by: Arthur Zucker <ArthurZ@users.noreply.huggingface.co>

Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -491,7 +491,7 @@
491
  "eos_token": "[e~[",
492
  "extra_special_tokens": {},
493
  "model_max_length": 40960000,
494
- "tokenizer_class": "GPT2Tokenizer",
495
  "unk_token": "]!d~[",
496
  "tool_parser_type": "minimax_m2"
497
  }
 
491
  "eos_token": "[e~[",
492
  "extra_special_tokens": {},
493
  "model_max_length": 40960000,
494
+ "tokenizer_class": "TokenizersBackend",
495
  "unk_token": "]!d~[",
496
  "tool_parser_type": "minimax_m2"
497
  }