pandurangpatil commited on
Commit
84060c4
·
verified ·
1 Parent(s): a58340a

Upload tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -4
tokenizer.json CHANGED
@@ -9,15 +9,13 @@
9
  "pre_tokenizer": {
10
  "type": "Metaspace",
11
  "replacement": "▁",
12
- "add_prefix_space": false,
13
- "prepend_scheme": "first"
14
  },
15
  "post_processor": null,
16
  "decoder": {
17
  "type": "Metaspace",
18
  "replacement": "▁",
19
- "add_prefix_space": false,
20
- "prepend_scheme": "first"
21
  },
22
  "model": {
23
  "type": "BPE",
 
9
  "pre_tokenizer": {
10
  "type": "Metaspace",
11
  "replacement": "▁",
12
+ "add_prefix_space": false
 
13
  },
14
  "post_processor": null,
15
  "decoder": {
16
  "type": "Metaspace",
17
  "replacement": "▁",
18
+ "add_prefix_space": false
 
19
  },
20
  "model": {
21
  "type": "BPE",