Models:
Catherine Stones
cathy-stones/
Medical-ChatBot
0
Downloads: 0
[4e3933]: / tokenizer_config.json

Download this file

35 lines (34 with data), 774 Bytes 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
{

  "add_bos_token": false,

  "add_prefix_space": false,

  "bos_token": {

    "__type": "AddedToken",

    "content": "<|endoftext|>",

    "lstrip": false,

    "normalized": true,

    "rstrip": false,

    "single_word": false

  },

  "eos_token": {

    "__type": "AddedToken",

    "content": "<|endoftext|>",

    "lstrip": false,

    "normalized": true,

    "rstrip": false,

    "single_word": false

  },

  "errors": "replace",

  "model_max_length": 1024,

  "name_or_path": "biomedLM_baize/checkpoint-1000",

  "pad_token": null,

  "special_tokens_map_file": null,

  "tokenizer_class": "GPT2Tokenizer",

  "unk_token": {

    "__type": "AddedToken",

    "content": "<|endoftext|>",

    "lstrip": false,

    "normalized": true,

    "rstrip": false,

    "single_word": false

  }

}