[d129b2]: / medicalbert / tokenizers / tokenizer_factory.py

Download this file

10 lines (6 with data), 210 Bytes

1
2
3
4
5
6
7
8
9
from transformers import BertTokenizer
class TokenizerFactory:
def __init__(self):
pass
def make_tokenizer(self, name):
return BertTokenizer.from_pretrained(name, do_lower_case=True)