[2d4573]: / collated_tasks / tasks / get_translation.py

Download this file

53 lines (44 with data), 2.8 kB

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import argparse
from transformers import MarianMTModel, MarianTokenizer
from utils.get_text_from_csv import get_df, get_notes_single_row_id
LANG_CODE = {'Malay_written_with_Latin': '>>zlm_Latn<<', 'Mauritian_Creole': '>>mfe<<', 'Haitian': '>>hat<<', 'Papiamento': '>>pap<<', 'Asturian': '>>ast<<',
'Catalan': '>>cat<<', 'Indonesian': '>>ind<<', 'Galician': '>>glg<<', 'Walloon': '>>wln<<', 'Spanish': '>>spa<<',
'French': '>>fra<<', 'Romanian': '>>ron<<', 'Portuguese': '>>por<<', 'Italian': '>>ita<<', 'Occitan': '>>oci<<',
'Aragonese': '>>arg<<', 'Minangkabau': '>>min<<'}
def get_translation(text, model_name, target_language):
print(f'Translating medical note using {model_name}')
partial_input = '\n'.join(text.split('\n')[:10])
print(f"Input text (truncated): {partial_input}\n...")
tokenizer = MarianTokenizer.from_pretrained(model_name)
# check supported languages of the selected model
#support_languages = tokenizer.supported_language_codes
model = MarianMTModel.from_pretrained(model_name)
sents = [f'{LANG_CODE[target_language]} ' + t for t in text.split('\n')]
translated = model.generate(**tokenizer(sents, return_tensors="pt", padding=True))
translated = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
return translated
if __name__ == "__main__":
# parse command line arguments
parser = argparse.ArgumentParser(description='Translate medical notes in MIMIC EVENTNOTES')
parser.add_argument('--mimic_dir', default='../../tutorials/data/mimic_data/', type=str, help='directory to mimic data')
#parser.add_argument('--model', default='Helsinki-NLP/opus-mt-en-roa', type=str, help='model for bert')
parser.add_argument('--target_language', default='Spanish', type=str, help='target language of translation, default to Spanish')
parser.add_argument('--row_id', default=178, type=int, help='row id of text to be processed')
parser.add_argument('--output_file', default='./output_translation.txt', type=str, help='output to save translated note')
args = parser.parse_args()
mimic_dir = args.mimic_dir
# model = args.model
model = 'Helsinki-NLP/opus-mt-en-ROMANCE'
target_language = args.target_language
row_id = args.row_id
output_file = args.output_file
print(f"Data file: {mimic_dir}NOTEEVENTS.csv")
df = get_df(mimic_dir + 'NOTEEVENTS.csv')
text = get_notes_single_row_id(df, row_id)
translated = get_translation(text, model, target_language)
with open(output_file, 'w') as f:
f.write('Original:\n')
f.write(text + '\n')
f.write('Translated:\n')
f.write('\n'.join(translated) + '\n')
print(f"Original & translated records written to {args.output_file}")