a b/tests/test_transforms.py
1
from __future__ import annotations
2
3
import datetime
4
5
import meds
6
7
from femr.transforms import delta_encode, remove_nones
8
from femr.transforms.stanford import (
9
    move_billing_codes,
10
    move_pre_birth,
11
    move_to_day_end,
12
    move_visit_start_to_first_event_start,
13
)
14
15
16
def cleanup(patient):
17
    for event in patient["events"]:
18
        for measurement in event["measurements"]:
19
            if "metadata" not in measurement:
20
                measurement["metadata"] = {}
21
22
            for k in ("numeric_value", "text_value", "datetime_value"):
23
                if k not in measurement:
24
                    measurement[k] = None
25
26
            if "table" not in measurement["metadata"]:
27
                measurement["metadata"]["table"] = None
28
29
30
def test_pre_birth() -> None:
31
    patient = {
32
        "patient_id": 123,
33
        "events": [
34
            {"time": datetime.datetime(1999, 7, 2), "measurements": [{"code": 1234}]},
35
            {"time": datetime.datetime(1999, 7, 9), "measurements": [{"code": meds.birth_code}]},
36
            {"time": datetime.datetime(1999, 7, 11), "measurements": [{"code": 12345}]},
37
        ],
38
    }
39
40
    expected = {
41
        "patient_id": 123,
42
        "events": [
43
            {"time": datetime.datetime(1999, 7, 9), "measurements": [{"code": 1234}]},
44
            {"time": datetime.datetime(1999, 7, 9), "measurements": [{"code": meds.birth_code}]},
45
            {"time": datetime.datetime(1999, 7, 11), "measurements": [{"code": 12345}]},
46
        ],
47
    }
48
49
    cleanup(patient)
50
    cleanup(expected)
51
52
    assert move_pre_birth(patient) == expected
53
54
55
def test_move_visit_start_ignores_other_visits() -> None:
56
    patient = {
57
        "patient_id": 123,
58
        "events": [
59
            {  # A non-visit event with no explicit start time
60
                "time": datetime.datetime(1999, 7, 2),
61
                "measurements": [{"code": 1234, "metadata": {"visit_id": 9999}}],
62
            },
63
            {  # A visit event with just date specified
64
                "time": datetime.datetime(1999, 7, 2),
65
                "measurements": [
66
                    {
67
                        "code": 4567,
68
                        "metadata": {
69
                            "table": "visit",
70
                            "visit_id": 9999,
71
                        },
72
                    }
73
                ],
74
            },
75
            {  # A non-visit event from a separate visit ID
76
                "time": datetime.datetime(1999, 7, 2, 11),
77
                "measurements": [{"code": 2345, "metadata": {"visit_id": 8888}}],
78
            },
79
            {  # First recorded non-visit event for visit ID 9999
80
                "time": datetime.datetime(1999, 7, 2, 12),
81
                "measurements": [{"code": 3456, "metadata": {"visit_id": 9999}}],
82
            },
83
        ],
84
    }
85
86
    # Note that events are implicitly sorted first by start time, then by code:
87
    # https://github.com/som-shahlab/femr/blob/main/src/femr/__init__.py#L69
88
    expected = {
89
        "patient_id": 123,
90
        "events": [
91
            {  # A non-visit event with no explicit start time
92
                "time": datetime.datetime(1999, 7, 2),
93
                "measurements": [{"code": 1234, "metadata": {"visit_id": 9999}}],
94
            },
95
            {  # A non-visit event from a separate visit ID
96
                "time": datetime.datetime(1999, 7, 2, 11),
97
                "measurements": [{"code": 2345, "metadata": {"visit_id": 8888}}],
98
            },
99
            {  # Now visit event has date and time specified
100
                "time": datetime.datetime(1999, 7, 2, 12),
101
                "measurements": [
102
                    {
103
                        "code": 4567,
104
                        "metadata": {
105
                            "table": "visit",
106
                            "visit_id": 9999,
107
                        },
108
                    }
109
                ],
110
            },
111
            {  # First recorded non-visit event for visit ID 9999
112
                "time": datetime.datetime(1999, 7, 2, 12),
113
                "measurements": [{"code": 3456, "metadata": {"visit_id": 9999}}],
114
            },
115
        ],
116
    }
117
118
    cleanup(patient)
119
    cleanup(expected)
120
121
    assert move_visit_start_to_first_event_start(patient) == expected
122
123
124
def test_move_visit_start_minute_after_midnight() -> None:
125
    patient = {
126
        "patient_id": 123,
127
        "events": [
128
            {
129
                "time": datetime.datetime(1999, 7, 2),
130
                "measurements": [
131
                    {"code": 3456, "metadata": {"visit_id": 9999, "table": "visit"}},
132
                    {"code": 1234, "metadata": {"visit_id": 9999}},
133
                ],
134
            },
135
            {
136
                "time": datetime.datetime(1999, 7, 2, 0, 1),
137
                "measurements": [{"code": 2345, "metadata": {"visit_id": 9999}}],
138
            },
139
            {
140
                "time": datetime.datetime(1999, 7, 2, 12),
141
                "measurements": [{"code": 4567, "metadata": {"visit_id": 9999}}],
142
            },
143
        ],
144
    }
145
146
    expected = {
147
        "patient_id": 123,
148
        "events": [
149
            {
150
                "time": datetime.datetime(1999, 7, 2),
151
                "measurements": [{"code": 1234, "metadata": {"visit_id": 9999}}],
152
            },
153
            {
154
                "time": datetime.datetime(1999, 7, 2, 0, 1),
155
                "measurements": [{"code": 3456, "metadata": {"visit_id": 9999, "table": "visit"}}],
156
            },
157
            {
158
                "time": datetime.datetime(1999, 7, 2, 0, 1),
159
                "measurements": [{"code": 2345, "metadata": {"visit_id": 9999}}],
160
            },
161
            {
162
                "time": datetime.datetime(1999, 7, 2, 12),
163
                "measurements": [{"code": 4567, "metadata": {"visit_id": 9999}}],
164
            },
165
        ],
166
    }
167
168
    cleanup(patient)
169
    cleanup(expected)
170
171
    assert move_visit_start_to_first_event_start(patient) == expected
172
173
174
def test_move_visit_start_doesnt_move_without_event() -> None:
175
    patient = {
176
        "patient_id": 123,
177
        "events": [
178
            {
179
                "time": datetime.datetime(1999, 7, 2),
180
                "measurements": [
181
                    {"code": 1234, "metadata": {"visit_id": 9999}},
182
                    {"code": 3456, "metadata": {"visit_id": 9999, "table": "visit"}},
183
                    {"code": 2345, "metadata": {"visit_id": 9999}},
184
                ],
185
            },
186
        ],
187
    }
188
189
    # None of the non-visit events have start time > '00:00:00' so visit event
190
    # start time is unchanged, though order changes based on code under resort.
191
    expected = {
192
        "patient_id": 123,
193
        "events": [
194
            {
195
                "time": datetime.datetime(1999, 7, 2),
196
                "measurements": [
197
                    {"code": 1234, "metadata": {"visit_id": 9999}},
198
                    {"code": 3456, "metadata": {"visit_id": 9999, "table": "visit"}},
199
                    {"code": 2345, "metadata": {"visit_id": 9999}},
200
                ],
201
            }
202
        ],
203
    }
204
205
    cleanup(patient)
206
    cleanup(expected)
207
208
    assert move_visit_start_to_first_event_start(patient) == expected
209
210
211
def test_move_to_day_end() -> None:
212
    patient = {
213
        "patient_id": 123,
214
        "events": [
215
            {"time": datetime.datetime(1999, 7, 2), "measurements": [{"code": 1234}]},
216
            {"time": datetime.datetime(1999, 7, 2, 12), "measurements": [{"code": 4321}]},
217
            {"time": datetime.datetime(1999, 7, 9), "measurements": [{"code": meds.birth_code}]},
218
        ],
219
    }
220
221
    expected = {
222
        "patient_id": 123,
223
        "events": [
224
            {"time": datetime.datetime(1999, 7, 2, 12), "measurements": [{"code": 4321}]},
225
            {"time": datetime.datetime(1999, 7, 2, 23, 59), "measurements": [{"code": 1234}]},
226
            {"time": datetime.datetime(1999, 7, 9, 23, 59), "measurements": [{"code": meds.birth_code}]},
227
        ],
228
    }
229
230
    cleanup(patient)
231
    cleanup(expected)
232
233
    print(move_to_day_end(patient))
234
    print(expected)
235
236
    assert move_to_day_end(patient) == expected
237
238
239
def test_remove_nones() -> None:
240
    patient = {
241
        "patient_id": 123,
242
        "events": [
243
            {"time": datetime.datetime(1999, 7, 2), "measurements": [{"code": 1234}]},  # No value, to be removed
244
            {"time": datetime.datetime(1999, 7, 2, 12), "measurements": [{"code": 1234, "numeric_value": 3}]},
245
            {"time": datetime.datetime(1999, 7, 9), "measurements": [{"code": meds.birth_code}]},
246
        ],
247
    }
248
249
    expected = {
250
        "patient_id": 123,
251
        "events": [
252
            {"time": datetime.datetime(1999, 7, 2, 12), "measurements": [{"code": 1234, "numeric_value": 3}]},
253
            {"time": datetime.datetime(1999, 7, 9), "measurements": [{"code": meds.birth_code}]},
254
        ],
255
    }
256
257
    cleanup(patient)
258
    cleanup(expected)
259
260
    assert remove_nones(patient) == expected
261
262
263
def test_delta_encode() -> None:
264
    patient = {
265
        "patient_id": 123,
266
        "events": [
267
            {"time": datetime.datetime(1999, 7, 2), "measurements": [{"code": 1234}]},
268
            {"time": datetime.datetime(1999, 7, 2), "measurements": [{"code": 1234}]},
269
            {"time": datetime.datetime(1999, 7, 2, 12), "measurements": [{"code": 1234, "numeric_value": 3}]},
270
            {"time": datetime.datetime(1999, 7, 2, 14), "measurements": [{"code": 1234, "numeric_value": 3}]},
271
            {"time": datetime.datetime(1999, 7, 2, 19), "measurements": [{"code": 1234, "numeric_value": 5}]},
272
            {"time": datetime.datetime(1999, 7, 2, 20), "measurements": [{"code": 1234, "numeric_value": 3}]},
273
        ],
274
    }
275
276
    expected = {
277
        "patient_id": 123,
278
        "events": [
279
            {"time": datetime.datetime(1999, 7, 2), "measurements": [{"code": 1234}]},
280
            {"time": datetime.datetime(1999, 7, 2, 12), "measurements": [{"code": 1234, "numeric_value": 3}]},
281
            {"time": datetime.datetime(1999, 7, 2, 19), "measurements": [{"code": 1234, "numeric_value": 5}]},
282
            {"time": datetime.datetime(1999, 7, 2, 20), "measurements": [{"code": 1234, "numeric_value": 3}]},
283
        ],
284
    }
285
286
    cleanup(patient)
287
    cleanup(expected)
288
289
    assert delta_encode(patient) == expected
290
291
292
def test_move_billing_codes() -> None:
293
    patient = {
294
        "patient_id": 123,
295
        "events": [
296
            {
297
                "time": datetime.datetime(1999, 7, 2),
298
                "measurements": [
299
                    {
300
                        "code": 1234,
301
                        "metadata": {
302
                            "end": datetime.datetime(1999, 7, 20),
303
                            "visit_id": 10,
304
                            "clarity_table": "lpch_pat_enc",
305
                        },
306
                    }
307
                ],
308
            },
309
            {
310
                "time": datetime.datetime(1999, 7, 9),
311
                "measurements": [
312
                    {
313
                        "code": meds.birth_code,
314
                        "metadata": {
315
                            "visit_id": 10,
316
                            "clarity_table": "lpch_pat_enc_dx",
317
                        },
318
                    }
319
                ],
320
            },
321
            {
322
                "time": datetime.datetime(1999, 7, 10),
323
                "measurements": [
324
                    {
325
                        "code": 42165,
326
                        "metadata": {
327
                            "visit_id": 10,
328
                            "clarity_table": "shc_pat_enc_dx",
329
                        },
330
                    }
331
                ],
332
            },
333
            {
334
                "time": datetime.datetime(1999, 7, 11),
335
                "measurements": [
336
                    {
337
                        "code": 12345,
338
                        "metadata": {
339
                            "visit_id": 10,
340
                        },
341
                    }
342
                ],
343
            },
344
            {
345
                "time": datetime.datetime(1999, 7, 13),
346
                "measurements": [
347
                    {
348
                        "code": 123,
349
                        "metadata": {
350
                            "visit_id": 11,
351
                        },
352
                    }
353
                ],
354
            },
355
        ],
356
    }
357
358
    expected = {
359
        "patient_id": 123,
360
        "events": [
361
            {
362
                "time": datetime.datetime(1999, 7, 2),
363
                "measurements": [
364
                    {
365
                        "code": 1234,
366
                        "metadata": {
367
                            "end": datetime.datetime(1999, 7, 20),
368
                            "visit_id": 10,
369
                            "clarity_table": "lpch_pat_enc",
370
                        },
371
                    }
372
                ],
373
            },
374
            {
375
                "time": datetime.datetime(1999, 7, 11),
376
                "measurements": [
377
                    {
378
                        "code": 12345,
379
                        "metadata": {
380
                            "visit_id": 10,
381
                        },
382
                    }
383
                ],
384
            },
385
            {
386
                "time": datetime.datetime(1999, 7, 13),
387
                "measurements": [
388
                    {
389
                        "code": 123,
390
                        "metadata": {
391
                            "visit_id": 11,
392
                        },
393
                    }
394
                ],
395
            },
396
            {
397
                "time": datetime.datetime(1999, 7, 20),
398
                "measurements": [
399
                    {
400
                        "code": meds.birth_code,
401
                        "metadata": {
402
                            "visit_id": 10,
403
                            "clarity_table": "lpch_pat_enc_dx",
404
                        },
405
                    }
406
                ],
407
            },
408
            {
409
                "time": datetime.datetime(1999, 7, 20),
410
                "measurements": [
411
                    {
412
                        "code": 42165,
413
                        "metadata": {
414
                            "visit_id": 10,
415
                            "clarity_table": "shc_pat_enc_dx",
416
                        },
417
                    }
418
                ],
419
            },
420
        ],
421
    }
422
423
    cleanup(patient)
424
    cleanup(expected)
425
426
    print(move_billing_codes(patient))
427
    print(expected)
428
429
    assert move_billing_codes(patient) == expected