{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%matplotlib inline\n", "from IPython.display import display, HTML" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Build Data Definitions" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import icu_data_defs" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "data_dict = icu_data_defs.data_dictionary('config/data_definitions.xlsx')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelunitsvariable_typeclinical_sourcelist_id
def_id
0heart ratebeats/minqnobservationNaN
1blood pressure systolicmmHgqnobservationNaN
2blood pressure diastolicmmHgqnobservationNaN
3blood pressure meanmmHgqnobservationNaN
4respiratory rateinsp/minqnobservationNaN
\n", "
" ], "text/plain": [ " label units variable_type clinical_source \\\n", "def_id \n", "0 heart rate beats/min qn observation \n", "1 blood pressure systolic mmHg qn observation \n", "2 blood pressure diastolic mmHg qn observation \n", "3 blood pressure mean mmHg qn observation \n", "4 respiratory rate insp/min qn observation \n", "\n", " list_id \n", "def_id \n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_dict.tables.definitions.head()" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "collapsed": true }, "outputs": [], "source": [ "inr_id = data_dict.add_definition(label='INR')\n", "pt_id = data_dict.add_definition(label='Prothrombin Time',units='seconds')" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelunitsvariable_typeclinical_sourcelist_id
def_id
23lactatemmol/LqnobservationNaN
24lactatemg/dLqnobservationNaN
25hemoglobing/dLqnobservationNaN
26INRNaNqnobservationNaN
27Prothrombin TimesecondsqnobservationNaN
\n", "
" ], "text/plain": [ " label units variable_type clinical_source list_id\n", "def_id \n", "23 lactate mmol/L qn observation NaN\n", "24 lactate mg/dL qn observation NaN\n", "25 hemoglobin g/dL qn observation NaN\n", "26 INR NaN qn observation NaN\n", "27 Prothrombin Time seconds qn observation NaN" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_dict.tables.definitions.tail()" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelunitsvariable_typeclinical_sourcelist_id
def_id
26INRNaNqnobservationNaN
27Prothrombin TimesecondsqnobservationNaN
\n", "
" ], "text/plain": [ " label units variable_type clinical_source list_id\n", "def_id \n", "26 INR NaN qn observation NaN\n", "27 Prothrombin Time seconds qn observation NaN" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from itertools import product\n", "\n", "panel_map = list(product([data_dict.table_names.definitions],[inr_id,pt_id]))\n", "panel_id = data_dict.add_panel('coagulation labs',panel_map)\n", "data_dict.get_panel_defintions(panel_id)" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
panel_namelist_id
panel_id
0blood pressure3
1vital signs4
2urine output5
3glasgow coma scale6
4normal saline7
5lactated ringers8
6common fluids9
7norepinephrine10
8vasopressin11
9common pressors12
10lactate13
11oxygen delivery labs14
12simple dataset15
13coagulation labs16
\n", "
" ], "text/plain": [ " panel_name list_id\n", "panel_id \n", "0 blood pressure 3\n", "1 vital signs 4\n", "2 urine output 5\n", "3 glasgow coma scale 6\n", "4 normal saline 7\n", "5 lactated ringers 8\n", "6 common fluids 9\n", "7 norepinephrine 10\n", "8 vasopressin 11\n", "9 common pressors 12\n", "10 lactate 13\n", "11 oxygen delivery labs 14\n", "12 simple dataset 15\n", "13 coagulation labs 16" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_dict.tables.panels" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelunitsvariable_typeclinical_sourcelist_id
def_id
0heart ratebeats/minqnobservationNaN
1blood pressure systolicmmHgqnobservationNaN
2blood pressure diastolicmmHgqnobservationNaN
3blood pressure meanmmHgqnobservationNaN
4respiratory rateinsp/minqnobservationNaN
5temperature bodydegFqnobservationNaN
6oxygen saturation pulse oximetrypercentqnobservationNaN
7weight bodykgqnobservationNaN
8output urinemLqnobservationNaN
9output urinemL/hrqnobservationNaN
10output urinemL/kg/hrqnobservationNaN
11glasgow coma scale motorNaNordobservation0.0
12glasgow coma scale eye openingNaNordobservation2.0
13glasgow coma scale verbalNaNordobservation1.0
14normal salinemLqninterventionNaN
15normal salinemL/hrqninterventionNaN
16lactated ringersmLqninterventionNaN
17lactated ringersmL/hrqninterventionNaN
18norepinephrinemcgqninterventionNaN
19norepinephrinemcg/minqninterventionNaN
20norepinephrinemcg/kg/minqninterventionNaN
21vasopressinunitsqninterventionNaN
22vasopressinunits/minqninterventionNaN
25hemoglobing/dLqnobservationNaN
23lactatemmol/LqnobservationNaN
24lactatemg/dLqnobservationNaN
\n", "
" ], "text/plain": [ " label units variable_type \\\n", "def_id \n", "0 heart rate beats/min qn \n", "1 blood pressure systolic mmHg qn \n", "2 blood pressure diastolic mmHg qn \n", "3 blood pressure mean mmHg qn \n", "4 respiratory rate insp/min qn \n", "5 temperature body degF qn \n", "6 oxygen saturation pulse oximetry percent qn \n", "7 weight body kg qn \n", "8 output urine mL qn \n", "9 output urine mL/hr qn \n", "10 output urine mL/kg/hr qn \n", "11 glasgow coma scale motor NaN ord \n", "12 glasgow coma scale eye opening NaN ord \n", "13 glasgow coma scale verbal NaN ord \n", "14 normal saline mL qn \n", "15 normal saline mL/hr qn \n", "16 lactated ringers mL qn \n", "17 lactated ringers mL/hr qn \n", "18 norepinephrine mcg qn \n", "19 norepinephrine mcg/min qn \n", "20 norepinephrine mcg/kg/min qn \n", "21 vasopressin units qn \n", "22 vasopressin units/min qn \n", "25 hemoglobin g/dL qn \n", "23 lactate mmol/L qn \n", "24 lactate mg/dL qn \n", "\n", " clinical_source list_id \n", "def_id \n", "0 observation NaN \n", "1 observation NaN \n", "2 observation NaN \n", "3 observation NaN \n", "4 observation NaN \n", "5 observation NaN \n", "6 observation NaN \n", "7 observation NaN \n", "8 observation NaN \n", "9 observation NaN \n", "10 observation NaN \n", "11 observation 0.0 \n", "12 observation 2.0 \n", "13 observation 1.0 \n", "14 intervention NaN \n", "15 intervention NaN \n", "16 intervention NaN \n", "17 intervention NaN \n", "18 intervention NaN \n", "19 intervention NaN \n", "20 intervention NaN \n", "21 intervention NaN \n", "22 intervention NaN \n", "25 observation NaN \n", "23 observation NaN \n", "24 observation NaN " ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_dict.get_panel_defintions(12)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "collapsed": true }, "outputs": [], "source": [ "data_dict.save()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# MIMIC Exploration" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import mimic\n", "from icu_data_defs import data_dictionary" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "reload(mimic)\n", "conn = mimic.connect()\n", "data_dict = data_dictionary('config/data_definitions.xlsx')\n", "explorer = mimic.explorer(conn)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "keep_dict = {}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Vital Signs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Heart Rate" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
227018HR_ApacheIVHR_ApacheIVcharteventsScores - APACHE IV (2)bpm110.000000
223765Orthostatic HR sittingOrthostatic HR sittingcharteventsRoutine Vital Signsbpm110.000000
1332pulsechartevents110.000000
227581BiPap bpm (S/T -Back up)BiPap bpm (S/T -Back up)charteventsRespiratorybpm110.000000
220047Heart Rate Alarm - LowHR Alarm - LowcharteventsAlarmsbpm110.000000
224363VAD Beat Rate LVAD Beat Rate LcharteventsHemodynamicsbpm110.000000
226764HrApacheIIValueHrApacheIIValuecharteventsScores - APACHE IIbpm110.000000
1341PULSEchartevents110.000000
220045Heart RateHRcharteventsRoutine Vital Signsbpm110.000000
223775VAD Beat Rate RVAD Beat Rate RcharteventsHemodynamicsbpm110.000000
220046Heart rate Alarm - HighHR Alarm - HighcharteventsAlarmsbpm110.000000
224751Temporary Pacemaker RateTemp Pacemaker RatecharteventsCardiovascular (Pacer Data)bpm110.000000
223764Orthostatic HR lyingOrthostatic HR lyingcharteventsRoutine Vital Signsbpm110.000000
1725Pulsechartevents110.000000
224422Spont RRSpont RRcharteventsRespiratorybpm110.000000
224647Orthostatic HR standingOrthostatic HR standingcharteventsRoutine Vital Signsbpm110.000000
211Heart Ratechartevents110.000000
220181Non Invasive Blood Pressure meanNBPmcharteventsRoutine Vital SignsmmHg100.666667
220052Arterial Blood Pressure meanABPmcharteventsRoutine Vital SignsmmHg100.666667
3494Lowest Heart Ratechartevents92.666667
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "227018 HR_ApacheIV HR_ApacheIV \n", "223765 Orthostatic HR sitting Orthostatic HR sitting \n", "1332 pulse \n", "227581 BiPap bpm (S/T -Back up) BiPap bpm (S/T -Back up) \n", "220047 Heart Rate Alarm - Low HR Alarm - Low \n", "224363 VAD Beat Rate L VAD Beat Rate L \n", "226764 HrApacheIIValue HrApacheIIValue \n", "1341 PULSE \n", "220045 Heart Rate HR \n", "223775 VAD Beat Rate R VAD Beat Rate R \n", "220046 Heart rate Alarm - High HR Alarm - High \n", "224751 Temporary Pacemaker Rate Temp Pacemaker Rate \n", "223764 Orthostatic HR lying Orthostatic HR lying \n", "1725 Pulse \n", "224422 Spont RR Spont RR \n", "224647 Orthostatic HR standing Orthostatic HR standing \n", "211 Heart Rate \n", "220181 Non Invasive Blood Pressure mean NBPm \n", "220052 Arterial Blood Pressure mean ABPm \n", "3494 Lowest Heart Rate \n", "\n", " linksto category unitname score \n", "itemid \n", "227018 chartevents Scores - APACHE IV (2) bpm 110.000000 \n", "223765 chartevents Routine Vital Signs bpm 110.000000 \n", "1332 chartevents 110.000000 \n", "227581 chartevents Respiratory bpm 110.000000 \n", "220047 chartevents Alarms bpm 110.000000 \n", "224363 chartevents Hemodynamics bpm 110.000000 \n", "226764 chartevents Scores - APACHE II bpm 110.000000 \n", "1341 chartevents 110.000000 \n", "220045 chartevents Routine Vital Signs bpm 110.000000 \n", "223775 chartevents Hemodynamics bpm 110.000000 \n", "220046 chartevents Alarms bpm 110.000000 \n", "224751 chartevents Cardiovascular (Pacer Data) bpm 110.000000 \n", "223764 chartevents Routine Vital Signs bpm 110.000000 \n", "1725 chartevents 110.000000 \n", "224422 chartevents Respiratory bpm 110.000000 \n", "224647 chartevents Routine Vital Signs bpm 110.000000 \n", "211 chartevents 110.000000 \n", "220181 chartevents Routine Vital Signs mmHg 100.666667 \n", "220052 chartevents Routine Vital Signs mmHg 100.666667 \n", "3494 chartevents 92.666667 " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'heart rate',\n", " 'beats',\n", " 'bpm',\n", " 'pulse'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false }, "outputs": [], "source": [ "label = data_dict.labels.HEART_RATE\n", "keep_dict[label] = [211,220045,1341,1725,1332]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Respiratory Rate" ] }, { "cell_type": "code", "execution_count": 66, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
3603Resp Ratechartevents110.000000
224688Respiratory Rate (Set)Respiratory Rate (Set)charteventsRespiratoryinsp/min110.000000
226774RRApacheIIValueRRApacheIIValuecharteventsScores - APACHE IIinsp/min110.000000
618Respiratory Ratechartevents110.000000
224690Respiratory Rate (Total)Respiratory Rate (Total)charteventsRespiratoryinsp/min110.000000
223875Fspn HighFspn HighcharteventsRespiratoryinsp/min110.000000
224689Respiratory Rate (spontaneous)Respiratory Rate (spontaneous)charteventsRespiratoryinsp/min110.000000
220210Respiratory RateRRcharteventsRespiratoryinsp/min110.000000
224161Resp Alarm - HighResp Alarm - HighcharteventsAlarmsinsp/min110.000000
227050RR_ApacheIVRR_ApacheIVcharteventsScores - APACHE IV (2)insp/min110.000000
224162Resp Alarm - LowResp Alarm - LowcharteventsAlarmsinsp/min110.000000
619Respiratory Rate Setchartevents102.666667
225949NIV MaskNIV MaskRespiratory97.333333
223840ETT Re-tapedETT Re-tapedRespiratory97.333333
223838ETT LocationETT LocationRespiratory97.333333
223837ETT Size (ID)ETT Size (ID)Respiratory97.333333
224373Sputum AmountSputum AmountRespiratory97.333333
223836Airway TypeAirway TypeRespiratory97.333333
226815Airway problemsAirway problemsRespiratory97.333333
223835Inspired O2 FractionFiO2charteventsRespiratoryNone97.333333
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "3603 Resp Rate \n", "224688 Respiratory Rate (Set) Respiratory Rate (Set) \n", "226774 RRApacheIIValue RRApacheIIValue \n", "618 Respiratory Rate \n", "224690 Respiratory Rate (Total) Respiratory Rate (Total) \n", "223875 Fspn High Fspn High \n", "224689 Respiratory Rate (spontaneous) Respiratory Rate (spontaneous) \n", "220210 Respiratory Rate RR \n", "224161 Resp Alarm - High Resp Alarm - High \n", "227050 RR_ApacheIV RR_ApacheIV \n", "224162 Resp Alarm - Low Resp Alarm - Low \n", "619 Respiratory Rate Set \n", "225949 NIV Mask NIV Mask \n", "223840 ETT Re-taped ETT Re-taped \n", "223838 ETT Location ETT Location \n", "223837 ETT Size (ID) ETT Size (ID) \n", "224373 Sputum Amount Sputum Amount \n", "223836 Airway Type Airway Type \n", "226815 Airway problems Airway problems \n", "223835 Inspired O2 Fraction FiO2 \n", "\n", " linksto category unitname score \n", "itemid \n", "3603 chartevents 110.000000 \n", "224688 chartevents Respiratory insp/min 110.000000 \n", "226774 chartevents Scores - APACHE II insp/min 110.000000 \n", "618 chartevents 110.000000 \n", "224690 chartevents Respiratory insp/min 110.000000 \n", "223875 chartevents Respiratory insp/min 110.000000 \n", "224689 chartevents Respiratory insp/min 110.000000 \n", "220210 chartevents Respiratory insp/min 110.000000 \n", "224161 chartevents Alarms insp/min 110.000000 \n", "227050 chartevents Scores - APACHE IV (2) insp/min 110.000000 \n", "224162 chartevents Alarms insp/min 110.000000 \n", "619 chartevents 102.666667 \n", "225949 Respiratory 97.333333 \n", "223840 Respiratory 97.333333 \n", "223838 Respiratory 97.333333 \n", "223837 Respiratory 97.333333 \n", "224373 Respiratory 97.333333 \n", "223836 Respiratory 97.333333 \n", "226815 Respiratory 97.333333 \n", "223835 chartevents Respiratory None 97.333333 " ] }, "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'respiratory rate',\n", " 'resp rate',\n", " 'insp/min',\n", " 'breath/min'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 67, "metadata": { "collapsed": true }, "outputs": [], "source": [ "label = data_dict.labels.RESPIRATORY_RATE\n", "keep_dict[label] = [220210,3603,618,8113,615,219]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Blood Pressure" ] }, { "cell_type": "code", "execution_count": 68, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
224315ABI Brachial BP LABI Brachial BP LcharteventsIABPmmHg110.0
227537ART Blood Pressure Alarm - HighART BP Alarm - HighcharteventsAlarmsmmHg110.0
220181Non Invasive Blood Pressure meanNBPmcharteventsRoutine Vital SignsmmHg110.0
220180Non Invasive Blood Pressure diastolicNBPdcharteventsRoutine Vital SignsmmHg110.0
220179Non Invasive Blood Pressure systolicNBPscharteventsRoutine Vital SignsmmHg110.0
220074Central Venous PressureCVPcharteventsHemodynamicsmmHg110.0
220073Central Venous Pressure Alarm - LowCVP Alarm - LowcharteventsAlarmsmmHg110.0
220072Central Venous Pressure Alarm - HighCVP Alarm - HighcharteventsAlarmsmmHg110.0
220069Left Artrial PressureLAPcharteventsHemodynamicsmmHg110.0
220066Pulmonary Artery Pressure Alarm - LowPAP Alarm - LowcharteventsAlarmsmmHg110.0
220063Pulmonary Artery Pressure Alarm - HighPAP Alarm - HighcharteventsAlarmsmmHg110.0
220061Pulmonary Artery Pressure meanPAPmcharteventsHemodynamicsmmHg110.0
220060Pulmonary Artery Pressure diastolicPAPdcharteventsHemodynamicsmmHg110.0
220059Pulmonary Artery Pressure systolicPAPscharteventsHemodynamicsmmHg110.0
220058Arterial Blood Pressure Alarm - HighABP Alarm - HighcharteventsAlarmsmmHg110.0
227538ART Blood Pressure Alarm - LowART BP Alarm - LowcharteventsAlarmsmmHg110.0
226096Orthostatic BPd standingOrthostatic BPd standingcharteventsRoutine Vital SignsmmHg110.0
224152Return PressureReturn PressurecharteventsDialysismmHg110.0
227066Cerebral Perfusion PressureCPPcharteventsHemodynamicsmmHg110.0
226062Venous CO2 PressurePCO2 (Venous)charteventsLabsmmHg110.0
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "224315 ABI Brachial BP L ABI Brachial BP L \n", "227537 ART Blood Pressure Alarm - High ART BP Alarm - High \n", "220181 Non Invasive Blood Pressure mean NBPm \n", "220180 Non Invasive Blood Pressure diastolic NBPd \n", "220179 Non Invasive Blood Pressure systolic NBPs \n", "220074 Central Venous Pressure CVP \n", "220073 Central Venous Pressure Alarm - Low CVP Alarm - Low \n", "220072 Central Venous Pressure Alarm - High CVP Alarm - High \n", "220069 Left Artrial Pressure LAP \n", "220066 Pulmonary Artery Pressure Alarm - Low PAP Alarm - Low \n", "220063 Pulmonary Artery Pressure Alarm - High PAP Alarm - High \n", "220061 Pulmonary Artery Pressure mean PAPm \n", "220060 Pulmonary Artery Pressure diastolic PAPd \n", "220059 Pulmonary Artery Pressure systolic PAPs \n", "220058 Arterial Blood Pressure Alarm - High ABP Alarm - High \n", "227538 ART Blood Pressure Alarm - Low ART BP Alarm - Low \n", "226096 Orthostatic BPd standing Orthostatic BPd standing \n", "224152 Return Pressure Return Pressure \n", "227066 Cerebral Perfusion Pressure CPP \n", "226062 Venous CO2 Pressure PCO2 (Venous) \n", "\n", " linksto category unitname score \n", "itemid \n", "224315 chartevents IABP mmHg 110.0 \n", "227537 chartevents Alarms mmHg 110.0 \n", "220181 chartevents Routine Vital Signs mmHg 110.0 \n", "220180 chartevents Routine Vital Signs mmHg 110.0 \n", "220179 chartevents Routine Vital Signs mmHg 110.0 \n", "220074 chartevents Hemodynamics mmHg 110.0 \n", "220073 chartevents Alarms mmHg 110.0 \n", "220072 chartevents Alarms mmHg 110.0 \n", "220069 chartevents Hemodynamics mmHg 110.0 \n", "220066 chartevents Alarms mmHg 110.0 \n", "220063 chartevents Alarms mmHg 110.0 \n", "220061 chartevents Hemodynamics mmHg 110.0 \n", "220060 chartevents Hemodynamics mmHg 110.0 \n", "220059 chartevents Hemodynamics mmHg 110.0 \n", "220058 chartevents Alarms mmHg 110.0 \n", "227538 chartevents Alarms mmHg 110.0 \n", "226096 chartevents Routine Vital Signs mmHg 110.0 \n", "224152 chartevents Dialysis mmHg 110.0 \n", "227066 chartevents Hemodynamics mmHg 110.0 \n", "226062 chartevents Labs mmHg 110.0 " ] }, "execution_count": 68, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'blood pressure',\n", " 'systolic',\n", " 'diastolic',\n", " 'mmHg'\n", " ])\n", "out_df.iloc[0:20]" ] }, { "cell_type": "code", "execution_count": 69, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
226063Venous O2 PressurePO2 (Venous)charteventsLabsmmHg110.0
225312ART BP meanART BP meancharteventsRoutine Vital SignsmmHg110.0
225310ART BP DiastolicART BP DiastoliccharteventsRoutine Vital SignsmmHg110.0
225309ART BP SystolicART BP SystoliccharteventsRoutine Vital SignsmmHg110.0
223751Non-Invasive Blood Pressure Alarm - HighNBP Alarm - HighcharteventsAlarmsmmHg110.0
227516PO2 (Mixed Venous)PO2 (Mixed Venous)charteventsLabsmmHg110.0
224166Doppler BPDoppler BPcharteventsRoutine Vital SignsmmHg110.0
224167Manual Blood Pressure Systolic LeftManual BPs LcharteventsRoutine Vital SignsmmHg110.0
226737AaDO2ApacheIIValueAaDO2ApacheIIValuecharteventsScores - APACHE IImmHg110.0
228148ABI Ankle BP R (Impella)ABI Ankle BP R (Impella)charteventsImpellammHg110.0
220224Arterial O2 pressurePO2 (Arterial)charteventsLabsmmHg110.0
220235Arterial CO2 PressurePCO2 (Arterial)charteventsLabsmmHg110.0
223752Non-Invasive Blood Pressure Alarm - LowNBP Alarm - LowcharteventsAlarmsmmHg110.0
223763Bladder PressureBladder PressurecharteventsRoutine Vital SignsmmHg110.0
224150Filter PressureFilter PressurecharteventsDialysismmHg110.0
224149Access PressureAccess PressurecharteventsDialysismmHg110.0
227991Intra Cranial Pressure #2 Alarm - LowIC2 Alarm - LowcharteventsAlarmsmmHg110.0
226094Orthostatic BPd sittingOrthostatic BPd sittingcharteventsRoutine Vital SignsmmHg110.0
226092Orthostatic BPd lyingOrthostatic BPd lyingcharteventsRoutine Vital SignsmmHg110.0
227990Intra Cranial Pressure #2 Alarm - HighIC2 Alarm - HighcharteventsAlarmsmmHg110.0
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "226063 Venous O2 Pressure PO2 (Venous) \n", "225312 ART BP mean ART BP mean \n", "225310 ART BP Diastolic ART BP Diastolic \n", "225309 ART BP Systolic ART BP Systolic \n", "223751 Non-Invasive Blood Pressure Alarm - High NBP Alarm - High \n", "227516 PO2 (Mixed Venous) PO2 (Mixed Venous) \n", "224166 Doppler BP Doppler BP \n", "224167 Manual Blood Pressure Systolic Left Manual BPs L \n", "226737 AaDO2ApacheIIValue AaDO2ApacheIIValue \n", "228148 ABI Ankle BP R (Impella) ABI Ankle BP R (Impella) \n", "220224 Arterial O2 pressure PO2 (Arterial) \n", "220235 Arterial CO2 Pressure PCO2 (Arterial) \n", "223752 Non-Invasive Blood Pressure Alarm - Low NBP Alarm - Low \n", "223763 Bladder Pressure Bladder Pressure \n", "224150 Filter Pressure Filter Pressure \n", "224149 Access Pressure Access Pressure \n", "227991 Intra Cranial Pressure #2 Alarm - Low IC2 Alarm - Low \n", "226094 Orthostatic BPd sitting Orthostatic BPd sitting \n", "226092 Orthostatic BPd lying Orthostatic BPd lying \n", "227990 Intra Cranial Pressure #2 Alarm - High IC2 Alarm - High \n", "\n", " linksto category unitname score \n", "itemid \n", "226063 chartevents Labs mmHg 110.0 \n", "225312 chartevents Routine Vital Signs mmHg 110.0 \n", "225310 chartevents Routine Vital Signs mmHg 110.0 \n", "225309 chartevents Routine Vital Signs mmHg 110.0 \n", "223751 chartevents Alarms mmHg 110.0 \n", "227516 chartevents Labs mmHg 110.0 \n", "224166 chartevents Routine Vital Signs mmHg 110.0 \n", "224167 chartevents Routine Vital Signs mmHg 110.0 \n", "226737 chartevents Scores - APACHE II mmHg 110.0 \n", "228148 chartevents Impella mmHg 110.0 \n", "220224 chartevents Labs mmHg 110.0 \n", "220235 chartevents Labs mmHg 110.0 \n", "223752 chartevents Alarms mmHg 110.0 \n", "223763 chartevents Routine Vital Signs mmHg 110.0 \n", "224150 chartevents Dialysis mmHg 110.0 \n", "224149 chartevents Dialysis mmHg 110.0 \n", "227991 chartevents Alarms mmHg 110.0 \n", "226094 chartevents Routine Vital Signs mmHg 110.0 \n", "226092 chartevents Routine Vital Signs mmHg 110.0 \n", "227990 chartevents Alarms mmHg 110.0 " ] }, "execution_count": 69, "metadata": {}, "output_type": "execute_result" } ], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] = [220179]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC] = [220180]\n", "out_df.iloc[20:40]" ] }, { "cell_type": "code", "execution_count": 70, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
227989Intra Cranial Pressure #2IC2charteventsHemodynamicsmmHg110.0
220056Arterial Blood Pressure Alarm - LowABP Alarm - LowcharteventsAlarmsmmHg110.0
220052Arterial Blood Pressure meanABPmcharteventsRoutine Vital SignsmmHg110.0
220051Arterial Blood Pressure diastolicABPdcharteventsRoutine Vital SignsmmHg110.0
220050Arterial Blood Pressure systolicABPscharteventsRoutine Vital SignsmmHg110.0
228145ABI Ankle BP LABI Ankle BP LcharteventsIABPmmHg110.0
228146ABI Brachial BP RABI Brachial BP RcharteventsIABPmmHg110.0
227039PO2_ApacheIVPO2_ApacheIVcharteventsScores - APACHE IV (2)mmHg110.0
220765Intra Cranial PressureICPcharteventsHemodynamicsmmHg110.0
227023MAP_ApacheIVMAP_ApacheIVcharteventsScores - APACHE IV (2)mmHg110.0
224309Assisted SystoleSYS - AssistedcharteventsIABPmmHg110.0
224310Augmented DiastoleAUGcharteventsIABPmmHg110.0
224311BAEDPDIA - AssistedcharteventsIABPmmHg110.0
224314ABI Brachial BP R (Impella)ABI Brachial BP R (Impella)charteventsImpellammHg110.0
224317ABI Ankle BP RABI Ankle BP RcharteventsIABPmmHg110.0
224318ABI Ankle BP L (Impella)ABI Ankle BP L (Impella)charteventsImpellammHg110.0
224322IABP MeanMAP - AssistedcharteventsIABPmmHg110.0
223771PCWPPCWPcharteventsHemodynamicsmmHg110.0
223768Intra Cranial Pressure Alarm - LowICP Alarm - LowcharteventsAlarmsmmHg110.0
223767Intra Cranial Pressure Alarm - HighICP Alarm - HighcharteventsAlarmsmmHg110.0
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "227989 Intra Cranial Pressure #2 IC2 \n", "220056 Arterial Blood Pressure Alarm - Low ABP Alarm - Low \n", "220052 Arterial Blood Pressure mean ABPm \n", "220051 Arterial Blood Pressure diastolic ABPd \n", "220050 Arterial Blood Pressure systolic ABPs \n", "228145 ABI Ankle BP L ABI Ankle BP L \n", "228146 ABI Brachial BP R ABI Brachial BP R \n", "227039 PO2_ApacheIV PO2_ApacheIV \n", "220765 Intra Cranial Pressure ICP \n", "227023 MAP_ApacheIV MAP_ApacheIV \n", "224309 Assisted Systole SYS - Assisted \n", "224310 Augmented Diastole AUG \n", "224311 BAEDP DIA - Assisted \n", "224314 ABI Brachial BP R (Impella) ABI Brachial BP R (Impella) \n", "224317 ABI Ankle BP R ABI Ankle BP R \n", "224318 ABI Ankle BP L (Impella) ABI Ankle BP L (Impella) \n", "224322 IABP Mean MAP - Assisted \n", "223771 PCWP PCWP \n", "223768 Intra Cranial Pressure Alarm - Low ICP Alarm - Low \n", "223767 Intra Cranial Pressure Alarm - High ICP Alarm - High \n", "\n", " linksto category unitname score \n", "itemid \n", "227989 chartevents Hemodynamics mmHg 110.0 \n", "220056 chartevents Alarms mmHg 110.0 \n", "220052 chartevents Routine Vital Signs mmHg 110.0 \n", "220051 chartevents Routine Vital Signs mmHg 110.0 \n", "220050 chartevents Routine Vital Signs mmHg 110.0 \n", "228145 chartevents IABP mmHg 110.0 \n", "228146 chartevents IABP mmHg 110.0 \n", "227039 chartevents Scores - APACHE IV (2) mmHg 110.0 \n", "220765 chartevents Hemodynamics mmHg 110.0 \n", "227023 chartevents Scores - APACHE IV (2) mmHg 110.0 \n", "224309 chartevents IABP mmHg 110.0 \n", "224310 chartevents IABP mmHg 110.0 \n", "224311 chartevents IABP mmHg 110.0 \n", "224314 chartevents Impella mmHg 110.0 \n", "224317 chartevents IABP mmHg 110.0 \n", "224318 chartevents Impella mmHg 110.0 \n", "224322 chartevents IABP mmHg 110.0 \n", "223771 chartevents Hemodynamics mmHg 110.0 \n", "223768 chartevents Alarms mmHg 110.0 \n", "223767 chartevents Alarms mmHg 110.0 " ] }, "execution_count": 70, "metadata": {}, "output_type": "execute_result" } ], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] += [225309,224167]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC] += [225310,224643]\n", "out_df.iloc[40:60]" ] }, { "cell_type": "code", "execution_count": 71, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
223766Orthostatic BPs standingOrthostatic BPs standingcharteventsRoutine Vital SignsmmHg110.0
228149ABI Brachial BP L (Impella)ABI Brachial BP L (Impella)charteventsImpellammHg110.0
224151Effluent PressureEffluent PressurecharteventsDialysismmHg110.0
224643Manual Blood Pressure Diastolic LeftManual BPd LcharteventsRoutine Vital SignsmmHg110.0
224654PAEDPDIA - UnassistedcharteventsIABPmmHg110.0
226855PCWP (mean) (PA Line)PCWP (mean) (PA Line)charteventsPA Line InsertionmmHg110.0
226857PA mean pressure (PA Line)PA mean pressure (PA Line)charteventsPA Line InsertionmmHg110.0
224646Orthostatic BPs sittingOrthostatic BPs sittingcharteventsRoutine Vital SignsmmHg110.0
227716Cerebral Perfusion Pressure Alarm - HighCPP Alarm - HighcharteventsAlarmsmmHg110.0
227717Cerebral Perfusion Pressure Alarm - LowCPP Alarm - LowcharteventsAlarmsmmHg110.0
224645Orthostatic BPs lyingOrthostatic BPs lyingcharteventsRoutine Vital SignsmmHg110.0
227242Manual Blood Pressure Diastolic RightManual BPd RcharteventsRoutine Vital SignsmmHg110.0
226766MapApacheIIValueMapApacheIIValuecharteventsScores - APACHE IImmHg110.0
227243Manual Blood Pressure Systolic RightManual BPs RcharteventsRoutine Vital SignsmmHg110.0
226853PA diastolic pressure(PA Line)PA diastolic pressure(PA Line)charteventsPA Line InsertionmmHg110.0
226849RA (mean) pressure (PA Line)RA (mean) pressure (PA Line)charteventsPA Line InsertionmmHg110.0
226850RV systolic pressure(PA Line)RV systolic pressure(PA Line)charteventsPA Line InsertionmmHg110.0
226851RV diastolic pressure(PA Line)RV diastolic pressure(PA Line)charteventsPA Line InsertionmmHg110.0
224659Vacuum AssistVacuum AssistcharteventsHemodynamicsmmHg110.0
226854PCWP (v wave) (PA Line)PCWP (v wave) (PA Line)charteventsPA Line InsertionmmHg110.0
\n", "
" ], "text/plain": [ " label \\\n", "itemid \n", "223766 Orthostatic BPs standing \n", "228149 ABI Brachial BP L (Impella) \n", "224151 Effluent Pressure \n", "224643 Manual Blood Pressure Diastolic Left \n", "224654 PAEDP \n", "226855 PCWP (mean) (PA Line) \n", "226857 PA mean pressure (PA Line) \n", "224646 Orthostatic BPs sitting \n", "227716 Cerebral Perfusion Pressure Alarm - High \n", "227717 Cerebral Perfusion Pressure Alarm - Low \n", "224645 Orthostatic BPs lying \n", "227242 Manual Blood Pressure Diastolic Right \n", "226766 MapApacheIIValue \n", "227243 Manual Blood Pressure Systolic Right \n", "226853 PA diastolic pressure(PA Line) \n", "226849 RA (mean) pressure (PA Line) \n", "226850 RV systolic pressure(PA Line) \n", "226851 RV diastolic pressure(PA Line) \n", "224659 Vacuum Assist \n", "226854 PCWP (v wave) (PA Line) \n", "\n", " abbreviation linksto category \\\n", "itemid \n", "223766 Orthostatic BPs standing chartevents Routine Vital Signs \n", "228149 ABI Brachial BP L (Impella) chartevents Impella \n", "224151 Effluent Pressure chartevents Dialysis \n", "224643 Manual BPd L chartevents Routine Vital Signs \n", "224654 DIA - Unassisted chartevents IABP \n", "226855 PCWP (mean) (PA Line) chartevents PA Line Insertion \n", "226857 PA mean pressure (PA Line) chartevents PA Line Insertion \n", "224646 Orthostatic BPs sitting chartevents Routine Vital Signs \n", "227716 CPP Alarm - High chartevents Alarms \n", "227717 CPP Alarm - Low chartevents Alarms \n", "224645 Orthostatic BPs lying chartevents Routine Vital Signs \n", "227242 Manual BPd R chartevents Routine Vital Signs \n", "226766 MapApacheIIValue chartevents Scores - APACHE II \n", "227243 Manual BPs R chartevents Routine Vital Signs \n", "226853 PA diastolic pressure(PA Line) chartevents PA Line Insertion \n", "226849 RA (mean) pressure (PA Line) chartevents PA Line Insertion \n", "226850 RV systolic pressure(PA Line) chartevents PA Line Insertion \n", "226851 RV diastolic pressure(PA Line) chartevents PA Line Insertion \n", "224659 Vacuum Assist chartevents Hemodynamics \n", "226854 PCWP (v wave) (PA Line) chartevents PA Line Insertion \n", "\n", " unitname score \n", "itemid \n", "223766 mmHg 110.0 \n", "228149 mmHg 110.0 \n", "224151 mmHg 110.0 \n", "224643 mmHg 110.0 \n", "224654 mmHg 110.0 \n", "226855 mmHg 110.0 \n", "226857 mmHg 110.0 \n", "224646 mmHg 110.0 \n", "227716 mmHg 110.0 \n", "227717 mmHg 110.0 \n", "224645 mmHg 110.0 \n", "227242 mmHg 110.0 \n", "226766 mmHg 110.0 \n", "227243 mmHg 110.0 \n", "226853 mmHg 110.0 \n", "226849 mmHg 110.0 \n", "226850 mmHg 110.0 \n", "226851 mmHg 110.0 \n", "224659 mmHg 110.0 \n", "226854 mmHg 110.0 " ] }, "execution_count": 71, "metadata": {}, "output_type": "execute_result" } ], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] += [220050]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC] += [220051]\n", "out_df.iloc[60:80]" ] }, { "cell_type": "code", "execution_count": 72, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
228158Purge PressurePurge PressurecharteventsImpellammHg110.000000
226852PA systolic pressure(PA Line)PA systolic pressure(PA Line)charteventsPA Line InsertionmmHg110.000000
224652Unassisted SystoleSYS - UnassistedcharteventsIABPmmHg110.000000
7643RVSYSTOLICchartevents102.666667
8441NBP [Diastolic]chartevents95.666667
8364ABP [Diastolic]chartevents95.666667
8448PAP [Diastolic]chartevents95.666667
455NBP [Systolic]chartevents94.333333
6ABP [Systolic]chartevents94.333333
492PAP [Systolic]chartevents94.333333
8508BP UAC [Diastolic]chartevents89.666667
8505BP PAL [Diastolic]chartevents89.666667
3319BP PAL [Systolic]chartevents88.000000
3325BP UAC [Systolic]chartevents88.000000
8502BP Cuff [Diastolic]chartevents87.666667
3313BP Cuff [Systolic]chartevents86.333333
153Diastolic Unloadingchartevents86.000000
227539ART Blood Pressure Alarm SourceART BP Alarm SourceAlarms84.666667
8440Manual BP [Diastolic]chartevents84.666667
666Systolic Unloadingchartevents84.666667
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "228158 Purge Pressure Purge Pressure \n", "226852 PA systolic pressure(PA Line) PA systolic pressure(PA Line) \n", "224652 Unassisted Systole SYS - Unassisted \n", "7643 RVSYSTOLIC \n", "8441 NBP [Diastolic] \n", "8364 ABP [Diastolic] \n", "8448 PAP [Diastolic] \n", "455 NBP [Systolic] \n", "6 ABP [Systolic] \n", "492 PAP [Systolic] \n", "8508 BP UAC [Diastolic] \n", "8505 BP PAL [Diastolic] \n", "3319 BP PAL [Systolic] \n", "3325 BP UAC [Systolic] \n", "8502 BP Cuff [Diastolic] \n", "3313 BP Cuff [Systolic] \n", "153 Diastolic Unloading \n", "227539 ART Blood Pressure Alarm Source ART BP Alarm Source \n", "8440 Manual BP [Diastolic] \n", "666 Systolic Unloading \n", "\n", " linksto category unitname score \n", "itemid \n", "228158 chartevents Impella mmHg 110.000000 \n", "226852 chartevents PA Line Insertion mmHg 110.000000 \n", "224652 chartevents IABP mmHg 110.000000 \n", "7643 chartevents 102.666667 \n", "8441 chartevents 95.666667 \n", "8364 chartevents 95.666667 \n", "8448 chartevents 95.666667 \n", "455 chartevents 94.333333 \n", "6 chartevents 94.333333 \n", "492 chartevents 94.333333 \n", "8508 chartevents 89.666667 \n", "8505 chartevents 89.666667 \n", "3319 chartevents 88.000000 \n", "3325 chartevents 88.000000 \n", "8502 chartevents 87.666667 \n", "3313 chartevents 86.333333 \n", "153 chartevents 86.000000 \n", "227539 Alarms 84.666667 \n", "8440 chartevents 84.666667 \n", "666 chartevents 84.666667 " ] }, "execution_count": 72, "metadata": {}, "output_type": "execute_result" } ], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] += [227243]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC] += [227242]\n", "out_df.iloc[80:100]" ] }, { "cell_type": "code", "execution_count": 73, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
6417low pressurechartevents84.333333
7133abd pressurechartevents84.333333
6944LOW PRESSUREchartevents84.333333
2027Low pressurechartevents84.333333
6107Low Pressurechartevents84.333333
442Manual BP [Systolic]chartevents83.000000
8503BP Left Arm [Diastolic]chartevents82.000000
8504BP Left Leg [Diastolic]chartevents82.000000
8368Arterial BP [Diastolic]chartevents82.000000
8506BP Right Arm [Diastolic]chartevents81.000000
8507BP Right Leg [Diastolic]chartevents81.000000
51Arterial BP [Systolic]chartevents80.000000
3315BP Left Arm [Systolic]chartevents80.000000
3317BP Left Leg [Systolic]chartevents80.000000
3321BP Right Arm [Systolic]chartevents79.000000
3323BP Right Leg [Systolic]chartevents79.000000
8555Arterial BP #2 [Diastolic]chartevents79.000000
29Access mmHgchartevents78.666667
44560bloodinputevents_cv78.666667
70015BLOODmicrobiologyeventsSPECIMEN78.666667
\n", "
" ], "text/plain": [ " label abbreviation linksto category \\\n", "itemid \n", "6417 low pressure chartevents \n", "7133 abd pressure chartevents \n", "6944 LOW PRESSURE chartevents \n", "2027 Low pressure chartevents \n", "6107 Low Pressure chartevents \n", "442 Manual BP [Systolic] chartevents \n", "8503 BP Left Arm [Diastolic] chartevents \n", "8504 BP Left Leg [Diastolic] chartevents \n", "8368 Arterial BP [Diastolic] chartevents \n", "8506 BP Right Arm [Diastolic] chartevents \n", "8507 BP Right Leg [Diastolic] chartevents \n", "51 Arterial BP [Systolic] chartevents \n", "3315 BP Left Arm [Systolic] chartevents \n", "3317 BP Left Leg [Systolic] chartevents \n", "3321 BP Right Arm [Systolic] chartevents \n", "3323 BP Right Leg [Systolic] chartevents \n", "8555 Arterial BP #2 [Diastolic] chartevents \n", "29 Access mmHg chartevents \n", "44560 blood inputevents_cv \n", "70015 BLOOD microbiologyevents SPECIMEN \n", "\n", " unitname score \n", "itemid \n", "6417 84.333333 \n", "7133 84.333333 \n", "6944 84.333333 \n", "2027 84.333333 \n", "6107 84.333333 \n", "442 83.000000 \n", "8503 82.000000 \n", "8504 82.000000 \n", "8368 82.000000 \n", "8506 81.000000 \n", "8507 81.000000 \n", "51 80.000000 \n", "3315 80.000000 \n", "3317 80.000000 \n", "3321 79.000000 \n", "3323 79.000000 \n", "8555 79.000000 \n", "29 78.666667 \n", "44560 78.666667 \n", "70015 78.666667 " ] }, "execution_count": 73, "metadata": {}, "output_type": "execute_result" } ], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] += [455,6]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC] += [8441,8364]\n", "out_df.iloc[100:120]" ] }, { "cell_type": "code", "execution_count": 74, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
51466BLOODNaNlabeventsHEMATOLOGYNaN78.666667
6127LO Presschartevents78.000000
46780Bladdar pressureinputevents_cv77.000000
1404Bladder pressurechartevents77.000000
993bladder pressurechartevents77.000000
6701Arterial BP #2 [Systolic]chartevents77.000000
1411Bladder Pressurechartevents77.000000
996BLADDER PRESSUREchartevents77.000000
44862BLADDER PRESSUREinputevents_cvFree Form Intake77.000000
770ASTcharteventsEnzymes76.666667
220587ASTASTcharteventsLabsNone76.666667
2347BLADDER PRESSURE.chartevents76.000000
6631pressure lowchartevents75.666667
3108PA PRESSUREchartevents75.333333
7504Driv pressurechartevents75.000000
2779bladder pressureschartevents75.000000
43763Bladder presureinputevents_cv74.333333
1509bld preschartevents73.666667
6540IntraABd pressurechartevents73.666667
8444Orthostat BP sitting [Diastolic]chartevents73.333333
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "51466 BLOOD NaN labevents \n", "6127 LO Press chartevents \n", "46780 Bladdar pressure inputevents_cv \n", "1404 Bladder pressure chartevents \n", "993 bladder pressure chartevents \n", "6701 Arterial BP #2 [Systolic] chartevents \n", "1411 Bladder Pressure chartevents \n", "996 BLADDER PRESSURE chartevents \n", "44862 BLADDER PRESSURE inputevents_cv \n", "770 AST chartevents \n", "220587 AST AST chartevents \n", "2347 BLADDER PRESSURE. chartevents \n", "6631 pressure low chartevents \n", "3108 PA PRESSURE chartevents \n", "7504 Driv pressure chartevents \n", "2779 bladder pressures chartevents \n", "43763 Bladder presure inputevents_cv \n", "1509 bld pres chartevents \n", "6540 IntraABd pressure chartevents \n", "8444 Orthostat BP sitting [Diastolic] chartevents \n", "\n", " category unitname score \n", "itemid \n", "51466 HEMATOLOGY NaN 78.666667 \n", "6127 78.000000 \n", "46780 77.000000 \n", "1404 77.000000 \n", "993 77.000000 \n", "6701 77.000000 \n", "1411 77.000000 \n", "996 77.000000 \n", "44862 Free Form Intake 77.000000 \n", "770 Enzymes 76.666667 \n", "220587 Labs None 76.666667 \n", "2347 76.000000 \n", "6631 75.666667 \n", "3108 75.333333 \n", "7504 75.000000 \n", "2779 75.000000 \n", "43763 74.333333 \n", "1509 73.666667 \n", "6540 73.666667 \n", "8444 73.333333 " ] }, "execution_count": 74, "metadata": {}, "output_type": "execute_result" } ], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] += [3313]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC] += [8502,8440]\n", "out_df.iloc[120:140]" ] }, { "cell_type": "code", "execution_count": 75, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
8446Orthostatic BP lying [Diastolic]chartevents73.333333
8445OrthostatBP standing [Diastolic]chartevents73.333333
6302low presschartevents72.666667
2562INTRA ABD PRESSUREchartevents72.333333
45682bladder pressure ininputevents_cvFree Form Intake72.333333
7124Low pressure Alarmchartevents72.333333
43744NS bladder pressureinputevents_cvFree Form Intake72.333333
228151Aortic Pressure Signal - DiastolicAortic Pressure Signal - DiastoliccharteventsImpellaNone72.000000
42735Intra-abd. Pressureinputevents_cvFree Form Intake71.666667
484Orthostatic BP lying [Systolic]chartevents71.333333
482OrthostatBP standing [Systolic]chartevents71.333333
480Orthostat BP sitting [Systolic]chartevents71.333333
44344BLADDER PRESSURE FLDinputevents_cvFree Form Intake71.000000
44187Blood emesisoutputeventsFree Form Intake71.000000
3238CSF PRESSUREchartevents71.000000
2965INTRAABDOM. PRESSUREchartevents70.333333
228152Aortic Pressure Signal - SystolicAortic Pressure Signal - SystoliccharteventsImpellaNone70.000000
50934HNaNlabeventsCHEMISTRYNaN70.000000
7574lumbar pressurechartevents69.666667
2704ABDOMINAL PRESSUREchartevents69.666667
\n", "
" ], "text/plain": [ " label \\\n", "itemid \n", "8446 Orthostatic BP lying [Diastolic] \n", "8445 OrthostatBP standing [Diastolic] \n", "6302 low press \n", "2562 INTRA ABD PRESSURE \n", "45682 bladder pressure in \n", "7124 Low pressure Alarm \n", "43744 NS bladder pressure \n", "228151 Aortic Pressure Signal - Diastolic \n", "42735 Intra-abd. Pressure \n", "484 Orthostatic BP lying [Systolic] \n", "482 OrthostatBP standing [Systolic] \n", "480 Orthostat BP sitting [Systolic] \n", "44344 BLADDER PRESSURE FLD \n", "44187 Blood emesis \n", "3238 CSF PRESSURE \n", "2965 INTRAABDOM. PRESSURE \n", "228152 Aortic Pressure Signal - Systolic \n", "50934 H \n", "7574 lumbar pressure \n", "2704 ABDOMINAL PRESSURE \n", "\n", " abbreviation linksto category \\\n", "itemid \n", "8446 chartevents \n", "8445 chartevents \n", "6302 chartevents \n", "2562 chartevents \n", "45682 inputevents_cv Free Form Intake \n", "7124 chartevents \n", "43744 inputevents_cv Free Form Intake \n", "228151 Aortic Pressure Signal - Diastolic chartevents Impella \n", "42735 inputevents_cv Free Form Intake \n", "484 chartevents \n", "482 chartevents \n", "480 chartevents \n", "44344 inputevents_cv Free Form Intake \n", "44187 outputevents Free Form Intake \n", "3238 chartevents \n", "2965 chartevents \n", "228152 Aortic Pressure Signal - Systolic chartevents Impella \n", "50934 NaN labevents CHEMISTRY \n", "7574 chartevents \n", "2704 chartevents \n", "\n", " unitname score \n", "itemid \n", "8446 73.333333 \n", "8445 73.333333 \n", "6302 72.666667 \n", "2562 72.333333 \n", "45682 72.333333 \n", "7124 72.333333 \n", "43744 72.333333 \n", "228151 None 72.000000 \n", "42735 71.666667 \n", "484 71.333333 \n", "482 71.333333 \n", "480 71.333333 \n", "44344 71.000000 \n", "44187 71.000000 \n", "3238 71.000000 \n", "2965 70.333333 \n", "228152 None 70.000000 \n", "50934 NaN 70.000000 \n", "7574 69.666667 \n", "2704 69.666667 " ] }, "execution_count": 75, "metadata": {}, "output_type": "execute_result" } ], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] += [442,3315,51,3317]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC] += [8368,8503,8504,8507,8506]\n", "out_df.iloc[140:160]" ] }, { "cell_type": "code", "execution_count": 76, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] += [3321,3323]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC] += [8555]\n", "keep_dict[data_dict.labels.BLOOD_PRESSURE_MEAN] = [220181,225312,220052]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC] += [6701]" ] }, { "cell_type": "code", "execution_count": 77, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
220179Non Invasive Blood Pressure systolicNBPscharteventsRoutine Vital SignsmmHg110.000000
225309ART BP SystolicART BP SystoliccharteventsRoutine Vital SignsmmHg110.000000
224167Manual Blood Pressure Systolic LeftManual BPs LcharteventsRoutine Vital SignsmmHg110.000000
220050Arterial Blood Pressure systolicABPscharteventsRoutine Vital SignsmmHg110.000000
227243Manual Blood Pressure Systolic RightManual BPs RcharteventsRoutine Vital SignsmmHg110.000000
455NBP [Systolic]chartevents94.333333
6ABP [Systolic]chartevents94.333333
3313BP Cuff [Systolic]chartevents86.333333
442Manual BP [Systolic]chartevents83.000000
3315BP Left Arm [Systolic]chartevents80.000000
51Arterial BP [Systolic]chartevents80.000000
3317BP Left Leg [Systolic]chartevents80.000000
3321BP Right Arm [Systolic]chartevents79.000000
3323BP Right Leg [Systolic]chartevents79.000000
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "220179 Non Invasive Blood Pressure systolic NBPs chartevents \n", "225309 ART BP Systolic ART BP Systolic chartevents \n", "224167 Manual Blood Pressure Systolic Left Manual BPs L chartevents \n", "220050 Arterial Blood Pressure systolic ABPs chartevents \n", "227243 Manual Blood Pressure Systolic Right Manual BPs R chartevents \n", "455 NBP [Systolic] chartevents \n", "6 ABP [Systolic] chartevents \n", "3313 BP Cuff [Systolic] chartevents \n", "442 Manual BP [Systolic] chartevents \n", "3315 BP Left Arm [Systolic] chartevents \n", "51 Arterial BP [Systolic] chartevents \n", "3317 BP Left Leg [Systolic] chartevents \n", "3321 BP Right Arm [Systolic] chartevents \n", "3323 BP Right Leg [Systolic] chartevents \n", "\n", " category unitname score \n", "itemid \n", "220179 Routine Vital Signs mmHg 110.000000 \n", "225309 Routine Vital Signs mmHg 110.000000 \n", "224167 Routine Vital Signs mmHg 110.000000 \n", "220050 Routine Vital Signs mmHg 110.000000 \n", "227243 Routine Vital Signs mmHg 110.000000 \n", "455 94.333333 \n", "6 94.333333 \n", "3313 86.333333 \n", "442 83.000000 \n", "3315 80.000000 \n", "51 80.000000 \n", "3317 80.000000 \n", "3321 79.000000 \n", "3323 79.000000 " ] }, "execution_count": 77, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df.loc[keep_dict[data_dict.labels.BLOOD_PRESSURE_SYSTOLIC]]" ] }, { "cell_type": "code", "execution_count": 79, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
220180Non Invasive Blood Pressure diastolicNBPdcharteventsRoutine Vital SignsmmHg110.000000
225310ART BP DiastolicART BP DiastoliccharteventsRoutine Vital SignsmmHg110.000000
224643Manual Blood Pressure Diastolic LeftManual BPd LcharteventsRoutine Vital SignsmmHg110.000000
220051Arterial Blood Pressure diastolicABPdcharteventsRoutine Vital SignsmmHg110.000000
227242Manual Blood Pressure Diastolic RightManual BPd RcharteventsRoutine Vital SignsmmHg110.000000
8441NBP [Diastolic]chartevents95.666667
8364ABP [Diastolic]chartevents95.666667
8502BP Cuff [Diastolic]chartevents87.666667
8440Manual BP [Diastolic]chartevents84.666667
8368Arterial BP [Diastolic]chartevents82.000000
8503BP Left Arm [Diastolic]chartevents82.000000
8504BP Left Leg [Diastolic]chartevents82.000000
8507BP Right Leg [Diastolic]chartevents81.000000
8506BP Right Arm [Diastolic]chartevents81.000000
8555Arterial BP #2 [Diastolic]chartevents79.000000
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "220180 Non Invasive Blood Pressure diastolic NBPd chartevents \n", "225310 ART BP Diastolic ART BP Diastolic chartevents \n", "224643 Manual Blood Pressure Diastolic Left Manual BPd L chartevents \n", "220051 Arterial Blood Pressure diastolic ABPd chartevents \n", "227242 Manual Blood Pressure Diastolic Right Manual BPd R chartevents \n", "8441 NBP [Diastolic] chartevents \n", "8364 ABP [Diastolic] chartevents \n", "8502 BP Cuff [Diastolic] chartevents \n", "8440 Manual BP [Diastolic] chartevents \n", "8368 Arterial BP [Diastolic] chartevents \n", "8503 BP Left Arm [Diastolic] chartevents \n", "8504 BP Left Leg [Diastolic] chartevents \n", "8507 BP Right Leg [Diastolic] chartevents \n", "8506 BP Right Arm [Diastolic] chartevents \n", "8555 Arterial BP #2 [Diastolic] chartevents \n", "\n", " category unitname score \n", "itemid \n", "220180 Routine Vital Signs mmHg 110.000000 \n", "225310 Routine Vital Signs mmHg 110.000000 \n", "224643 Routine Vital Signs mmHg 110.000000 \n", "220051 Routine Vital Signs mmHg 110.000000 \n", "227242 Routine Vital Signs mmHg 110.000000 \n", "8441 95.666667 \n", "8364 95.666667 \n", "8502 87.666667 \n", "8440 84.666667 \n", "8368 82.000000 \n", "8503 82.000000 \n", "8504 82.000000 \n", "8507 81.000000 \n", "8506 81.000000 \n", "8555 79.000000 " ] }, "execution_count": 79, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df.loc[keep_dict[data_dict.labels.BLOOD_PRESSURE_DIASTOLIC]]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### O2 Sat" ] }, { "cell_type": "code", "execution_count": 80, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
50817OXYGEN SATURATIONNaNlabeventsBLOOD GASNaN110.000000
220277O2 saturation pulseoxymetrySpO2charteventsRespiratory%110.000000
646SpO2chartevents110.000000
228232PAR-Oxygen saturationPAR-Oxygen saturationRoutine Vital Signs102.666667
3785PO2charteventsABG's100.666667
3837pO2charteventsABG'S100.666667
50821PO2NaNlabeventsBLOOD GASNaN100.666667
6719SpO2-Lchartevents96.666667
1341PULSEchartevents78.666667
1725Pulsechartevents78.666667
1332pulsechartevents78.666667
50816OXYGENNaNlabeventsBLOOD GASNaN78.000000
2955JVO2 SATchartevents78.000000
2574MVO2 SATchartevents78.000000
227873Recovery O2 Sat - Aerobic CapacityRecovery O2 Sat - Aerobic CapacitycharteventsOT Notes%76.666667
223772SvO2SvO2charteventsHemodynamics%76.666667
223769O2 Saturation Pulseoxymetry Alarm - HighSpO2 Alarm - HighcharteventsAlarms%76.666667
227867Activity O2 Sat - Aerobic CapacityActivity O2 Sat - Aerobic CapacitycharteventsOT Notes%76.666667
223770O2 Saturation Pulseoxymetry Alarm - LowSpO2 Alarm - LowcharteventsAlarms%76.666667
226861ART %O2 saturation (PA Line)ART %O2 saturation (PA Line)charteventsPA Line Insertion%76.666667
225674Mixed Venous O2% SatMixed Venous O2% SatcharteventsLabs%76.666667
226862PA %O2 Saturation (PA Line)PA %O2 Saturation (PA Line)charteventsPA Line Insertion%76.666667
227686Central Venous O2% SatCentral Venous O2% SatcharteventsLabs%76.666667
227546SVV (Arterial)SVV (Arterial)charteventsHemodynamics%76.666667
227549ScvO2 (Presep)ScvO2 (Presep)charteventsHemodynamics%76.666667
226993ApacheIV_LOSApacheIV_LOScharteventsScores - APACHE IV (2)%76.666667
227008Ejection FractionEjection FractioncharteventsScores - APACHE IV (2)%76.666667
227010FiO2_ApacheIVFiO2_ApacheIVcharteventsScores - APACHE IV (2)%76.666667
226272EF (CCO)EF (CCO)charteventsHemodynamics%76.666667
228184SVV (PiCCO)SVV (PiCCO)charteventsPiCCO%76.666667
227861Rest O2 Sat - Aerobic CapacityRest O2 Sat - Aerobic CapacitycharteventsOT Notes%76.666667
226253SpO2 Desat LimitSpO2 Desat LimitcharteventsAlarms%76.666667
224704ATC %ATC %charteventsRespiratory%76.666667
228378TFCd (NICOM)TFCd (NICOM)charteventsNICOM%76.666667
226745APACHE II Predecited Death RateAPACHE II PDRcharteventsScores - APACHE II%76.666667
227919Rest O2 Sat - Aerobic Activity ResponseRest O2 Sat - Aerobic Activity ResponsecharteventsOT Notes%76.666667
226744APACHE II PDR - AdjustedAPACHE II PDR - AdjustedcharteventsScores - APACHE II%76.666667
228377SVI ChangeSVI ChangecharteventsNICOM%76.666667
226754FiO2ApacheIIValueFiO2ApacheIIValuecharteventsScores - APACHE II%76.666667
228375Stroke Volume Index (SVI NICOM)Stroke Volume Index (SVI NICOM)charteventsNICOM%76.666667
\n", "
" ], "text/plain": [ " label \\\n", "itemid \n", "50817 OXYGEN SATURATION \n", "220277 O2 saturation pulseoxymetry \n", "646 SpO2 \n", "228232 PAR-Oxygen saturation \n", "3785 PO2 \n", "3837 pO2 \n", "50821 PO2 \n", "6719 SpO2-L \n", "1341 PULSE \n", "1725 Pulse \n", "1332 pulse \n", "50816 OXYGEN \n", "2955 JVO2 SAT \n", "2574 MVO2 SAT \n", "227873 Recovery O2 Sat - Aerobic Capacity \n", "223772 SvO2 \n", "223769 O2 Saturation Pulseoxymetry Alarm - High \n", "227867 Activity O2 Sat - Aerobic Capacity \n", "223770 O2 Saturation Pulseoxymetry Alarm - Low \n", "226861 ART %O2 saturation (PA Line) \n", "225674 Mixed Venous O2% Sat \n", "226862 PA %O2 Saturation (PA Line) \n", "227686 Central Venous O2% Sat \n", "227546 SVV (Arterial) \n", "227549 ScvO2 (Presep) \n", "226993 ApacheIV_LOS \n", "227008 Ejection Fraction \n", "227010 FiO2_ApacheIV \n", "226272 EF (CCO) \n", "228184 SVV (PiCCO) \n", "227861 Rest O2 Sat - Aerobic Capacity \n", "226253 SpO2 Desat Limit \n", "224704 ATC % \n", "228378 TFCd (NICOM) \n", "226745 APACHE II Predecited Death Rate \n", "227919 Rest O2 Sat - Aerobic Activity Response \n", "226744 APACHE II PDR - Adjusted \n", "228377 SVI Change \n", "226754 FiO2ApacheIIValue \n", "228375 Stroke Volume Index (SVI NICOM) \n", "\n", " abbreviation linksto \\\n", "itemid \n", "50817 NaN labevents \n", "220277 SpO2 chartevents \n", "646 chartevents \n", "228232 PAR-Oxygen saturation \n", "3785 chartevents \n", "3837 chartevents \n", "50821 NaN labevents \n", "6719 chartevents \n", "1341 chartevents \n", "1725 chartevents \n", "1332 chartevents \n", "50816 NaN labevents \n", "2955 chartevents \n", "2574 chartevents \n", "227873 Recovery O2 Sat - Aerobic Capacity chartevents \n", "223772 SvO2 chartevents \n", "223769 SpO2 Alarm - High chartevents \n", "227867 Activity O2 Sat - Aerobic Capacity chartevents \n", "223770 SpO2 Alarm - Low chartevents \n", "226861 ART %O2 saturation (PA Line) chartevents \n", "225674 Mixed Venous O2% Sat chartevents \n", "226862 PA %O2 Saturation (PA Line) chartevents \n", "227686 Central Venous O2% Sat chartevents \n", "227546 SVV (Arterial) chartevents \n", "227549 ScvO2 (Presep) chartevents \n", "226993 ApacheIV_LOS chartevents \n", "227008 Ejection Fraction chartevents \n", "227010 FiO2_ApacheIV chartevents \n", "226272 EF (CCO) chartevents \n", "228184 SVV (PiCCO) chartevents \n", "227861 Rest O2 Sat - Aerobic Capacity chartevents \n", "226253 SpO2 Desat Limit chartevents \n", "224704 ATC % chartevents \n", "228378 TFCd (NICOM) chartevents \n", "226745 APACHE II PDR chartevents \n", "227919 Rest O2 Sat - Aerobic Activity Response chartevents \n", "226744 APACHE II PDR - Adjusted chartevents \n", "228377 SVI Change chartevents \n", "226754 FiO2ApacheIIValue chartevents \n", "228375 Stroke Volume Index (SVI NICOM) chartevents \n", "\n", " category unitname score \n", "itemid \n", "50817 BLOOD GAS NaN 110.000000 \n", "220277 Respiratory % 110.000000 \n", "646 110.000000 \n", "228232 Routine Vital Signs 102.666667 \n", "3785 ABG's 100.666667 \n", "3837 ABG'S 100.666667 \n", "50821 BLOOD GAS NaN 100.666667 \n", "6719 96.666667 \n", "1341 78.666667 \n", "1725 78.666667 \n", "1332 78.666667 \n", "50816 BLOOD GAS NaN 78.000000 \n", "2955 78.000000 \n", "2574 78.000000 \n", "227873 OT Notes % 76.666667 \n", "223772 Hemodynamics % 76.666667 \n", "223769 Alarms % 76.666667 \n", "227867 OT Notes % 76.666667 \n", "223770 Alarms % 76.666667 \n", "226861 PA Line Insertion % 76.666667 \n", "225674 Labs % 76.666667 \n", "226862 PA Line Insertion % 76.666667 \n", "227686 Labs % 76.666667 \n", "227546 Hemodynamics % 76.666667 \n", "227549 Hemodynamics % 76.666667 \n", "226993 Scores - APACHE IV (2) % 76.666667 \n", "227008 Scores - APACHE IV (2) % 76.666667 \n", "227010 Scores - APACHE IV (2) % 76.666667 \n", "226272 Hemodynamics % 76.666667 \n", "228184 PiCCO % 76.666667 \n", "227861 OT Notes % 76.666667 \n", "226253 Alarms % 76.666667 \n", "224704 Respiratory % 76.666667 \n", "228378 NICOM % 76.666667 \n", "226745 Scores - APACHE II % 76.666667 \n", "227919 OT Notes % 76.666667 \n", "226744 Scores - APACHE II % 76.666667 \n", "228377 NICOM % 76.666667 \n", "226754 Scores - APACHE II % 76.666667 \n", "228375 NICOM % 76.666667 " ] }, "execution_count": 80, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'oxygen saturation',\n", " 'O2sat',\n", " 'pulse oximetry',\n", " '%',\n", " 'spo2'\n", " ])\n", "out_df.head(40)" ] }, { "cell_type": "code", "execution_count": 81, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.OXYGEN_SATURATION_PULSE_OXIMETRY] = [646,220277,228232]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Body Temp" ] }, { "cell_type": "code", "execution_count": 82, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
50825TEMPERATURENaNlabeventsBLOOD GASNaN110.000000
226170Head of Bead MeasurementHOB MeasurementcharteventsTreatmentsDegree104.666667
223762Temperature CelsiusTemperature CcharteventsRoutine Vital Signs?C104.666667
223761Temperature FahrenheitTemperature FcharteventsRoutine Vital Signs?F104.666667
678Temperature Fchartevents104.666667
676Temperature Cchartevents104.666667
591RLE [Temperature]chartevents98.000000
597RUE [Temperature]chartevents98.000000
224027Skin TemperatureSkin TempSkin - Assessment97.333333
224642Temperature SiteTemp SiteRoutine Vital Signs97.333333
645Skin [Temperature]chartevents95.666667
679Temperature F (calc)chartevents92.333333
677Temperature C (calc)chartevents92.333333
227054TemperatureF_ApacheIVTemperatureF_ApacheIVcharteventsScores - APACHE IV (2)?F89.333333
224769LUE TempLUE TempCardiovascular88.000000
224771RLE TempRLE TempCardiovascular88.000000
224773LLE TempLLE TempCardiovascular88.000000
2798arm 90 degreeschartevents88.000000
224674Changes in TemperatureChanges in TemperatureToxicology88.000000
224767RUE TempRUE TempCardiovascular88.000000
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "50825 TEMPERATURE NaN labevents \n", "226170 Head of Bead Measurement HOB Measurement chartevents \n", "223762 Temperature Celsius Temperature C chartevents \n", "223761 Temperature Fahrenheit Temperature F chartevents \n", "678 Temperature F chartevents \n", "676 Temperature C chartevents \n", "591 RLE [Temperature] chartevents \n", "597 RUE [Temperature] chartevents \n", "224027 Skin Temperature Skin Temp \n", "224642 Temperature Site Temp Site \n", "645 Skin [Temperature] chartevents \n", "679 Temperature F (calc) chartevents \n", "677 Temperature C (calc) chartevents \n", "227054 TemperatureF_ApacheIV TemperatureF_ApacheIV chartevents \n", "224769 LUE Temp LUE Temp \n", "224771 RLE Temp RLE Temp \n", "224773 LLE Temp LLE Temp \n", "2798 arm 90 degrees chartevents \n", "224674 Changes in Temperature Changes in Temperature \n", "224767 RUE Temp RUE Temp \n", "\n", " category unitname score \n", "itemid \n", "50825 BLOOD GAS NaN 110.000000 \n", "226170 Treatments Degree 104.666667 \n", "223762 Routine Vital Signs ?C 104.666667 \n", "223761 Routine Vital Signs ?F 104.666667 \n", "678 104.666667 \n", "676 104.666667 \n", "591 98.000000 \n", "597 98.000000 \n", "224027 Skin - Assessment 97.333333 \n", "224642 Routine Vital Signs 97.333333 \n", "645 95.666667 \n", "679 92.333333 \n", "677 92.333333 \n", "227054 Scores - APACHE IV (2) ?F 89.333333 \n", "224769 Cardiovascular 88.000000 \n", "224771 Cardiovascular 88.000000 \n", "224773 Cardiovascular 88.000000 \n", "2798 88.000000 \n", "224674 Toxicology 88.000000 \n", "224767 Cardiovascular 88.000000 " ] }, "execution_count": 82, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'temperature',\n", " 'temp',\n", " 'celcius',\n", " 'farenheit',\n", " 'degrees',\n", " 'deg'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 83, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.TEMPERATURE_BODY] = [223761,678,223762,676]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Weight" ] }, { "cell_type": "code", "execution_count": 97, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
763Daily Weightchartevents110.000000
226512Admission Weight (Kg)Admission Weight (Kg)charteventsGeneralkg110.000000
226846Feeding WeightFeeding WeightcharteventsGeneralkg110.000000
224639Daily WeightDaily WeightcharteventsGeneralkg110.000000
3693Weight Kgchartevents96.666667
968EKGchartevents96.666667
225402EKGEKGprocedureevents_mv4-ProceduresNone96.666667
226184Estimated Protein Needs/KgEstimated Protein Needs/KgcharteventsGeneralg/kg88.000000
733Weight Changechartevents85.333333
226707HeightHeightcharteventsGeneralInch83.000000
4183Birthweight (kg)chartevents81.666667
228179ELWI (PiCCO)ELWI (PiCCO)charteventsPiCCOml/kg81.333333
580Previous Weightchartevents81.333333
43622cc/kginputevents_cv81.333333
581Previous WeightFchartevents80.000000
7000ideal body weightchartevents78.000000
45271Chucks Pad Weightinputevents_cv78.000000
3723Birth Weight (kg)chartevents77.666667
3581Present Weight (lb)chartevents76.000000
3692Weight Change (gms)chartevents76.000000
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "763 Daily Weight \n", "226512 Admission Weight (Kg) Admission Weight (Kg) \n", "226846 Feeding Weight Feeding Weight \n", "224639 Daily Weight Daily Weight \n", "3693 Weight Kg \n", "968 EKG \n", "225402 EKG EKG \n", "226184 Estimated Protein Needs/Kg Estimated Protein Needs/Kg \n", "733 Weight Change \n", "226707 Height Height \n", "4183 Birthweight (kg) \n", "228179 ELWI (PiCCO) ELWI (PiCCO) \n", "580 Previous Weight \n", "43622 cc/kg \n", "581 Previous WeightF \n", "7000 ideal body weight \n", "45271 Chucks Pad Weight \n", "3723 Birth Weight (kg) \n", "3581 Present Weight (lb) \n", "3692 Weight Change (gms) \n", "\n", " linksto category unitname score \n", "itemid \n", "763 chartevents 110.000000 \n", "226512 chartevents General kg 110.000000 \n", "226846 chartevents General kg 110.000000 \n", "224639 chartevents General kg 110.000000 \n", "3693 chartevents 96.666667 \n", "968 chartevents 96.666667 \n", "225402 procedureevents_mv 4-Procedures None 96.666667 \n", "226184 chartevents General g/kg 88.000000 \n", "733 chartevents 85.333333 \n", "226707 chartevents General Inch 83.000000 \n", "4183 chartevents 81.666667 \n", "228179 chartevents PiCCO ml/kg 81.333333 \n", "580 chartevents 81.333333 \n", "43622 inputevents_cv 81.333333 \n", "581 chartevents 80.000000 \n", "7000 chartevents 78.000000 \n", "45271 inputevents_cv 78.000000 \n", "3723 chartevents 77.666667 \n", "3581 chartevents 76.000000 \n", "3692 chartevents 76.000000 " ] }, "execution_count": 97, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'weight',\n", " 'daily weight',\n", " 'kg' \n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 98, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.WEIGHT_BODY] =[763,224639,3693]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Urine Output" ] }, { "cell_type": "code", "execution_count": 84, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
226560VoidVoidoutputeventsOutputmL110.000000
43332voidinputevents_cv110.000000
7672Foleychartevents110.000000
3686Voidchartevents110.000000
45967foleyinputevents_cv110.000000
43053URINE OUToutputevents110.000000
226559FoleyFoleyoutputeventsOutputmL110.000000
44103ER urine outoutputevents100.666667
44834er urine outoutputevents100.666667
227519Urine output_ApacheIVUrine outputcharteventsScores - APACHE IV (2)None100.666667
44706urine outputoutputevents100.666667
42892EW URINE OUToutputevents100.666667
45415ED Urine OUToutputevents100.666667
43987urine out oroutputevents100.666667
42666E.R. URINE OUToutputevents96.666667
44237E.R. urine outoutputevents96.666667
42592VICU URINE OUToutputevents95.333333
40069Urine Out Voidoutputevents95.333333
46423ed foleyinputevents_cv94.666667
43931Floor urine outoutputevents93.333333
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "226560 Void Void outputevents \n", "43332 void inputevents_cv \n", "7672 Foley chartevents \n", "3686 Void chartevents \n", "45967 foley inputevents_cv \n", "43053 URINE OUT outputevents \n", "226559 Foley Foley outputevents \n", "44103 ER urine out outputevents \n", "44834 er urine out outputevents \n", "227519 Urine output_ApacheIV Urine output chartevents \n", "44706 urine output outputevents \n", "42892 EW URINE OUT outputevents \n", "45415 ED Urine OUT outputevents \n", "43987 urine out or outputevents \n", "42666 E.R. URINE OUT outputevents \n", "44237 E.R. urine out outputevents \n", "42592 VICU URINE OUT outputevents \n", "40069 Urine Out Void outputevents \n", "46423 ed foley inputevents_cv \n", "43931 Floor urine out outputevents \n", "\n", " category unitname score \n", "itemid \n", "226560 Output mL 110.000000 \n", "43332 110.000000 \n", "7672 110.000000 \n", "3686 110.000000 \n", "45967 110.000000 \n", "43053 110.000000 \n", "226559 Output mL 110.000000 \n", "44103 100.666667 \n", "44834 100.666667 \n", "227519 Scores - APACHE IV (2) None 100.666667 \n", "44706 100.666667 \n", "42892 100.666667 \n", "45415 100.666667 \n", "43987 100.666667 \n", "42666 96.666667 \n", "44237 96.666667 \n", "42592 95.333333 \n", "40069 95.333333 \n", "46423 94.666667 \n", "43931 93.333333 " ] }, "execution_count": 84, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'urine out',\n", " 'void',\n", " 'foley'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 85, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
226560VoidVoidoutputeventsOutputmL110.000000
43332voidinputevents_cv110.000000
7672Foleychartevents110.000000
3686Voidchartevents110.000000
45967foleyinputevents_cv110.000000
43053URINE OUToutputevents110.000000
226559FoleyFoleyoutputeventsOutputmL110.000000
44103ER urine outoutputevents100.666667
44834er urine outoutputevents100.666667
227519Urine output_ApacheIVUrine outputcharteventsScores - APACHE IV (2)None100.666667
44706urine outputoutputevents100.666667
42892EW URINE OUToutputevents100.666667
45415ED Urine OUToutputevents100.666667
43987urine out oroutputevents100.666667
42666E.R. URINE OUToutputevents96.666667
44237E.R. urine outoutputevents96.666667
42592VICU URINE OUToutputevents95.333333
40069Urine Out Voidoutputevents95.333333
46423ed foleyinputevents_cv94.666667
43931Floor urine outoutputevents93.333333
43348urine output/kgoutputevents93.333333
40405Urine Out Otheroutputevents93.333333
42810angio urine outoutputevents93.333333
46180urine out-angiooutputevents93.333333
40055Urine Out Foleyoutputevents93.333333
44253Urine out angiooutputevents93.333333
44325ED URINE OUTPUToutputevents93.333333
41857urine out in eroutputevents93.333333
44824EW urine outputoutputevents93.333333
45991ew-urine outputoutputevents93.333333
42042ANGIO URINE OUToutputevents93.333333
46177URINE OUT-ANGIOoutputevents93.333333
44684floor urine outoutputevents93.333333
46658ED Urine outputoutputevents93.333333
46578URINE OUTPUT-ERoutputevents93.333333
42765FARR 6 URINE OUToutputevents91.333333
70081URINEmicrobiologyeventsSPECIMEN90.666667
3819Urine LeukocytescharteventsUrine90.666667
70079URINEmicrobiologyeventsSPECIMEN90.666667
43462urineoutputevents90.666667
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "226560 Void Void outputevents \n", "43332 void inputevents_cv \n", "7672 Foley chartevents \n", "3686 Void chartevents \n", "45967 foley inputevents_cv \n", "43053 URINE OUT outputevents \n", "226559 Foley Foley outputevents \n", "44103 ER urine out outputevents \n", "44834 er urine out outputevents \n", "227519 Urine output_ApacheIV Urine output chartevents \n", "44706 urine output outputevents \n", "42892 EW URINE OUT outputevents \n", "45415 ED Urine OUT outputevents \n", "43987 urine out or outputevents \n", "42666 E.R. URINE OUT outputevents \n", "44237 E.R. urine out outputevents \n", "42592 VICU URINE OUT outputevents \n", "40069 Urine Out Void outputevents \n", "46423 ed foley inputevents_cv \n", "43931 Floor urine out outputevents \n", "43348 urine output/kg outputevents \n", "40405 Urine Out Other outputevents \n", "42810 angio urine out outputevents \n", "46180 urine out-angio outputevents \n", "40055 Urine Out Foley outputevents \n", "44253 Urine out angio outputevents \n", "44325 ED URINE OUTPUT outputevents \n", "41857 urine out in er outputevents \n", "44824 EW urine output outputevents \n", "45991 ew-urine output outputevents \n", "42042 ANGIO URINE OUT outputevents \n", "46177 URINE OUT-ANGIO outputevents \n", "44684 floor urine out outputevents \n", "46658 ED Urine output outputevents \n", "46578 URINE OUTPUT-ER outputevents \n", "42765 FARR 6 URINE OUT outputevents \n", "70081 URINE microbiologyevents \n", "3819 Urine Leukocytes chartevents \n", "70079 URINE microbiologyevents \n", "43462 urine outputevents \n", "\n", " category unitname score \n", "itemid \n", "226560 Output mL 110.000000 \n", "43332 110.000000 \n", "7672 110.000000 \n", "3686 110.000000 \n", "45967 110.000000 \n", "43053 110.000000 \n", "226559 Output mL 110.000000 \n", "44103 100.666667 \n", "44834 100.666667 \n", "227519 Scores - APACHE IV (2) None 100.666667 \n", "44706 100.666667 \n", "42892 100.666667 \n", "45415 100.666667 \n", "43987 100.666667 \n", "42666 96.666667 \n", "44237 96.666667 \n", "42592 95.333333 \n", "40069 95.333333 \n", "46423 94.666667 \n", "43931 93.333333 \n", "43348 93.333333 \n", "40405 93.333333 \n", "42810 93.333333 \n", "46180 93.333333 \n", "40055 93.333333 \n", "44253 93.333333 \n", "44325 93.333333 \n", "41857 93.333333 \n", "44824 93.333333 \n", "45991 93.333333 \n", "42042 93.333333 \n", "46177 93.333333 \n", "44684 93.333333 \n", "46658 93.333333 \n", "46578 93.333333 \n", "42765 91.333333 \n", "70081 SPECIMEN 90.666667 \n", "3819 Urine 90.666667 \n", "70079 SPECIMEN 90.666667 \n", "43462 90.666667 " ] }, "execution_count": 85, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df.head(40)" ] }, { "cell_type": "code", "execution_count": 86, "metadata": { "collapsed": true }, "outputs": [], "source": [ "out_df = out_df[out_df.linksto.isin(['outputevents','chartevents'])]" ] }, { "cell_type": "code", "execution_count": 87, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
226560VoidVoidoutputeventsOutputmL110.000000
7672Foleychartevents110.000000
3686Voidchartevents110.000000
43053URINE OUToutputevents110.000000
226559FoleyFoleyoutputeventsOutputmL110.000000
44103ER urine outoutputevents100.666667
44834er urine outoutputevents100.666667
227519Urine output_ApacheIVUrine outputcharteventsScores - APACHE IV (2)None100.666667
44706urine outputoutputevents100.666667
42892EW URINE OUToutputevents100.666667
45415ED Urine OUToutputevents100.666667
43987urine out oroutputevents100.666667
42666E.R. URINE OUToutputevents96.666667
44237E.R. urine outoutputevents96.666667
42592VICU URINE OUToutputevents95.333333
40069Urine Out Voidoutputevents95.333333
43931Floor urine outoutputevents93.333333
43348urine output/kgoutputevents93.333333
40405Urine Out Otheroutputevents93.333333
42810angio urine outoutputevents93.333333
46180urine out-angiooutputevents93.333333
40055Urine Out Foleyoutputevents93.333333
44253Urine out angiooutputevents93.333333
44325ED URINE OUTPUToutputevents93.333333
41857urine out in eroutputevents93.333333
44824EW urine outputoutputevents93.333333
45991ew-urine outputoutputevents93.333333
42042ANGIO URINE OUToutputevents93.333333
46177URINE OUT-ANGIOoutputevents93.333333
44684floor urine outoutputevents93.333333
46658ED Urine outputoutputevents93.333333
46578URINE OUTPUT-ERoutputevents93.333333
42765FARR 6 URINE OUToutputevents91.333333
3819Urine LeukocytescharteventsUrine90.666667
43462urineoutputevents90.666667
3816Urine GlucosecharteventsUrine90.666667
3817Urine HemecharteventsUrine90.666667
3818Urine KetonescharteventsUrine90.666667
3822Urine ProteincharteventsUrine90.666667
6298foley d/cchartevents90.666667
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "226560 Void Void outputevents \n", "7672 Foley chartevents \n", "3686 Void chartevents \n", "43053 URINE OUT outputevents \n", "226559 Foley Foley outputevents \n", "44103 ER urine out outputevents \n", "44834 er urine out outputevents \n", "227519 Urine output_ApacheIV Urine output chartevents \n", "44706 urine output outputevents \n", "42892 EW URINE OUT outputevents \n", "45415 ED Urine OUT outputevents \n", "43987 urine out or outputevents \n", "42666 E.R. URINE OUT outputevents \n", "44237 E.R. urine out outputevents \n", "42592 VICU URINE OUT outputevents \n", "40069 Urine Out Void outputevents \n", "43931 Floor urine out outputevents \n", "43348 urine output/kg outputevents \n", "40405 Urine Out Other outputevents \n", "42810 angio urine out outputevents \n", "46180 urine out-angio outputevents \n", "40055 Urine Out Foley outputevents \n", "44253 Urine out angio outputevents \n", "44325 ED URINE OUTPUT outputevents \n", "41857 urine out in er outputevents \n", "44824 EW urine output outputevents \n", "45991 ew-urine output outputevents \n", "42042 ANGIO URINE OUT outputevents \n", "46177 URINE OUT-ANGIO outputevents \n", "44684 floor urine out outputevents \n", "46658 ED Urine output outputevents \n", "46578 URINE OUTPUT-ER outputevents \n", "42765 FARR 6 URINE OUT outputevents \n", "3819 Urine Leukocytes chartevents \n", "43462 urine outputevents \n", "3816 Urine Glucose chartevents \n", "3817 Urine Heme chartevents \n", "3818 Urine Ketones chartevents \n", "3822 Urine Protein chartevents \n", "6298 foley d/c chartevents \n", "\n", " category unitname score \n", "itemid \n", "226560 Output mL 110.000000 \n", "7672 110.000000 \n", "3686 110.000000 \n", "43053 110.000000 \n", "226559 Output mL 110.000000 \n", "44103 100.666667 \n", "44834 100.666667 \n", "227519 Scores - APACHE IV (2) None 100.666667 \n", "44706 100.666667 \n", "42892 100.666667 \n", "45415 100.666667 \n", "43987 100.666667 \n", "42666 96.666667 \n", "44237 96.666667 \n", "42592 95.333333 \n", "40069 95.333333 \n", "43931 93.333333 \n", "43348 93.333333 \n", "40405 93.333333 \n", "42810 93.333333 \n", "46180 93.333333 \n", "40055 93.333333 \n", "44253 93.333333 \n", "44325 93.333333 \n", "41857 93.333333 \n", "44824 93.333333 \n", "45991 93.333333 \n", "42042 93.333333 \n", "46177 93.333333 \n", "44684 93.333333 \n", "46658 93.333333 \n", "46578 93.333333 \n", "42765 91.333333 \n", "3819 Urine 90.666667 \n", "43462 90.666667 \n", "3816 Urine 90.666667 \n", "3817 Urine 90.666667 \n", "3818 Urine 90.666667 \n", "3822 Urine 90.666667 \n", "6298 90.666667 " ] }, "execution_count": 87, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df.head(40)" ] }, { "cell_type": "code", "execution_count": 91, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "out_df.drop(227519,axis=0,inplace=True)" ] }, { "cell_type": "code", "execution_count": 96, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.OUTPUT_URINE] = out_df.loc[:46578].index.unique().tolist()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "https://github.com/MIT-LCP/mimic-code/blob/travis/concepts/cookbook/uo.sql" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.OUTPUT_URINE] = keep_dict[data_dict.labels.OUTPUT_URINE]\n", "to_add = [43175,40094,40715,40473,40085,40057,40056,40428,40086,40096,40651]\n", "to_add += [227510,226561,226584,226563,226564,226565,226567,226557,226558 ]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Glasgow Coma Scale (GCS)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Eye Opening" ] }, { "cell_type": "code", "execution_count": 99, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
184Eye Openingchartevents110.000000
220739GCS - Eye OpeningEye OpeningNeurological110.000000
198GCS Totalchartevents76.666667
70033EYEmicrobiologyeventsSPECIMEN72.000000
3742BasoscharteventsCSF71.333333
3791PolyscharteventsCSF71.333333
3779MonoscharteventsCSF71.333333
2834ICSchartevents67.000000
227011GCSEye_ApacheIVGCSEye_ApacheIVScores - APACHE IV (2)65.333333
226755GcsApacheIIScoreGcsApacheIIScorecharteventsScores - APACHE IINone64.666667
41556nginputevents_cv64.000000
227012GCSMotor_ApacheIVGCSMotor_ApacheIVScores - APACHE IV (2)63.333333
227013GcsScore_ApacheIVGcsScore_ApacheIVcharteventsScores - APACHE IV (2)None63.333333
5700Gent eye ointchartevents63.000000
227014GCSVerbal_ApacheIVGCSVerbal_ApacheIVScores - APACHE IV (2)62.666667
50931GLUCOSENaNlabeventsCHEMISTRYNaN62.333333
1529GlucosecharteventsChemistry62.333333
50809GLUCOSENaNlabeventsBLOOD GASNaN62.333333
51478GLUCOSENaNlabeventsHEMATOLOGYNaN62.333333
223901GCS - Motor ResponseMotor ResponseNeurological61.666667
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "184 Eye Opening chartevents \n", "220739 GCS - Eye Opening Eye Opening \n", "198 GCS Total chartevents \n", "70033 EYE microbiologyevents \n", "3742 Basos chartevents \n", "3791 Polys chartevents \n", "3779 Monos chartevents \n", "2834 ICS chartevents \n", "227011 GCSEye_ApacheIV GCSEye_ApacheIV \n", "226755 GcsApacheIIScore GcsApacheIIScore chartevents \n", "41556 ng inputevents_cv \n", "227012 GCSMotor_ApacheIV GCSMotor_ApacheIV \n", "227013 GcsScore_ApacheIV GcsScore_ApacheIV chartevents \n", "5700 Gent eye oint chartevents \n", "227014 GCSVerbal_ApacheIV GCSVerbal_ApacheIV \n", "50931 GLUCOSE NaN labevents \n", "1529 Glucose chartevents \n", "50809 GLUCOSE NaN labevents \n", "51478 GLUCOSE NaN labevents \n", "223901 GCS - Motor Response Motor Response \n", "\n", " category unitname score \n", "itemid \n", "184 110.000000 \n", "220739 Neurological 110.000000 \n", "198 76.666667 \n", "70033 SPECIMEN 72.000000 \n", "3742 CSF 71.333333 \n", "3791 CSF 71.333333 \n", "3779 CSF 71.333333 \n", "2834 67.000000 \n", "227011 Scores - APACHE IV (2) 65.333333 \n", "226755 Scores - APACHE II None 64.666667 \n", "41556 64.000000 \n", "227012 Scores - APACHE IV (2) 63.333333 \n", "227013 Scores - APACHE IV (2) None 63.333333 \n", "5700 63.000000 \n", "227014 Scores - APACHE IV (2) 62.666667 \n", "50931 CHEMISTRY NaN 62.333333 \n", "1529 Chemistry 62.333333 \n", "50809 BLOOD GAS NaN 62.333333 \n", "51478 HEMATOLOGY NaN 62.333333 \n", "223901 Neurological 61.666667 " ] }, "execution_count": 99, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'glasgow come scale',\n", " 'GCS',\n", " 'eye opening'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 100, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.GLASGOW_COMA_SCALE_EYE_OPENING] = [184,220739]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Motor" ] }, { "cell_type": "code", "execution_count": 101, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
228405Motor L LegMotor L LegNeurological85.333333
228404Motor L ArmMotor L ArmNeurological85.333333
228407Motor R LegMotor R LegNeurological85.333333
228406Motor R ArmMotor R ArmNeurological85.333333
227120Motor DeficitMotor DeficitPain/Sedation80.666667
453Motor Deficitschartevents78.666667
454Motor Responsechartevents78.666667
223901GCS - Motor ResponseMotor ResponseNeurological78.666667
198GCS Totalchartevents76.666667
227012GCSMotor_ApacheIVGCSMotor_ApacheIVScores - APACHE IV (2)73.333333
3779MonoscharteventsCSF71.333333
3742BasoscharteventsCSF71.333333
3791PolyscharteventsCSF71.333333
226757GCSMotorApacheIIValueGCSMotorApacheIIValueScores - APACHE II68.666667
2834ICSchartevents67.000000
225472PneumothoraxPneumothoraxprocedureevents_mv3-Significant EventsNone66.000000
227011GCSEye_ApacheIVGCSEye_ApacheIVScores - APACHE IV (2)65.333333
226755GcsApacheIIScoreGcsApacheIIScorecharteventsScores - APACHE IINone64.666667
220739GCS - Eye OpeningEye OpeningNeurological64.333333
227013GcsScore_ApacheIVGcsScore_ApacheIVcharteventsScores - APACHE IV (2)None63.333333
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "228405 Motor L Leg Motor L Leg \n", "228404 Motor L Arm Motor L Arm \n", "228407 Motor R Leg Motor R Leg \n", "228406 Motor R Arm Motor R Arm \n", "227120 Motor Deficit Motor Deficit \n", "453 Motor Deficits chartevents \n", "454 Motor Response chartevents \n", "223901 GCS - Motor Response Motor Response \n", "198 GCS Total chartevents \n", "227012 GCSMotor_ApacheIV GCSMotor_ApacheIV \n", "3779 Monos chartevents \n", "3742 Basos chartevents \n", "3791 Polys chartevents \n", "226757 GCSMotorApacheIIValue GCSMotorApacheIIValue \n", "2834 ICS chartevents \n", "225472 Pneumothorax Pneumothorax procedureevents_mv \n", "227011 GCSEye_ApacheIV GCSEye_ApacheIV \n", "226755 GcsApacheIIScore GcsApacheIIScore chartevents \n", "220739 GCS - Eye Opening Eye Opening \n", "227013 GcsScore_ApacheIV GcsScore_ApacheIV chartevents \n", "\n", " category unitname score \n", "itemid \n", "228405 Neurological 85.333333 \n", "228404 Neurological 85.333333 \n", "228407 Neurological 85.333333 \n", "228406 Neurological 85.333333 \n", "227120 Pain/Sedation 80.666667 \n", "453 78.666667 \n", "454 78.666667 \n", "223901 Neurological 78.666667 \n", "198 76.666667 \n", "227012 Scores - APACHE IV (2) 73.333333 \n", "3779 CSF 71.333333 \n", "3742 CSF 71.333333 \n", "3791 CSF 71.333333 \n", "226757 Scores - APACHE II 68.666667 \n", "2834 67.000000 \n", "225472 3-Significant Events None 66.000000 \n", "227011 Scores - APACHE IV (2) 65.333333 \n", "226755 Scores - APACHE II None 64.666667 \n", "220739 Neurological 64.333333 \n", "227013 Scores - APACHE IV (2) None 63.333333 " ] }, "execution_count": 101, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'glasgow come scale',\n", " 'GCS',\n", " 'motor',\n", " 'motor response'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 102, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.GLASGOW_COMA_SCALE_MOTOR] = [454,223901]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Verbal" ] }, { "cell_type": "code", "execution_count": 103, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
223900GCS - Verbal ResponseVerbal ResponseNeurological110.000000
723Verbal Responsechartevents110.000000
224756ResponseResponseNeurological90.000000
41610ERinputevents_cvFree Form Intake76.666667
227014GCSVerbal_ApacheIVGCSVerbal_ApacheIVScores - APACHE IV (2)76.666667
44473erinputevents_cvFree Form Intake76.666667
198GCS Totalchartevents76.666667
226758GCSVerbalApacheIIValueGCSVerbalApacheIIValueScores - APACHE II72.000000
3779MonoscharteventsCSF71.333333
3742BasoscharteventsCSF71.333333
3791PolyscharteventsCSF71.333333
224413TOF ResponseTOF ResponsePain/Sedation69.666667
40450EBLinputevents_cv67.000000
41693Verapamilinputevents_cvFree Form Intake67.000000
1968Verapamilchartevents67.000000
42047eblinputevents_cv67.000000
222318VerapamilVerapamilinputevents_mvMedicationsmg67.000000
46484verapamilinputevents_cvFree Form Intake67.000000
2834ICSchartevents67.000000
224409Pain Level ResponsePain Level ResponsePain/Sedation66.000000
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "223900 GCS - Verbal Response Verbal Response \n", "723 Verbal Response chartevents \n", "224756 Response Response \n", "41610 ER inputevents_cv \n", "227014 GCSVerbal_ApacheIV GCSVerbal_ApacheIV \n", "44473 er inputevents_cv \n", "198 GCS Total chartevents \n", "226758 GCSVerbalApacheIIValue GCSVerbalApacheIIValue \n", "3779 Monos chartevents \n", "3742 Basos chartevents \n", "3791 Polys chartevents \n", "224413 TOF Response TOF Response \n", "40450 EBL inputevents_cv \n", "41693 Verapamil inputevents_cv \n", "1968 Verapamil chartevents \n", "42047 ebl inputevents_cv \n", "222318 Verapamil Verapamil inputevents_mv \n", "46484 verapamil inputevents_cv \n", "2834 ICS chartevents \n", "224409 Pain Level Response Pain Level Response \n", "\n", " category unitname score \n", "itemid \n", "223900 Neurological 110.000000 \n", "723 110.000000 \n", "224756 Neurological 90.000000 \n", "41610 Free Form Intake 76.666667 \n", "227014 Scores - APACHE IV (2) 76.666667 \n", "44473 Free Form Intake 76.666667 \n", "198 76.666667 \n", "226758 Scores - APACHE II 72.000000 \n", "3779 CSF 71.333333 \n", "3742 CSF 71.333333 \n", "3791 CSF 71.333333 \n", "224413 Pain/Sedation 69.666667 \n", "40450 67.000000 \n", "41693 Free Form Intake 67.000000 \n", "1968 67.000000 \n", "42047 67.000000 \n", "222318 Medications mg 67.000000 \n", "46484 Free Form Intake 67.000000 \n", "2834 67.000000 \n", "224409 Pain/Sedation 66.000000 " ] }, "execution_count": 103, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'glasgow come scale',\n", " 'GCS',\n", " 'verbal',\n", " 'verbal response'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 104, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.GLASGOW_COMA_SCALE_VERBAL] = [723,223900]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Labs\n", "\n", "### Lactate" ] }, { "cell_type": "code", "execution_count": 105, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
1531Lactic AcidcharteventsChemistry110.000000
225668Lactic AcidLactic AcidcharteventsLabsNone110.000000
50813LACTATENaNlabeventsBLOOD GASNaN110.000000
220228HemoglobinHemoglobincharteventsLabsg/dl102.666667
2773sjlactatechartevents102.000000
225835Na PhosNa Phosinputevents_mvMedicationsmmol96.666667
225834K PhosK Phosinputevents_mvMedicationsmmol96.666667
227526CitrateCitrateinputevents_mvMedicationsmmol96.666667
225925Potassium PhosphatePotassium Phosphateinputevents_mvNutrition - Supplementsmmol96.666667
818Lactic Acid(0.5-2.0)charteventsChemistry91.333333
220955Ringers LactateRingers Lactateinputevents_mvFluids - Other (Not In Use)mL86.000000
2638CEREBRAL LACTATEchartevents84.000000
1634lactated ringerschartevents84.000000
30021Lactated Ringersinputevents_cv84.000000
1520ACTcharteventsCoags83.333333
220507Activated Clotting TimeACTcharteventsLabsNone83.333333
1671actchartevents83.333333
221319Alteplase (TPA)Alteplase (TPA)inputevents_mvMedicationsmg81.333333
221347AmiodaroneAmiodaroneinputevents_mvMedicationsmg81.333333
221342AminophyllineAminophyllineinputevents_mvMedicationsmg81.333333
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "1531 Lactic Acid chartevents \n", "225668 Lactic Acid Lactic Acid chartevents \n", "50813 LACTATE NaN labevents \n", "220228 Hemoglobin Hemoglobin chartevents \n", "2773 sjlactate chartevents \n", "225835 Na Phos Na Phos inputevents_mv \n", "225834 K Phos K Phos inputevents_mv \n", "227526 Citrate Citrate inputevents_mv \n", "225925 Potassium Phosphate Potassium Phosphate inputevents_mv \n", "818 Lactic Acid(0.5-2.0) chartevents \n", "220955 Ringers Lactate Ringers Lactate inputevents_mv \n", "2638 CEREBRAL LACTATE chartevents \n", "1634 lactated ringers chartevents \n", "30021 Lactated Ringers inputevents_cv \n", "1520 ACT chartevents \n", "220507 Activated Clotting Time ACT chartevents \n", "1671 act chartevents \n", "221319 Alteplase (TPA) Alteplase (TPA) inputevents_mv \n", "221347 Amiodarone Amiodarone inputevents_mv \n", "221342 Aminophylline Aminophylline inputevents_mv \n", "\n", " category unitname score \n", "itemid \n", "1531 Chemistry 110.000000 \n", "225668 Labs None 110.000000 \n", "50813 BLOOD GAS NaN 110.000000 \n", "220228 Labs g/dl 102.666667 \n", "2773 102.000000 \n", "225835 Medications mmol 96.666667 \n", "225834 Medications mmol 96.666667 \n", "227526 Medications mmol 96.666667 \n", "225925 Nutrition - Supplements mmol 96.666667 \n", "818 Chemistry 91.333333 \n", "220955 Fluids - Other (Not In Use) mL 86.000000 \n", "2638 84.000000 \n", "1634 84.000000 \n", "30021 84.000000 \n", "1520 Coags 83.333333 \n", "220507 Labs None 83.333333 \n", "1671 83.333333 \n", "221319 Medications mg 81.333333 \n", "221347 Medications mg 81.333333 \n", "221342 Medications mg 81.333333 " ] }, "execution_count": 105, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'lactate',\n", " 'lactic acid',\n", " 'mmol/L',\n", " 'mg/dL'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 106, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.LACTATE] = [1531,50813,225668,818]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Hemoglobin" ] }, { "cell_type": "code", "execution_count": 107, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
51222HEMOGLOBINNaNlabeventsHEMATOLOGYNaN110.000000
220228HemoglobinHemoglobincharteventsLabsg/dl110.000000
814HemoglobincharteventsHematology110.000000
1165Hgbchartevents110.000000
50811HEMOGLOBINNaNlabeventsBLOOD GASNaN110.000000
51225HEMOGLOBIN FNaNlabeventsHEMATOLOGYNaN104.000000
51224HEMOGLOBIN CNaNlabeventsHEMATOLOGYNaN104.000000
50814METHEMOGLOBINNaNlabeventsBLOOD GASNaN101.333333
51223HEMOGLOBIN A2NaNlabeventsHEMATOLOGYNaN101.333333
7965methhemoglobinchartevents98.666667
50852% HEMOGLOBIN A1CNaNlabeventsCHEMISTRYNaN96.666667
51212FETAL HEMOGLOBINNaNlabeventsHEMATOLOGYNaN94.666667
50805CARBOXYHEMOGLOBINNaNlabeventsBLOOD GASNaN92.666667
50855ABSOLUTE HEMOGLOBINNaNlabeventsCHEMISTRYNaN89.333333
51226HEMOGLOBLIN ANaNlabeventsHEMATOLOGYNaN88.000000
51227HEMOGLOBLIN SNaNlabeventsHEMATOLOGYNaN88.000000
42232THYMOGLOBLINinputevents_cvFree Form Intake81.333333
45486Hemoinputevents_cv81.333333
42117THYMOGLOBULINinputevents_cvFree Form Intake78.666667
50934HNaNlabeventsCHEMISTRYNaN76.666667
\n", "
" ], "text/plain": [ " label abbreviation linksto category \\\n", "itemid \n", "51222 HEMOGLOBIN NaN labevents HEMATOLOGY \n", "220228 Hemoglobin Hemoglobin chartevents Labs \n", "814 Hemoglobin chartevents Hematology \n", "1165 Hgb chartevents \n", "50811 HEMOGLOBIN NaN labevents BLOOD GAS \n", "51225 HEMOGLOBIN F NaN labevents HEMATOLOGY \n", "51224 HEMOGLOBIN C NaN labevents HEMATOLOGY \n", "50814 METHEMOGLOBIN NaN labevents BLOOD GAS \n", "51223 HEMOGLOBIN A2 NaN labevents HEMATOLOGY \n", "7965 methhemoglobin chartevents \n", "50852 % HEMOGLOBIN A1C NaN labevents CHEMISTRY \n", "51212 FETAL HEMOGLOBIN NaN labevents HEMATOLOGY \n", "50805 CARBOXYHEMOGLOBIN NaN labevents BLOOD GAS \n", "50855 ABSOLUTE HEMOGLOBIN NaN labevents CHEMISTRY \n", "51226 HEMOGLOBLIN A NaN labevents HEMATOLOGY \n", "51227 HEMOGLOBLIN S NaN labevents HEMATOLOGY \n", "42232 THYMOGLOBLIN inputevents_cv Free Form Intake \n", "45486 Hemo inputevents_cv \n", "42117 THYMOGLOBULIN inputevents_cv Free Form Intake \n", "50934 H NaN labevents CHEMISTRY \n", "\n", " unitname score \n", "itemid \n", "51222 NaN 110.000000 \n", "220228 g/dl 110.000000 \n", "814 110.000000 \n", "1165 110.000000 \n", "50811 NaN 110.000000 \n", "51225 NaN 104.000000 \n", "51224 NaN 104.000000 \n", "50814 NaN 101.333333 \n", "51223 NaN 101.333333 \n", "7965 98.666667 \n", "50852 NaN 96.666667 \n", "51212 NaN 94.666667 \n", "50805 NaN 92.666667 \n", "50855 NaN 89.333333 \n", "51226 NaN 88.000000 \n", "51227 NaN 88.000000 \n", "42232 81.333333 \n", "45486 81.333333 \n", "42117 78.666667 \n", "50934 NaN 76.666667 " ] }, "execution_count": 107, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'hgb',\n", " 'hemoglobin',\n", " 'g/dL'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 108, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.HEMOGLOBIN] = [51222,220228,814,1165,50811]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Fluids\n", "\n", "### Normal Saline" ] }, { "cell_type": "code", "execution_count": 109, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
41913NSinputevents_cvFree Form Intake110.000000
6190Normal salinechartevents110.000000
301433% Normal Salineinputevents_cv104.333333
30168Normal Saline_GUinputevents_cv103.333333
30160D5 Normal Salineinputevents_cv103.333333
30161.3% normal Salineinputevents_cv103.333333
30018.9% Normal Salineinputevents_cv103.333333
30020.45% Normal Salineinputevents_cv101.333333
30176.25% Normal Salineinputevents_cv101.333333
303520.9% Normal Salineinputevents_cv100.333333
220962Saline 3%Saline 3%inputevents_mvFluids - Other (Not In Use)mL98.666667
303530.45% Normal Salineinputevents_cv98.333333
44440Normal Saline Bolusinputevents_cvFree Form Intake97.333333
44053normal saline bolusinputevents_cvFree Form Intake97.333333
4647normal saline boluschartevents97.333333
43354normal saline flushsinputevents_cvFree Form Intake96.000000
221213Saline 30%Saline 30%inputevents_mvFluids - Other (Not In Use)mL95.000000
220959Saline 0,3%Saline 0,3%inputevents_mvFluids - Other (Not In Use)mL92.000000
220954Saline 0,9%Saline 0,9%inputevents_mvFluids - Other (Not In Use)mL92.000000
220960Saline 0,45%Saline 0,45%inputevents_mvFluids - Other (Not In Use)mL89.333333
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "41913 NS inputevents_cv \n", "6190 Normal saline chartevents \n", "30143 3% Normal Saline inputevents_cv \n", "30168 Normal Saline_GU inputevents_cv \n", "30160 D5 Normal Saline inputevents_cv \n", "30161 .3% normal Saline inputevents_cv \n", "30018 .9% Normal Saline inputevents_cv \n", "30020 .45% Normal Saline inputevents_cv \n", "30176 .25% Normal Saline inputevents_cv \n", "30352 0.9% Normal Saline inputevents_cv \n", "220962 Saline 3% Saline 3% inputevents_mv \n", "30353 0.45% Normal Saline inputevents_cv \n", "44440 Normal Saline Bolus inputevents_cv \n", "44053 normal saline bolus inputevents_cv \n", "4647 normal saline bolus chartevents \n", "43354 normal saline flushs inputevents_cv \n", "221213 Saline 30% Saline 30% inputevents_mv \n", "220959 Saline 0,3% Saline 0,3% inputevents_mv \n", "220954 Saline 0,9% Saline 0,9% inputevents_mv \n", "220960 Saline 0,45% Saline 0,45% inputevents_mv \n", "\n", " category unitname score \n", "itemid \n", "41913 Free Form Intake 110.000000 \n", "6190 110.000000 \n", "30143 104.333333 \n", "30168 103.333333 \n", "30160 103.333333 \n", "30161 103.333333 \n", "30018 103.333333 \n", "30020 101.333333 \n", "30176 101.333333 \n", "30352 100.333333 \n", "220962 Fluids - Other (Not In Use) mL 98.666667 \n", "30353 98.333333 \n", "44440 Free Form Intake 97.333333 \n", "44053 Free Form Intake 97.333333 \n", "4647 97.333333 \n", "43354 Free Form Intake 96.000000 \n", "221213 Fluids - Other (Not In Use) mL 95.000000 \n", "220959 Fluids - Other (Not In Use) mL 92.000000 \n", "220954 Fluids - Other (Not In Use) mL 92.000000 \n", "220960 Fluids - Other (Not In Use) mL 89.333333 " ] }, "execution_count": 109, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'saline',\n", " 'NS',\n", " '0.9%',\n", " 'normal saline'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 110, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.NORMAL_SALINE] = [41913,6190,20018,30252,44440,44053,4647,220954]" ] }, { "cell_type": "code", "execution_count": 111, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
41913NSinputevents_cvFree Form Intake110.000000
6190Normal salinechartevents110.000000
301433% Normal Salineinputevents_cv104.333333
30168Normal Saline_GUinputevents_cv103.333333
30160D5 Normal Salineinputevents_cv103.333333
30161.3% normal Salineinputevents_cv103.333333
30018.9% Normal Salineinputevents_cv103.333333
30020.45% Normal Salineinputevents_cv101.333333
30176.25% Normal Salineinputevents_cv101.333333
303520.9% Normal Salineinputevents_cv100.333333
220962Saline 3%Saline 3%inputevents_mvFluids - Other (Not In Use)mL98.666667
303530.45% Normal Salineinputevents_cv98.333333
44440Normal Saline Bolusinputevents_cvFree Form Intake97.333333
44053normal saline bolusinputevents_cvFree Form Intake97.333333
4647normal saline boluschartevents97.333333
43354normal saline flushsinputevents_cvFree Form Intake96.000000
221213Saline 30%Saline 30%inputevents_mvFluids - Other (Not In Use)mL95.000000
220959Saline 0,3%Saline 0,3%inputevents_mvFluids - Other (Not In Use)mL92.000000
220954Saline 0,9%Saline 0,9%inputevents_mvFluids - Other (Not In Use)mL92.000000
220960Saline 0,45%Saline 0,45%inputevents_mvFluids - Other (Not In Use)mL89.333333
220961Saline 0,65%Saline 0,65%inputevents_mvFluids - Other (Not In Use)mL89.333333
221212Saline 0,18%Saline 0,18%inputevents_mvFluids - Other (Not In Use)mL89.333333
41392ns binputevents_cvFree Form Intake88.000000
225825D5NSD5NSinputevents_mvFluids/IntakemL88.000000
5333saline flushchartevents88.000000
30060D5NSinputevents_cv88.000000
226401GU Irrigant - Normal SalineGU Irrigant - Normal Salineinputevents_mvFluids/IntakemL87.666667
220958Saline 0,255%Saline 0,255%inputevents_mvFluids - Other (Not In Use)mL86.666667
26193% NSchartevents84.666667
227344IV/Saline lockIV/Saline lockRestraint/Support Systems83.333333
30190NS .9%inputevents_cv82.333333
225158NaCl 0.9%NaCl 0.9%inputevents_mvFluids/IntakemL82.333333
45298ED NSinputevents_cvFree Form Intake81.333333
2072Pinspchartevents81.333333
2107pinspchartevents81.333333
7092Tinspchartevents81.333333
6384PINSPchartevents81.333333
44498er nsinputevents_cvFree Form Intake81.333333
3204Pinsp.chartevents79.000000
2404.45%nschartevents79.000000
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "41913 NS \n", "6190 Normal saline \n", "30143 3% Normal Saline \n", "30168 Normal Saline_GU \n", "30160 D5 Normal Saline \n", "30161 .3% normal Saline \n", "30018 .9% Normal Saline \n", "30020 .45% Normal Saline \n", "30176 .25% Normal Saline \n", "30352 0.9% Normal Saline \n", "220962 Saline 3% Saline 3% \n", "30353 0.45% Normal Saline \n", "44440 Normal Saline Bolus \n", "44053 normal saline bolus \n", "4647 normal saline bolus \n", "43354 normal saline flushs \n", "221213 Saline 30% Saline 30% \n", "220959 Saline 0,3% Saline 0,3% \n", "220954 Saline 0,9% Saline 0,9% \n", "220960 Saline 0,45% Saline 0,45% \n", "220961 Saline 0,65% Saline 0,65% \n", "221212 Saline 0,18% Saline 0,18% \n", "41392 ns b \n", "225825 D5NS D5NS \n", "5333 saline flush \n", "30060 D5NS \n", "226401 GU Irrigant - Normal Saline GU Irrigant - Normal Saline \n", "220958 Saline 0,255% Saline 0,255% \n", "2619 3% NS \n", "227344 IV/Saline lock IV/Saline lock \n", "30190 NS .9% \n", "225158 NaCl 0.9% NaCl 0.9% \n", "45298 ED NS \n", "2072 Pinsp \n", "2107 pinsp \n", "7092 Tinsp \n", "6384 PINSP \n", "44498 er ns \n", "3204 Pinsp. \n", "2404 .45%ns \n", "\n", " linksto category unitname score \n", "itemid \n", "41913 inputevents_cv Free Form Intake 110.000000 \n", "6190 chartevents 110.000000 \n", "30143 inputevents_cv 104.333333 \n", "30168 inputevents_cv 103.333333 \n", "30160 inputevents_cv 103.333333 \n", "30161 inputevents_cv 103.333333 \n", "30018 inputevents_cv 103.333333 \n", "30020 inputevents_cv 101.333333 \n", "30176 inputevents_cv 101.333333 \n", "30352 inputevents_cv 100.333333 \n", "220962 inputevents_mv Fluids - Other (Not In Use) mL 98.666667 \n", "30353 inputevents_cv 98.333333 \n", "44440 inputevents_cv Free Form Intake 97.333333 \n", "44053 inputevents_cv Free Form Intake 97.333333 \n", "4647 chartevents 97.333333 \n", "43354 inputevents_cv Free Form Intake 96.000000 \n", "221213 inputevents_mv Fluids - Other (Not In Use) mL 95.000000 \n", "220959 inputevents_mv Fluids - Other (Not In Use) mL 92.000000 \n", "220954 inputevents_mv Fluids - Other (Not In Use) mL 92.000000 \n", "220960 inputevents_mv Fluids - Other (Not In Use) mL 89.333333 \n", "220961 inputevents_mv Fluids - Other (Not In Use) mL 89.333333 \n", "221212 inputevents_mv Fluids - Other (Not In Use) mL 89.333333 \n", "41392 inputevents_cv Free Form Intake 88.000000 \n", "225825 inputevents_mv Fluids/Intake mL 88.000000 \n", "5333 chartevents 88.000000 \n", "30060 inputevents_cv 88.000000 \n", "226401 inputevents_mv Fluids/Intake mL 87.666667 \n", "220958 inputevents_mv Fluids - Other (Not In Use) mL 86.666667 \n", "2619 chartevents 84.666667 \n", "227344 Restraint/Support Systems 83.333333 \n", "30190 inputevents_cv 82.333333 \n", "225158 inputevents_mv Fluids/Intake mL 82.333333 \n", "45298 inputevents_cv Free Form Intake 81.333333 \n", "2072 chartevents 81.333333 \n", "2107 chartevents 81.333333 \n", "7092 chartevents 81.333333 \n", "6384 chartevents 81.333333 \n", "44498 inputevents_cv Free Form Intake 81.333333 \n", "3204 chartevents 79.000000 \n", "2404 chartevents 79.000000 " ] }, "execution_count": 111, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df.head(40)" ] }, { "cell_type": "code", "execution_count": 112, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.NORMAL_SALINE] += [30190,225158]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Lactated Ringers" ] }, { "cell_type": "code", "execution_count": 113, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
225828LRLRinputevents_mvFluids/IntakemL110.000000
44367LRinputevents_cvFree Form Intake110.000000
2971LRchartevents110.000000
1634lactated ringerschartevents110.000000
30021Lactated Ringersinputevents_cv110.000000
45532IR Lactated ringersinputevents_cvFree Form Intake104.000000
225943SolutionSolutioninputevents_mvFluids/IntakemL88.000000
225827D5LRD5LRinputevents_mvFluids/IntakemL88.000000
50813LACTATENaNlabeventsBLOOD GASNaN84.000000
220953RingersRingersinputevents_mvFluids - Other (Not In Use)mL84.000000
46207OR LRinputevents_cvFree Form Intake81.333333
44184LR Bolusinputevents_cvFree Form Intake70.000000
44521LR bolusinputevents_cvFree Form Intake70.000000
46781lr bolusinputevents_cvFree Form Intake70.000000
44815LR BOLUSinputevents_cvFree Form Intake70.000000
44915D5LR 40Kinputevents_cvFree Form Intake70.000000
46538PD solution ininputevents_cvFree Form Intake68.333333
44837ED URINEoutputevents68.000000
8313Nystatin solutionchartevents67.666667
30125Milrinoneinputevents_cv67.333333
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "225828 LR LR inputevents_mv \n", "44367 LR inputevents_cv \n", "2971 LR chartevents \n", "1634 lactated ringers chartevents \n", "30021 Lactated Ringers inputevents_cv \n", "45532 IR Lactated ringers inputevents_cv \n", "225943 Solution Solution inputevents_mv \n", "225827 D5LR D5LR inputevents_mv \n", "50813 LACTATE NaN labevents \n", "220953 Ringers Ringers inputevents_mv \n", "46207 OR LR inputevents_cv \n", "44184 LR Bolus inputevents_cv \n", "44521 LR bolus inputevents_cv \n", "46781 lr bolus inputevents_cv \n", "44815 LR BOLUS inputevents_cv \n", "44915 D5LR 40K inputevents_cv \n", "46538 PD solution in inputevents_cv \n", "44837 ED URINE outputevents \n", "8313 Nystatin solution chartevents \n", "30125 Milrinone inputevents_cv \n", "\n", " category unitname score \n", "itemid \n", "225828 Fluids/Intake mL 110.000000 \n", "44367 Free Form Intake 110.000000 \n", "2971 110.000000 \n", "1634 110.000000 \n", "30021 110.000000 \n", "45532 Free Form Intake 104.000000 \n", "225943 Fluids/Intake mL 88.000000 \n", "225827 Fluids/Intake mL 88.000000 \n", "50813 BLOOD GAS NaN 84.000000 \n", "220953 Fluids - Other (Not In Use) mL 84.000000 \n", "46207 Free Form Intake 81.333333 \n", "44184 Free Form Intake 70.000000 \n", "44521 Free Form Intake 70.000000 \n", "46781 Free Form Intake 70.000000 \n", "44815 Free Form Intake 70.000000 \n", "44915 Free Form Intake 70.000000 \n", "46538 Free Form Intake 68.333333 \n", "44837 68.000000 \n", "8313 67.666667 \n", "30125 67.333333 " ] }, "execution_count": 113, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'LR',\n", " 'ringers solution',\n", " 'lactated ringers'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 114, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
225828LRLRinputevents_mvFluids/IntakemL110.000000
44367LRinputevents_cvFree Form Intake110.000000
2971LRchartevents110.000000
1634lactated ringerschartevents110.000000
30021Lactated Ringersinputevents_cv110.000000
45532IR Lactated ringersinputevents_cvFree Form Intake104.000000
225943SolutionSolutioninputevents_mvFluids/IntakemL88.000000
225827D5LRD5LRinputevents_mvFluids/IntakemL88.000000
50813LACTATENaNlabeventsBLOOD GASNaN84.000000
220953RingersRingersinputevents_mvFluids - Other (Not In Use)mL84.000000
46207OR LRinputevents_cvFree Form Intake81.333333
44184LR Bolusinputevents_cvFree Form Intake70.000000
44521LR bolusinputevents_cvFree Form Intake70.000000
46781lr bolusinputevents_cvFree Form Intake70.000000
44815LR BOLUSinputevents_cvFree Form Intake70.000000
44915D5LR 40Kinputevents_cvFree Form Intake70.000000
46538PD solution ininputevents_cvFree Form Intake68.333333
44837ED URINEoutputevents68.000000
8313Nystatin solutionchartevents67.666667
30125Milrinoneinputevents_cv67.333333
221986MilrinoneMilrinoneinputevents_mvMedicationsmg67.333333
30159D5 Ringers Lact.inputevents_cv67.333333
228159Purge Solution Flow RatePurge Solution Flow RatecharteventsImpellaml/hr66.666667
225953Solution (Peritoneal Dialysis)Solution (PD)Dialysis66.000000
45983Pitocin/LRinputevents_cvFree Form Intake65.333333
42409D5LR W/40Kinputevents_cvFree Form Intake65.333333
42978D5LR 20KCLinputevents_cvFree Form Intake65.333333
1189finger stickchartevents65.000000
5743NT suctionchartevents64.666667
1520ACTcharteventsCoags64.666667
220507Activated Clotting TimeACTcharteventsLabsNone64.666667
1671actchartevents64.666667
8339Neo Opium Solutionchartevents64.333333
42288LR w/40 kclinputevents_cvFree Form Intake64.000000
225072Living situationLiving situationAdm History/FHPA63.666667
42345LR w/ 40 mEqinputevents_cvFree Form Intake63.333333
2773sjlactatechartevents63.333333
5088fingerschartevents63.333333
42265LR W/ 20 KCLinputevents_cvFree Form Intake63.333333
220955Ringers LactateRingers Lactateinputevents_mvFluids - Other (Not In Use)mL63.000000
\n", "
" ], "text/plain": [ " label abbreviation \\\n", "itemid \n", "225828 LR LR \n", "44367 LR \n", "2971 LR \n", "1634 lactated ringers \n", "30021 Lactated Ringers \n", "45532 IR Lactated ringers \n", "225943 Solution Solution \n", "225827 D5LR D5LR \n", "50813 LACTATE NaN \n", "220953 Ringers Ringers \n", "46207 OR LR \n", "44184 LR Bolus \n", "44521 LR bolus \n", "46781 lr bolus \n", "44815 LR BOLUS \n", "44915 D5LR 40K \n", "46538 PD solution in \n", "44837 ED URINE \n", "8313 Nystatin solution \n", "30125 Milrinone \n", "221986 Milrinone Milrinone \n", "30159 D5 Ringers Lact. \n", "228159 Purge Solution Flow Rate Purge Solution Flow Rate \n", "225953 Solution (Peritoneal Dialysis) Solution (PD) \n", "45983 Pitocin/LR \n", "42409 D5LR W/40K \n", "42978 D5LR 20KCL \n", "1189 finger stick \n", "5743 NT suction \n", "1520 ACT \n", "220507 Activated Clotting Time ACT \n", "1671 act \n", "8339 Neo Opium Solution \n", "42288 LR w/40 kcl \n", "225072 Living situation Living situation \n", "42345 LR w/ 40 mEq \n", "2773 sjlactate \n", "5088 fingers \n", "42265 LR W/ 20 KCL \n", "220955 Ringers Lactate Ringers Lactate \n", "\n", " linksto category unitname score \n", "itemid \n", "225828 inputevents_mv Fluids/Intake mL 110.000000 \n", "44367 inputevents_cv Free Form Intake 110.000000 \n", "2971 chartevents 110.000000 \n", "1634 chartevents 110.000000 \n", "30021 inputevents_cv 110.000000 \n", "45532 inputevents_cv Free Form Intake 104.000000 \n", "225943 inputevents_mv Fluids/Intake mL 88.000000 \n", "225827 inputevents_mv Fluids/Intake mL 88.000000 \n", "50813 labevents BLOOD GAS NaN 84.000000 \n", "220953 inputevents_mv Fluids - Other (Not In Use) mL 84.000000 \n", "46207 inputevents_cv Free Form Intake 81.333333 \n", "44184 inputevents_cv Free Form Intake 70.000000 \n", "44521 inputevents_cv Free Form Intake 70.000000 \n", "46781 inputevents_cv Free Form Intake 70.000000 \n", "44815 inputevents_cv Free Form Intake 70.000000 \n", "44915 inputevents_cv Free Form Intake 70.000000 \n", "46538 inputevents_cv Free Form Intake 68.333333 \n", "44837 outputevents 68.000000 \n", "8313 chartevents 67.666667 \n", "30125 inputevents_cv 67.333333 \n", "221986 inputevents_mv Medications mg 67.333333 \n", "30159 inputevents_cv 67.333333 \n", "228159 chartevents Impella ml/hr 66.666667 \n", "225953 Dialysis 66.000000 \n", "45983 inputevents_cv Free Form Intake 65.333333 \n", "42409 inputevents_cv Free Form Intake 65.333333 \n", "42978 inputevents_cv Free Form Intake 65.333333 \n", "1189 chartevents 65.000000 \n", "5743 chartevents 64.666667 \n", "1520 chartevents Coags 64.666667 \n", "220507 chartevents Labs None 64.666667 \n", "1671 chartevents 64.666667 \n", "8339 chartevents 64.333333 \n", "42288 inputevents_cv Free Form Intake 64.000000 \n", "225072 Adm History/FHPA 63.666667 \n", "42345 inputevents_cv Free Form Intake 63.333333 \n", "2773 chartevents 63.333333 \n", "5088 chartevents 63.333333 \n", "42265 inputevents_cv Free Form Intake 63.333333 \n", "220955 inputevents_mv Fluids - Other (Not In Use) mL 63.000000 " ] }, "execution_count": 114, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df.head(40)" ] }, { "cell_type": "code", "execution_count": 115, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.LACTATED_RINGERS] = [225828,44367,2971,1634,30021,220953,46207,44184,44521,46781,44815]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Pressors" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Norepinephrine" ] }, { "cell_type": "code", "execution_count": 117, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
30047Levophedinputevents_cv110.000000
221906NorepinephrineNorepinephrineinputevents_mvMedicationsmg110.000000
30120Levophed-kinputevents_cv102.666667
30044Epinephrineinputevents_cv102.000000
221289EpinephrineEpinephrineinputevents_mvMedicationsmg102.000000
5752Epinephrinchartevents98.666667
51201EPINEPHERINENaNlabeventsHEMATOLOGYNaN88.666667
30119Epinephrine-kinputevents_cv84.666667
30309Epinephrine Dripinputevents_cv77.333333
30127Neosynephrineinputevents_cv72.333333
225922NephramineNephramineinputevents_mvNutrition - SupplementsmL71.333333
3112epinephrine mcg/minchartevents71.000000
50820PHNaNlabeventsBLOOD GASNaN70.000000
51491PHNaNlabeventsHEMATOLOGYNaN70.000000
51094PHNaNlabeventsCHEMISTRYNaN70.000000
50831PHNaNlabeventsBLOOD GASNaN70.000000
45183EDinputevents_cvFree Form Intake70.000000
7459Phchartevents70.000000
1673PHchartevents70.000000
30128Neosynephrine-kinputevents_cv69.666667
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "30047 Levophed inputevents_cv \n", "221906 Norepinephrine Norepinephrine inputevents_mv \n", "30120 Levophed-k inputevents_cv \n", "30044 Epinephrine inputevents_cv \n", "221289 Epinephrine Epinephrine inputevents_mv \n", "5752 Epinephrin chartevents \n", "51201 EPINEPHERINE NaN labevents \n", "30119 Epinephrine-k inputevents_cv \n", "30309 Epinephrine Drip inputevents_cv \n", "30127 Neosynephrine inputevents_cv \n", "225922 Nephramine Nephramine inputevents_mv \n", "3112 epinephrine mcg/min chartevents \n", "50820 PH NaN labevents \n", "51491 PH NaN labevents \n", "51094 PH NaN labevents \n", "50831 PH NaN labevents \n", "45183 ED inputevents_cv \n", "7459 Ph chartevents \n", "1673 PH chartevents \n", "30128 Neosynephrine-k inputevents_cv \n", "\n", " category unitname score \n", "itemid \n", "30047 110.000000 \n", "221906 Medications mg 110.000000 \n", "30120 102.666667 \n", "30044 102.000000 \n", "221289 Medications mg 102.000000 \n", "5752 98.666667 \n", "51201 HEMATOLOGY NaN 88.666667 \n", "30119 84.666667 \n", "30309 77.333333 \n", "30127 72.333333 \n", "225922 Nutrition - Supplements mL 71.333333 \n", "3112 71.000000 \n", "50820 BLOOD GAS NaN 70.000000 \n", "51491 HEMATOLOGY NaN 70.000000 \n", "51094 CHEMISTRY NaN 70.000000 \n", "50831 BLOOD GAS NaN 70.000000 \n", "45183 Free Form Intake 70.000000 \n", "7459 70.000000 \n", "1673 70.000000 \n", "30128 69.666667 " ] }, "execution_count": 117, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'levophed',\n", " 'norepinephrine'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 118, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.NOREPINEPHRINE] = [30047,221906,30120]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Vasopressin" ] }, { "cell_type": "code", "execution_count": 119, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelabbreviationlinkstocategoryunitnamescore
itemid
30051Vasopressininputevents_cv110.000000
2445Vasopressinchartevents110.000000
222315VasopressinVasopressininputevents_mvMedicationsunits110.000000
1136vasopressinchartevents110.000000
1222VASOPRESSINchartevents110.000000
2334vasopressin u/hrchartevents97.333333
2561VASOPRESSIN U/HRchartevents97.333333
7341Vasopressin u/hrchartevents96.666667
46570vassopressininputevents_cvFree Form Intake94.333333
42802VASOPRESSIN CC/HR.inputevents_cvFree Form Intake94.000000
6255VAsopressin 0.04 schartevents92.333333
2765VASOPRESSIN UNIT/Rchartevents92.333333
2248VASOPRESSIN UNIT/MINchartevents90.666667
42273vasopressin unit/mininputevents_cvFree Form Intake90.666667
1327vasopressin unit/minchartevents90.666667
6269Dressingchartevents68.666667
46Angio Dressing #2chartevents68.000000
6691DRIV PRESSchartevents68.000000
45Angio Dressing #1chartevents68.000000
228448Angio Dressing # 4Angio Dressing # 4Cardiovascular67.333333
\n", "
" ], "text/plain": [ " label abbreviation linksto \\\n", "itemid \n", "30051 Vasopressin inputevents_cv \n", "2445 Vasopressin chartevents \n", "222315 Vasopressin Vasopressin inputevents_mv \n", "1136 vasopressin chartevents \n", "1222 VASOPRESSIN chartevents \n", "2334 vasopressin u/hr chartevents \n", "2561 VASOPRESSIN U/HR chartevents \n", "7341 Vasopressin u/hr chartevents \n", "46570 vassopressin inputevents_cv \n", "42802 VASOPRESSIN CC/HR. inputevents_cv \n", "6255 VAsopressin 0.04 s chartevents \n", "2765 VASOPRESSIN UNIT/R chartevents \n", "2248 VASOPRESSIN UNIT/MIN chartevents \n", "42273 vasopressin unit/min inputevents_cv \n", "1327 vasopressin unit/min chartevents \n", "6269 Dressing chartevents \n", "46 Angio Dressing #2 chartevents \n", "6691 DRIV PRESS chartevents \n", "45 Angio Dressing #1 chartevents \n", "228448 Angio Dressing # 4 Angio Dressing # 4 \n", "\n", " category unitname score \n", "itemid \n", "30051 110.000000 \n", "2445 110.000000 \n", "222315 Medications units 110.000000 \n", "1136 110.000000 \n", "1222 110.000000 \n", "2334 97.333333 \n", "2561 97.333333 \n", "7341 96.666667 \n", "46570 Free Form Intake 94.333333 \n", "42802 Free Form Intake 94.000000 \n", "6255 92.333333 \n", "2765 92.333333 \n", "2248 90.666667 \n", "42273 Free Form Intake 90.666667 \n", "1327 90.666667 \n", "6269 68.666667 \n", "46 68.000000 \n", "6691 68.000000 \n", "45 68.000000 \n", "228448 Cardiovascular 67.333333 " ] }, "execution_count": 119, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_df = explorer.search([\n", " 'vasopressin',\n", " 'argipressin',\n", " 'arginine vasopressin'\n", " ])\n", "out_df.head(20)" ] }, { "cell_type": "code", "execution_count": 120, "metadata": { "collapsed": true }, "outputs": [], "source": [ "keep_dict[data_dict.labels.VASOPRESSIN] = out_df.loc[:1327].index.tolist()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Now build our mapping dataframe" ] }, { "cell_type": "code", "execution_count": 124, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import pandas as pd\n", "from itertools import product" ] }, { "cell_type": "code", "execution_count": 131, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "tuples = []\n", "\n", "for label,items in keep_dict.iteritems():\n", " tuples += list(product([label],items))\n", "\n", "item_map = pd.DataFrame(tuples,columns=['label','itemid'])" ] }, { "cell_type": "code", "execution_count": 132, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelitemid
0glasgow coma scale eye opening184
1glasgow coma scale eye opening220739
2glasgow coma scale motor454
3glasgow coma scale motor223901
4blood pressure systolic220179
\n", "
" ], "text/plain": [ " label itemid\n", "0 glasgow coma scale eye opening 184\n", "1 glasgow coma scale eye opening 220739\n", "2 glasgow coma scale motor 454\n", "3 glasgow coma scale motor 223901\n", "4 blood pressure systolic 220179" ] }, "execution_count": 132, "metadata": {}, "output_type": "execute_result" } ], "source": [ "item_map.head()" ] }, { "cell_type": "code", "execution_count": 133, "metadata": { "collapsed": true }, "outputs": [], "source": [ "item_map.to_csv('config/mimic_item_map.csv',index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Extract MIMIC III data" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import pandas as pd\n", "import mimic\n", "from constants import ALL\n", "import icu_data_defs \n", "import utils\n", "import logger\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(icu_data_defs)\n", "reload(mimic)\n", "reload(logger)\n", "\n", "def extract_labels(conn,labels,item_map_fname,hdf5_fname,hadm_ids=ALL):\n", " logger.log('Start extracting {} labels'.format(len(labels)),new_level=True)\n", " extractor = mimic.mimic_extractor(conn,item_map_fname)\n", " for label in labels:\n", " logger.log(label.upper(),new_level=True)\n", " df = extractor.extract_label(label,hadm_ids)\n", " if df is None: continue\n", " utils.save_df(df,hdf5_fname,'extract/{}'.format(label))\n", " display(df.head())\n", " print df.shape\n", " del df\n", " logger.end_log()\n", " return \n", "\n", "#connect to the mimic database\n", "conn = mimic.connect()\n", "\n", "#these are the default config files we will be using\n", "item_map_fname = 'config/mimic_item_map.csv'\n", "mimic_data_sef_fname = 'config/data_definitions.xlsx'\n", "\n", "#get all labels\n", "data_dict = icu_data_defs.data_dictionary(mimic_data_sef_fname)\n", "simple_data = data_dict.get_panel_defintions(12) #12 is \"simple data\"\n", "labels = simple_data.label.unique().tolist()\n", "\n", "#where we will be storing this extraction\n", "hdf5_fname = 'data/mimic_data'" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 03:26:58) Start extracting 5 labels\n", "(2017-06-03 03:26:58)>> HEART RATE\n", "(2017-06-03 03:26:59)>>>> Extracting 5 items from chartevents\n", "(2017-06-03 03:26:59)<<<< DONE (0.0s)\n", "(2017-06-03 03:26:59)>>>> Combine DF\n", "(2017-06-03 03:26:59)<<<< DONE (0.0s)\n", "(2017-06-03 03:26:59)>>>> Clean UOM\n", "(2017-06-03 03:26:59)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
01438382129-07-14 00:00:0062beat/min211
11438382129-07-14 00:30:0067beat/min211
21438382129-07-14 01:00:0067beat/min211
31438382129-07-14 01:20:0063beat/min211
41438382129-07-14 01:25:0063beat/min211
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 143838 2129-07-14 00:00:00 62 beat/min 211\n", "1 143838 2129-07-14 00:30:00 67 beat/min 211\n", "2 143838 2129-07-14 01:00:00 67 beat/min 211\n", "3 143838 2129-07-14 01:20:00 63 beat/min 211\n", "4 143838 2129-07-14 01:25:00 63 beat/min 211" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(1649, 5)\n", "(2017-06-03 03:27:01)<< DONE (3.0s)\n", "(2017-06-03 03:27:01)>> BLOOD PRESSURE SYSTOLIC\n", "(2017-06-03 03:27:01)>>>> Extracting 14 items from chartevents\n", "(2017-06-03 03:27:02)<<<< DONE (1.0s)\n", "(2017-06-03 03:27:02)>>>> Combine DF\n", "(2017-06-03 03:27:02)<<<< DONE (0.0s)\n", "(2017-06-03 03:27:02)>>>> Clean UOM\n", "(2017-06-03 03:27:02)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
01438382129-07-13 22:40:00145mmHg51
11438382129-07-13 23:00:00136mmHg51
21438382129-07-13 23:30:00179mmHg51
31438382129-07-14 00:00:00131mmHg51
41438382129-07-14 00:30:00146mmHg51
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 143838 2129-07-13 22:40:00 145 mmHg 51\n", "1 143838 2129-07-13 23:00:00 136 mmHg 51\n", "2 143838 2129-07-13 23:30:00 179 mmHg 51\n", "3 143838 2129-07-14 00:00:00 131 mmHg 51\n", "4 143838 2129-07-14 00:30:00 146 mmHg 51" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(644, 5)\n", "(2017-06-03 03:27:02)<< DONE (1.0s)\n", "(2017-06-03 03:27:02)>> BLOOD PRESSURE DIASTOLIC\n", "(2017-06-03 03:27:02)>>>> Extracting 15 items from chartevents\n", "(2017-06-03 03:27:02)<<<< DONE (0.0s)\n", "(2017-06-03 03:27:02)>>>> Combine DF\n", "(2017-06-03 03:27:02)<<<< DONE (0.0s)\n", "(2017-06-03 03:27:02)>>>> Clean UOM\n", "(2017-06-03 03:27:02)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
01135402161-10-12 21:00:0053mmHg220051
11135402161-10-12 14:00:0084mmHg220051
21135402161-10-12 10:30:0054mmHg220051
31135402161-10-12 15:00:0060mmHg220051
41135402161-10-12 16:00:0059mmHg220051
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 113540 2161-10-12 21:00:00 53 mmHg 220051\n", "1 113540 2161-10-12 14:00:00 84 mmHg 220051\n", "2 113540 2161-10-12 10:30:00 54 mmHg 220051\n", "3 113540 2161-10-12 15:00:00 60 mmHg 220051\n", "4 113540 2161-10-12 16:00:00 59 mmHg 220051" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(640, 5)\n", "(2017-06-03 03:27:02)<< DONE (0.0s)\n", "(2017-06-03 03:27:02)>> BLOOD PRESSURE MEAN\n", "(2017-06-03 03:27:03)>>>> Extracting 3 items from chartevents\n", "(2017-06-03 03:27:03)<<<< DONE (0.0s)\n", "(2017-06-03 03:27:03)>>>> Combine DF\n", "(2017-06-03 03:27:03)<<<< DONE (0.0s)\n", "(2017-06-03 03:27:03)>>>> Clean UOM\n", "(2017-06-03 03:27:03)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
01135402161-10-12 21:00:0076mmHg220052
11135402161-10-12 12:00:0091mmHg220052
21135402161-10-12 14:00:0082mmHg220052
31135402161-10-12 10:30:0070mmHg220052
41135402161-10-12 15:00:0080mmHg220052
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 113540 2161-10-12 21:00:00 76 mmHg 220052\n", "1 113540 2161-10-12 12:00:00 91 mmHg 220052\n", "2 113540 2161-10-12 14:00:00 82 mmHg 220052\n", "3 113540 2161-10-12 10:30:00 70 mmHg 220052\n", "4 113540 2161-10-12 15:00:00 80 mmHg 220052" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(58, 5)\n", "(2017-06-03 03:27:03)<< DONE (1.0s)\n", "(2017-06-03 03:27:03)>> RESPIRATORY RATE\n", "(2017-06-03 03:27:03)>>>> Extracting 4 items from chartevents\n", "(2017-06-03 03:27:04)<<<< DONE (1.0s)\n", "(2017-06-03 03:27:04)>>>> Combine DF\n", "(2017-06-03 03:27:04)<<<< DONE (0.0s)\n", "(2017-06-03 03:27:04)>>>> Clean UOM\n", "(2017-06-03 03:27:04)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
01438382129-07-13 23:30:0017breath/min618
11438382129-07-14 00:00:0020breath/min618
21438382129-07-14 00:30:0017breath/min618
31438382129-07-14 01:00:0013breath/min618
41438382129-07-14 01:20:0015breath/min618
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 143838 2129-07-13 23:30:00 17 breath/min 618\n", "1 143838 2129-07-14 00:00:00 20 breath/min 618\n", "2 143838 2129-07-14 00:30:00 17 breath/min 618\n", "3 143838 2129-07-14 01:00:00 13 breath/min 618\n", "4 143838 2129-07-14 01:20:00 15 breath/min 618" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(1647, 5)\n", "(2017-06-03 03:27:04)<< DONE (1.0s)\n", "(2017-06-03 03:27:04) DONE (6.0s)\n" ] } ], "source": [ "hadm_ids = mimic.sample_hadm_ids(5,conn)\n", "extract_labels(conn,labels[0:5],item_map_fname,hdf5_fname,hadm_ids)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 03:27:10) Start extracting 18 labels\n", "(2017-06-03 03:27:10)>> HEART RATE\n", "(2017-06-03 03:27:10)>>>> Extracting 5 items from chartevents\n", "(2017-06-03 03:28:03)<<<< DONE (53.0s)\n", "(2017-06-03 03:28:03)>>>> Combine DF\n", "(2017-06-03 03:28:03)<<<< DONE (0.0s)\n", "(2017-06-03 03:28:03)>>>> Clean UOM\n", "(2017-06-03 03:28:07)<<<< DONE (4.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-26 03:00:0079beat/min211
1106266.02114-12-02 06:00:00116beat/min211
2106266.02114-12-02 07:00:00116beat/min211
3106266.02114-12-02 08:00:00144beat/min211
4106266.02114-12-03 08:00:00144beat/min211
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-26 03:00:00 79 beat/min 211\n", "1 106266.0 2114-12-02 06:00:00 116 beat/min 211\n", "2 106266.0 2114-12-02 07:00:00 116 beat/min 211\n", "3 106266.0 2114-12-02 08:00:00 144 beat/min 211\n", "4 106266.0 2114-12-03 08:00:00 144 beat/min 211" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(7952939, 5)\n", "(2017-06-03 03:28:20)<< DONE (70.0s)\n", "(2017-06-03 03:28:20)>> BLOOD PRESSURE SYSTOLIC\n", "(2017-06-03 03:28:21)>>>> Extracting 14 items from chartevents\n", "(2017-06-03 03:29:45)<<<< DONE (84.0s)\n", "(2017-06-03 03:29:45)>>>> Combine DF\n", "(2017-06-03 03:29:45)<<<< DONE (0.0s)\n", "(2017-06-03 03:29:45)>>>> Clean UOM\n", "(2017-06-03 03:29:48)<<<< DONE (3.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0176203.02154-11-12 22:00:0051
1197569.02133-03-09 00:00:00117mmHg51
2197569.02133-03-09 01:00:00117mmHg51
3197569.02133-03-09 02:00:00127mmHg51
4197569.02133-03-09 03:00:00126mmHg51
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 176203.0 2154-11-12 22:00:00 51\n", "1 197569.0 2133-03-09 00:00:00 117 mmHg 51\n", "2 197569.0 2133-03-09 01:00:00 117 mmHg 51\n", "3 197569.0 2133-03-09 02:00:00 127 mmHg 51\n", "4 197569.0 2133-03-09 03:00:00 126 mmHg 51" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(6374824, 5)\n", "(2017-06-03 03:30:00)<< DONE (100.0s)\n", "(2017-06-03 03:30:00)>> BLOOD PRESSURE DIASTOLIC\n", "(2017-06-03 03:30:00)>>>> Extracting 15 items from chartevents\n", "(2017-06-03 03:31:18)<<<< DONE (78.0s)\n", "(2017-06-03 03:31:18)>>>> Combine DF\n", "(2017-06-03 03:31:19)<<<< DONE (1.0s)\n", "(2017-06-03 03:31:19)>>>> Clean UOM\n", "(2017-06-03 03:31:22)<<<< DONE (3.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0124321.02157-10-21 12:15:0074mmHg220051
1124321.02157-10-21 13:00:0067mmHg220051
2124321.02157-10-21 14:00:0064mmHg220051
3124321.02157-10-21 15:00:0076mmHg220051
4124321.02157-10-21 16:17:0080mmHg220051
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 124321.0 2157-10-21 12:15:00 74 mmHg 220051\n", "1 124321.0 2157-10-21 13:00:00 67 mmHg 220051\n", "2 124321.0 2157-10-21 14:00:00 64 mmHg 220051\n", "3 124321.0 2157-10-21 15:00:00 76 mmHg 220051\n", "4 124321.0 2157-10-21 16:17:00 80 mmHg 220051" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(6371249, 5)\n", "(2017-06-03 03:31:34)<< DONE (94.0s)\n", "(2017-06-03 03:31:34)>> BLOOD PRESSURE MEAN\n", "(2017-06-03 03:31:34)>>>> Extracting 3 items from chartevents\n", "(2017-06-03 03:32:13)<<<< DONE (39.0s)\n", "(2017-06-03 03:32:13)>>>> Combine DF\n", "(2017-06-03 03:32:13)<<<< DONE (0.0s)\n", "(2017-06-03 03:32:13)>>>> Clean UOM\n", "(2017-06-03 03:32:14)<<<< DONE (1.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
01243212157-10-21 13:00:0086mmHg220052
11243212157-10-21 14:00:0086mmHg220052
21243212157-10-21 15:00:0098mmHg220052
31243212157-10-21 16:17:00104mmHg220052
41243212157-10-21 17:00:00104mmHg220052
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 124321 2157-10-21 13:00:00 86 mmHg 220052\n", "1 124321 2157-10-21 14:00:00 86 mmHg 220052\n", "2 124321 2157-10-21 15:00:00 98 mmHg 220052\n", "3 124321 2157-10-21 16:17:00 104 mmHg 220052\n", "4 124321 2157-10-21 17:00:00 104 mmHg 220052" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2536271, 5)\n", "(2017-06-03 03:32:18)<< DONE (44.0s)\n", "(2017-06-03 03:32:18)>> RESPIRATORY RATE\n", "(2017-06-03 03:32:18)>>>> Extracting 4 items from chartevents\n", "(2017-06-03 03:34:08)<<<< DONE (110.0s)\n", "(2017-06-03 03:34:08)>>>> Combine DF\n", "(2017-06-03 03:34:08)<<<< DONE (0.0s)\n", "(2017-06-03 03:34:08)>>>> Clean UOM\n", "(2017-06-03 03:34:13)<<<< DONE (5.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-25 00:00:0020breath/min618
1122659.02131-05-22 14:00:0029breath/min618
2188670.02183-08-23 19:00:0024breath/min618
3188670.02183-08-23 20:00:0024breath/min618
4185910.02166-08-10 13:30:0022breath/min618
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-25 00:00:00 20 breath/min 618\n", "1 122659.0 2131-05-22 14:00:00 29 breath/min 618\n", "2 188670.0 2183-08-23 19:00:00 24 breath/min 618\n", "3 188670.0 2183-08-23 20:00:00 24 breath/min 618\n", "4 185910.0 2166-08-10 13:30:00 22 breath/min 618" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(7810019, 5)\n", "(2017-06-03 03:34:27)<< DONE (129.0s)\n", "(2017-06-03 03:34:27)>> TEMPERATURE BODY\n", "(2017-06-03 03:34:27)>>>> Extracting 4 items from chartevents\n", "(2017-06-03 03:35:01)<<<< DONE (34.0s)\n", "(2017-06-03 03:35:01)>>>> Combine DF\n", "(2017-06-03 03:35:01)<<<< DONE (0.0s)\n", "(2017-06-03 03:35:01)>>>> Clean UOM\n", "(2017-06-03 03:35:02)<<<< DONE (1.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-25 00:00:0098.400001525878906degF678
1188670.02183-08-23 20:00:0099.300003051757812degF678
2185910.02166-08-12 08:00:0098.699996948242188degF678
3185910.02166-08-13 17:00:0099.900001525878906degF678
4166707.02122-02-11 20:00:0037.900001525878906degC676
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-25 00:00:00 98.400001525878906 degF 678\n", "1 188670.0 2183-08-23 20:00:00 99.300003051757812 degF 678\n", "2 185910.0 2166-08-12 08:00:00 98.699996948242188 degF 678\n", "3 185910.0 2166-08-13 17:00:00 99.900001525878906 degF 678\n", "4 166707.0 2122-02-11 20:00:00 37.900001525878906 degC 676" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(1751447, 5)\n", "(2017-06-03 03:35:05)<< DONE (38.0s)\n", "(2017-06-03 03:35:05)>> OXYGEN SATURATION PULSE OXIMETRY\n", "(2017-06-03 03:35:06)>>>> Extracting 2 items from chartevents\n", "(2017-06-03 03:35:59)<<<< DONE (53.0s)\n", "(2017-06-03 03:35:59)>>>> Combine DF\n", "(2017-06-03 03:35:59)<<<< DONE (0.0s)\n", "(2017-06-03 03:35:59)>>>> Clean UOM\n", "(2017-06-03 03:36:02)<<<< DONE (3.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-25 00:00:00100percent646
1185910.02166-08-25 00:15:00100percent646
2122659.02131-05-22 14:00:00100percent646
3188670.02183-08-23 19:00:0097percent646
4188670.02183-08-23 20:00:0097percent646
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-25 00:00:00 100 percent 646\n", "1 185910.0 2166-08-25 00:15:00 100 percent 646\n", "2 122659.0 2131-05-22 14:00:00 100 percent 646\n", "3 188670.0 2183-08-23 19:00:00 97 percent 646\n", "4 188670.0 2183-08-23 20:00:00 97 percent 646" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(6099827, 5)\n", "(2017-06-03 03:36:14)<< DONE (69.0s)\n", "(2017-06-03 03:36:14)>> WEIGHT BODY\n", "(2017-06-03 03:36:14)>>>> Extracting 3 items from chartevents\n", "(2017-06-03 03:36:48)<<<< DONE (34.0s)\n", "(2017-06-03 03:36:48)>>>> Combine DF\n", "(2017-06-03 03:36:49)<<<< DONE (1.0s)\n", "(2017-06-03 03:36:49)>>>> Clean UOM\n", "(2017-06-03 03:36:49)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-20 06:00:00102.5kg763
1166707.02122-02-15 07:00:0099.400001525878906kg763
2182104.02131-05-05 06:00:00109.69999694824219kg763
3157907.02129-06-10 05:00:00103.19999694824219kg763
4166707.02122-02-14 06:00:00101.80000305175781kg763
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-20 06:00:00 102.5 kg 763\n", "1 166707.0 2122-02-15 07:00:00 99.400001525878906 kg 763\n", "2 182104.0 2131-05-05 06:00:00 109.69999694824219 kg 763\n", "3 157907.0 2129-06-10 05:00:00 103.19999694824219 kg 763\n", "4 166707.0 2122-02-14 06:00:00 101.80000305175781 kg 763" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(95425, 5)\n", "(2017-06-03 03:36:49)<< DONE (35.0s)\n", "(2017-06-03 03:36:49)>> OUTPUT URINE\n", "(2017-06-03 03:36:49)>>>> Extracting 2 items from chartevents\n", "(2017-06-03 03:37:28)<<<< DONE (39.0s)\n", "(2017-06-03 03:37:28)>>>> Extracting 29 items from outputevents\n", "(2017-06-03 03:37:59)<<<< DONE (31.0s)\n", "(2017-06-03 03:37:59)>>>> Combine DF\n", "(2017-06-03 03:37:59)<<<< DONE (0.0s)\n", "(2017-06-03 03:37:59)>>>> Clean UOM\n", "(2017-06-03 03:38:09)<<<< DONE (10.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0106266.02114-12-03 08:00:00Voiding qsml3686
1106266.02114-12-09 09:00:00Voiding qsml3686
2106266.02114-12-09 14:00:00Voiding qsml3686
3106266.02114-12-05 21:30:00Voiding qsml3686
4106266.02114-12-04 16:00:00Voiding qsml3686
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 106266.0 2114-12-03 08:00:00 Voiding qs ml 3686\n", "1 106266.0 2114-12-09 09:00:00 Voiding qs ml 3686\n", "2 106266.0 2114-12-09 14:00:00 Voiding qs ml 3686\n", "3 106266.0 2114-12-05 21:30:00 Voiding qs ml 3686\n", "4 106266.0 2114-12-04 16:00:00 Voiding qs ml 3686" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(3644639, 5)\n", "(2017-06-03 03:38:15)<< DONE (86.0s)\n", "(2017-06-03 03:38:15)>> GLASGOW COMA SCALE MOTOR\n", "(2017-06-03 03:38:15)>>>> Extracting 1 items from chartevents\n", "(2017-06-03 03:39:12)<<<< DONE (57.0s)\n", "(2017-06-03 03:39:12)>>>> Combine DF\n", "(2017-06-03 03:39:12)<<<< DONE (0.0s)\n", "(2017-06-03 03:39:12)>>>> Clean UOM\n", "(2017-06-03 03:39:12)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-25 00:00:006 Obeys Commands454
1188670.02183-08-23 20:00:006 Obeys Commands454
2185910.02166-08-27 12:00:005 Localizes Pain454
3185910.02166-09-03 12:00:006 Obeys Commands454
4101757.02133-01-03 16:00:006 Obeys Commands454
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-25 00:00:00 6 Obeys Commands 454\n", "1 188670.0 2183-08-23 20:00:00 6 Obeys Commands 454\n", "2 185910.0 2166-08-27 12:00:00 5 Localizes Pain 454\n", "3 185910.0 2166-09-03 12:00:00 6 Obeys Commands 454\n", "4 101757.0 2133-01-03 16:00:00 6 Obeys Commands 454" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(952565, 5)\n", "(2017-06-03 03:39:13)<< DONE (58.0s)\n", "(2017-06-03 03:39:13)>> GLASGOW COMA SCALE EYE OPENING\n", "(2017-06-03 03:39:13)>>>> Extracting 1 items from chartevents\n", "(2017-06-03 03:39:57)<<<< DONE (44.0s)\n", "(2017-06-03 03:39:57)>>>> Combine DF\n", "(2017-06-03 03:39:57)<<<< DONE (0.0s)\n", "(2017-06-03 03:39:57)>>>> Clean UOM\n", "(2017-06-03 03:39:57)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0188670.02183-08-23 20:00:004 Spontaneously184
1185910.02166-08-15 20:00:004 Spontaneously184
2185910.02166-08-16 16:00:003 To speech184
3185910.02166-08-23 04:00:003 To speech184
4185910.02166-08-27 12:00:001 No Response184
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 188670.0 2183-08-23 20:00:00 4 Spontaneously 184\n", "1 185910.0 2166-08-15 20:00:00 4 Spontaneously 184\n", "2 185910.0 2166-08-16 16:00:00 3 To speech 184\n", "3 185910.0 2166-08-23 04:00:00 3 To speech 184\n", "4 185910.0 2166-08-27 12:00:00 1 No Response 184" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(956672, 5)\n", "(2017-06-03 03:39:58)<< DONE (45.0s)\n", "(2017-06-03 03:39:58)>> GLASGOW COMA SCALE VERBAL\n", "(2017-06-03 03:39:59)>>>> Extracting 1 items from chartevents\n", "(2017-06-03 03:40:22)<<<< DONE (23.0s)\n", "(2017-06-03 03:40:22)>>>> Combine DF\n", "(2017-06-03 03:40:22)<<<< DONE (0.0s)\n", "(2017-06-03 03:40:22)>>>> Clean UOM\n", "(2017-06-03 03:40:22)<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-25 00:00:001.0 ET/Trach723
1188670.02183-08-23 20:00:004 Confused723
2185910.02166-08-12 08:00:001.0 ET/Trach723
3185910.02166-08-27 12:00:001.0 ET/Trach723
4166707.02122-02-11 20:00:005 Oriented723
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-25 00:00:00 1.0 ET/Trach 723\n", "1 188670.0 2183-08-23 20:00:00 4 Confused 723\n", "2 185910.0 2166-08-12 08:00:00 1.0 ET/Trach 723\n", "3 185910.0 2166-08-27 12:00:00 1.0 ET/Trach 723\n", "4 166707.0 2122-02-11 20:00:00 5 Oriented 723" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(954700, 5)\n", "(2017-06-03 03:40:24)<< DONE (26.0s)\n", "(2017-06-03 03:40:24)>> NORMAL SALINE\n", "(2017-06-03 03:40:24)>>>> Extracting 2 items from chartevents\n", "(2017-06-03 03:40:24)<<<< DONE (0.0s)\n", "(2017-06-03 03:40:24)>>>> Extracting 4 items from inputevents_cv\n", "(2017-06-03 03:40:25)<<<< DONE (1.0s)\n", "(2017-06-03 03:40:25)>>>> Extracting 2 items from inputevents_mv\n", "(2017-06-03 03:40:43)<<<< DONE (18.0s)\n", "(2017-06-03 03:40:43)>>>> Combine DF\n", "(2017-06-03 03:40:43)<<<< DONE (0.0s)\n", "(2017-06-03 03:40:43)>>>> Clean UOM\n", "(2017-06-03 03:40:46)<<<< DONE (3.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
01104682134-02-13 03:45:00givenml4647
11104682134-02-13 07:05:00givenml4647
21754192122-08-16 01:00:00givenml4647
31434942147-07-02 23:00:00givenml4647
41434942147-07-02 23:43:00givenml4647
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 110468 2134-02-13 03:45:00 given ml 4647\n", "1 110468 2134-02-13 07:05:00 given ml 4647\n", "2 175419 2122-08-16 01:00:00 given ml 4647\n", "3 143494 2147-07-02 23:00:00 given ml 4647\n", "4 143494 2147-07-02 23:43:00 given ml 4647" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(817373, 5)\n", "(2017-06-03 03:40:46)<< DONE (22.0s)\n", "(2017-06-03 03:40:46)>> LACTATED RINGERS\n", "(2017-06-03 03:40:46)>>>> Extracting 2 items from chartevents\n", "(2017-06-03 03:40:46)<<<< DONE (0.0s)\n", "(2017-06-03 03:40:46)>>>> Extracting 7 items from inputevents_cv\n", "(2017-06-03 03:40:50)<<<< DONE (4.0s)\n", "(2017-06-03 03:40:50)>>>> Extracting 2 items from inputevents_mv\n", "(2017-06-03 03:40:51)<<<< DONE (1.0s)\n", "(2017-06-03 03:40:51)>>>> Combine DF\n", "(2017-06-03 03:40:51)<<<< DONE (0.0s)\n", "(2017-06-03 03:40:51)>>>> Clean UOM\n", "(2017-06-03 03:40:53)<<<< DONE (2.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0194762.02110-11-22 18:00:00100ml1634
1178769.02181-06-01 05:00:00100ml2971
2178769.02181-06-01 01:00:00100ml2971
3178769.02181-06-01 02:00:00100ml2971
4178769.02181-06-01 03:00:00100ml2971
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 194762.0 2110-11-22 18:00:00 100 ml 1634\n", "1 178769.0 2181-06-01 05:00:00 100 ml 2971\n", "2 178769.0 2181-06-01 01:00:00 100 ml 2971\n", "3 178769.0 2181-06-01 02:00:00 100 ml 2971\n", "4 178769.0 2181-06-01 03:00:00 100 ml 2971" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(504306, 5)\n", "(2017-06-03 03:40:53)<< DONE (7.0s)\n", "(2017-06-03 03:40:53)>> NOREPINEPHRINE\n", "(2017-06-03 03:40:53)>>>> Extracting 2 items from inputevents_cv\n", "(2017-06-03 03:41:10)<<<< DONE (17.0s)\n", "(2017-06-03 03:41:10)>>>> Extracting 1 items from inputevents_mv\n", "(2017-06-03 03:41:12)<<<< DONE (2.0s)\n", "(2017-06-03 03:41:12)>>>> Combine DF\n", "(2017-06-03 03:41:12)<<<< DONE (0.0s)\n", "(2017-06-03 03:41:12)>>>> Clean UOM\n", "(2017-06-03 03:41:16)<<<< DONE (4.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0181516.02121-01-27 00:00:00NaNmg30120
1194502.02102-12-20 08:00:00NaNmg30047
2172260.02124-11-11 16:00:00NaNmg30047
3114829.02107-01-15 22:00:00NaNmg30047
4181516.02121-01-27 01:00:00NaNmg30120
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 181516.0 2121-01-27 00:00:00 NaN mg 30120\n", "1 194502.0 2102-12-20 08:00:00 NaN mg 30047\n", "2 172260.0 2124-11-11 16:00:00 NaN mg 30047\n", "3 114829.0 2107-01-15 22:00:00 NaN mg 30047\n", "4 181516.0 2121-01-27 01:00:00 NaN mg 30120" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(1136938, 5)\n", "(2017-06-03 03:41:16)<< DONE (23.0s)\n", "(2017-06-03 03:41:16)>> VASOPRESSIN\n", "(2017-06-03 03:41:16)>>>> Extracting 10 items from chartevents\n", "(2017-06-03 03:41:17)<<<< DONE (1.0s)\n", "(2017-06-03 03:41:17)>>>> Extracting 4 items from inputevents_cv\n", "(2017-06-03 03:41:22)<<<< DONE (5.0s)\n", "(2017-06-03 03:41:22)>>>> Extracting 1 items from inputevents_mv\n", "(2017-06-03 03:41:23)<<<< DONE (1.0s)\n", "(2017-06-03 03:41:23)>>>> Combine DF\n", "(2017-06-03 03:41:23)<<<< DONE (0.0s)\n", "(2017-06-03 03:41:23)>>>> Clean UOM\n", "(2017-06-03 03:41:24)<<<< DONE (1.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0126005.02126-09-05 16:30:00U1136
1126005.02126-09-05 16:45:00units/hour1136
2115221.02194-12-09 05:00:003U1136
3115221.02194-12-09 06:00:003units/hour1136
4115221.02194-12-09 04:00:003units/hour1136
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 126005.0 2126-09-05 16:30:00 U 1136\n", "1 126005.0 2126-09-05 16:45:00 units/hour 1136\n", "2 115221.0 2194-12-09 05:00:00 3 U 1136\n", "3 115221.0 2194-12-09 06:00:00 3 units/hour 1136\n", "4 115221.0 2194-12-09 04:00:00 3 units/hour 1136" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(339184, 5)\n", "(2017-06-03 03:41:24)<< DONE (8.0s)\n", "(2017-06-03 03:41:24)>> HEMOGLOBIN\n", "(2017-06-03 03:41:24)>>>> Extracting 3 items from chartevents\n", "(2017-06-03 03:43:00)<<<< DONE (96.0s)\n", "(2017-06-03 03:43:00)>>>> Extracting 2 items from labevents\n", "(2017-06-03 03:43:27)<<<< DONE (27.0s)\n", "(2017-06-03 03:43:27)>>>> Combine DF\n", "(2017-06-03 03:43:27)<<<< DONE (0.0s)\n", "(2017-06-03 03:43:27)>>>> Clean UOM\n", "(2017-06-03 03:43:30)<<<< DONE (3.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-13 01:53:009.5gm/dl814
1185910.02166-08-22 02:14:0010gm/dl814
2185910.02166-08-18 03:46:0010.100000381469727gm/dl814
3157907.02129-06-11 02:00:009.8999996185302734gm/dl814
4185910.02166-08-30 02:07:009gm/dl814
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-13 01:53:00 9.5 gm/dl 814\n", "1 185910.0 2166-08-22 02:14:00 10 gm/dl 814\n", "2 185910.0 2166-08-18 03:46:00 10.100000381469727 gm/dl 814\n", "3 157907.0 2129-06-11 02:00:00 9.8999996185302734 gm/dl 814\n", "4 185910.0 2166-08-30 02:07:00 9 gm/dl 814" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(1167921, 5)\n", "(2017-06-03 03:43:31)<< DONE (127.0s)\n", "(2017-06-03 03:43:31)>> LACTATE\n", "(2017-06-03 03:43:31)>>>> Extracting 3 items from chartevents\n", "(2017-06-03 03:44:30)<<<< DONE (59.0s)\n", "(2017-06-03 03:44:30)>>>> Extracting 1 items from labevents\n", "(2017-06-03 03:44:33)<<<< DONE (3.0s)\n", "(2017-06-03 03:44:33)>>>> Combine DF\n", "(2017-06-03 03:44:33)<<<< DONE (0.0s)\n", "(2017-06-03 03:44:33)>>>> Clean UOM\n", "(2017-06-03 03:44:34)<<<< DONE (1.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-11 21:20:001.5mmol/L818
1185910.02166-08-11 21:20:001.5mmol/L1531
2175413.02170-04-11 15:00:00mmol/L818
3146828.02186-10-03 21:54:001.8mmol/L818
4146828.02186-10-03 21:54:001.8mmol/L1531
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-11 21:20:00 1.5 mmol/L 818\n", "1 185910.0 2166-08-11 21:20:00 1.5 mmol/L 1531\n", "2 175413.0 2170-04-11 15:00:00 mmol/L 818\n", "3 146828.0 2186-10-03 21:54:00 1.8 mmol/L 818\n", "4 146828.0 2186-10-03 21:54:00 1.8 mmol/L 1531" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(393608, 5)\n", "(2017-06-03 03:44:35)<< DONE (64.0s)\n", "(2017-06-03 03:44:35) DONE (1045.0s)\n" ] } ], "source": [ "df_all = extract_labels(conn,labels,item_map_fname,hdf5_fname)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Transform (and \"Load\") MIMIC III Data" ] }, { "cell_type": "code", "execution_count": 152, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import icu_data_defs \n", "import mimic\n", "from sklearn.pipeline import Pipeline\n", "import logger\n", "import pandas as pd\n", "from constants import column_names\n", "import utils\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "code", "execution_count": 153, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(utils)\n", "reload(logger)\n", "reload(mimic)\n", "def transform_extracted_labels(labels,hdf5_fname):\n", " \n", " pipeline = Pipeline([\n", " ('clean',mimic.clean_extract()),\n", " ('unstack',mimic.unstacker()),\n", " ('clean_uom',mimic.clean_uom())\n", " ])\n", " \n", " for label in labels:\n", " logger.log('Opening {}'.format(label.upper()))\n", " df = utils.open_df(hdf5_fname,'extract/{}'.format(label))\n", " logger.log('Transforming {} - {}'.format(label.upper(),df.shape))\n", " df_transformed = pipeline.transform(df)\n", " # Add label guess to column index\n", " df = utils.add_same_val__index_level(df,label,'label',axis=1)\n", " display(df_transformed.head())\n", " utils.save_df(df_transformed,hdf5_fname,'transformed/{}'.format(label))\n", " del df,df_transformed\n", " logger.end_log_level()\n", " \n", " return\n", " \n", "mimic_data_sef_fname = 'config/data_definitions.xlsx'\n", "hdf5_fname = 'data/mimic_data'\n", "\n", "#get all labels\n", "data_dict = icu_data_defs.data_dictionary(mimic_data_sef_fname)\n", "simple_data = data_dict.get_panel_defintions(12) #12 is \"simple data\"\n", "labels = simple_data.label.unique().tolist()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-02 05:11:14) Opening HEART RATE\n", "(2017-06-02 05:11:19) DONE (5.0s)\n", "(2017-06-02 05:11:19) Transforming HEART RATE - (7952939, 5)\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtransform_extracted_labels\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhdf5_fname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[1;32m\u001b[0m in \u001b[0;36mtransform_extracted_labels\u001b[1;34m(labels, hdf5_fname)\u001b[0m\n\u001b[0;32m 14\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen_df\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhdf5_fname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'extract/{}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 15\u001b[0m \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Transforming {} - {}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m \u001b[0mdf_transformed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpipeline\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 17\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_transformed\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_df\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_transformed\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhdf5_fname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'transformed/{}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\sklearn\\pipeline.pyc\u001b[0m in \u001b[0;36m_transform\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m 446\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtransform\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 447\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mtransform\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 448\u001b[1;33m \u001b[0mXt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 449\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mXt\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 450\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\icu_ml_project\\v5\\mimic.pyc\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, df)\u001b[0m\n\u001b[0;32m 189\u001b[0m \u001b[0mFORMAT\u001b[0m \u001b[0mpre\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0munstack\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 190\u001b[0m \"\"\"\n\u001b[1;32m--> 191\u001b[1;33m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_replace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnan\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 192\u001b[0m \u001b[1;31m#drop NAN record_id, timestamps, or value\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 193\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msubset\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcolumn_names\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcolumn_names\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDATETIME\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcolumn_names\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mVALUE\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'any'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\core\\generic.pyc\u001b[0m in \u001b[0;36mreplace\u001b[1;34m(self, to_replace, value, inplace, limit, regex, method, axis)\u001b[0m\n\u001b[0;32m 3538\u001b[0m new_data = self._data.replace(to_replace=to_replace,\n\u001b[0;32m 3539\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3540\u001b[1;33m regex=regex)\n\u001b[0m\u001b[0;32m 3541\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3542\u001b[0m msg = ('Invalid \"to_replace\" type: '\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\core\\internals.pyc\u001b[0m in \u001b[0;36mreplace\u001b[1;34m(self, **kwargs)\u001b[0m\n\u001b[0;32m 3172\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3173\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mreplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3174\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'replace'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3175\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3176\u001b[0m def replace_list(self, src_list, dest_list, inplace=False, regex=False,\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\core\\internals.pyc\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, f, axes, filter, do_integrity_check, consolidate, **kwargs)\u001b[0m\n\u001b[0;32m 3033\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3034\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'mgr'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3035\u001b[1;33m \u001b[0mapplied\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3036\u001b[0m \u001b[0mresult_blocks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_extend_blocks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mapplied\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult_blocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3037\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\core\\internals.pyc\u001b[0m in \u001b[0;36mreplace\u001b[1;34m(self, to_replace, value, inplace, filter, regex, convert, mgr)\u001b[0m\n\u001b[0;32m 1926\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1927\u001b[0m \u001b[0mfilter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfilter\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mregex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mregex\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1928\u001b[1;33m convert=convert, mgr=mgr)\n\u001b[0m\u001b[0;32m 1929\u001b[0m \u001b[1;32melif\u001b[0m \u001b[0mboth_lists\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1930\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mto_rep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_replace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\core\\internals.pyc\u001b[0m in \u001b[0;36mreplace\u001b[1;34m(self, to_replace, value, inplace, filter, regex, convert, mgr)\u001b[0m\n\u001b[0;32m 623\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 624\u001b[0m \u001b[0moriginal_to_replace\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_replace\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 625\u001b[1;33m \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0misnull\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 626\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 627\u001b[0m \u001b[1;31m# try to replace, if we raise an error, convert to ObjectBlock and\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\types\\missing.pyc\u001b[0m in \u001b[0;36misnull\u001b[1;34m(obj)\u001b[0m\n\u001b[0;32m 43\u001b[0m \u001b[0mpandas\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnotnull\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mboolean\u001b[0m \u001b[0minverse\u001b[0m \u001b[0mof\u001b[0m \u001b[0mpandas\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 44\u001b[0m \"\"\"\n\u001b[1;32m---> 45\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0m_isnull\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 46\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 47\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\types\\missing.pyc\u001b[0m in \u001b[0;36m_isnull_new\u001b[1;34m(obj)\u001b[0m\n\u001b[0;32m 53\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"isnull is not defined for MultiIndex\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 54\u001b[0m \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mABCSeries\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mABCIndexClass\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 55\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0m_isnull_ndarraylike\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 56\u001b[0m \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mABCGeneric\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 57\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_constructor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0misnull\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\types\\missing.pyc\u001b[0m in \u001b[0;36m_isnull_ndarraylike\u001b[1;34m(obj)\u001b[0m\n\u001b[0;32m 139\u001b[0m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbool\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 140\u001b[0m \u001b[0mvec\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misnullobj\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 141\u001b[1;33m \u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvec\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 142\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 143\u001b[0m \u001b[1;32melif\u001b[0m \u001b[0mneeds_i8_conversion\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "transform_extracted_labels(labels,hdf5_fname)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "store = pd.HDFStore(hdf5_fname)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "\n", "File path: data/mimic_data\n", "/extract/blood pressure diastolic frame (shape->[6371249,5])\n", "/extract/blood pressure mean frame (shape->[2536271,5])\n", "/extract/blood pressure systolic frame (shape->[6374824,5])\n", "/extract/glasgow coma scale eye opening frame (shape->[956672,5]) \n", "/extract/glasgow coma scale motor frame (shape->[952565,5]) \n", "/extract/glasgow coma scale verbal frame (shape->[954700,5]) \n", "/extract/heart rate frame (shape->[7952939,5])\n", "/extract/hemoglobin frame (shape->[1167921,5])\n", "/extract/lactate frame (shape->[393608,5]) \n", "/extract/lactated ringers frame (shape->[504306,5]) \n", "/extract/norepinephrine frame (shape->[1136938,5])\n", "/extract/normal saline frame (shape->[817373,5]) \n", "/extract/output urine frame (shape->[3644639,5])\n", "/extract/oxygen saturation pulse oximetry frame (shape->[6099827,5])\n", "/extract/respiratory rate frame (shape->[7810019,5])\n", "/extract/temperature body frame (shape->[1751447,5])\n", "/extract/vasopressin frame (shape->[339184,5]) \n", "/extract/weight body frame (shape->[95425,5]) \n", "/transformed/blood pressure diastolic frame \n", "/transformed/blood pressure mean frame \n", "/transformed/blood pressure systolic frame \n", "/transformed/glasgow coma scale eye opening frame \n", "/transformed/glasgow coma scale motor frame \n", "/transformed/glasgow coma scale verbal frame \n", "/transformed/heart rate frame \n", "/transformed/hemoglobin frame \n", "/transformed/lactate frame \n", "/transformed/lactated ringers frame \n", "/transformed/norepinephrine frame \n", "/transformed/normal saline frame \n", "/transformed/output urine frame \n", "/transformed/oxygen saturation pulse oximetry frame \n", "/transformed/respiratory rate frame \n", "/transformed/temperature body frame \n", "/transformed/vasopressin frame \n", "/transformed/weight body frame " ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "store" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": true }, "outputs": [], "source": [ "store.close()" ] }, { "cell_type": "code", "execution_count": 116, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 116, "metadata": {}, "output_type": "execute_result" } ], "source": [ "reload(utils)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Cleaners" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Design decision: If a given column has both NUMERIC CATEGORICAL and QUANTITATIVE data, all NUMERIC CATEGORICAL data will actually be treated like QUANTITATIVE data, unfortunately." ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import transformers\n", "import utils\n", "from sklearn.pipeline import Pipeline\n", "from units import MedicalUreg\n", "import icu_data_defs\n", "import units\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(transformers)\n", "reload(utils)\n", "mimic_data_sef_fname = 'config/data_definitions.xlsx'\n", "hdf5_fname = 'data/mimic_data'\n", "medical_units = 'config/medical_units.txt'\n", "\n", "#get all labels\n", "data_dict = icu_data_defs.data_dictionary(mimic_data_sef_fname)\n", "label = 'lactate'\n", "\n", "standard_pipeline = Pipeline([\n", " ('aggregate_same_datetime',transformers.same_index_aggregator(agg_func=lambda x:x.iloc[0])),\n", " ('split_dtype',transformers.split_dtype()),\n", " ('format_columns',transformers.format_columns(data_dict,MedicalUreg(medical_units))),\n", " ('drop_small_columns',transformers.remove_small_columns(threshold=50))\n", " ])" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "transformers.py:103: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", " df_out.dropna(how='all',inplace=True,axis=1)\n" ] }, { "data": { "text/plain": [ "(113L, 0, '0.0%')" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = utils.open_df('data/mimic_data','transformed/{}'.format(label))\n", "df_cleaned = standard_pipeline.transform(df)\n", "utils.data_loss(df,df_cleaned)" ] }, { "cell_type": "code", "execution_count": 216, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labellactate
statusknownunknown
variable_typeqnqnnom
unitsmmol/Lno_unitsno_units
description50813225668818153150813(mmol/L)818
iddatetime
1000012117-09-11 09:32:001.9NaNNaNNaNNaNNaN
1000032150-04-17 19:12:001.11.1NaNNaNNaNNaN
1000062108-04-08 10:58:004.5NaN4.5NaNNaNNaN
1000072145-03-31 00:44:003.1NaNNaNNaNNaNNaN
2145-04-02 14:10:001.9NaNNaNNaNNaNNaN
1000092162-05-17 13:19:001.11.1NaNNaNNaNNaN
2162-05-17 17:14:001.51.5NaNNaNNaNNaN
1000102109-12-10 10:25:000.6NaNNaNNaNNaNNaN
2109-12-10 12:11:000.9NaNNaNNaNNaNNaN
2109-12-10 13:05:001.0NaNNaNNaNNaNNaN
2109-12-10 13:58:000.8NaNNaNNaNNaNNaN
1000112177-08-29 04:44:003.8NaNNaNNaNNaNNaN
2177-08-29 06:55:002.32.3NaNNaNNaNNaN
1000122177-03-14 07:38:002.3NaNNaNNaNNaNNaN
2177-03-14 11:42:002.52.5NaNNaNNaNNaN
2177-03-15 08:05:002.12.1NaNNaNNaNNaN
2177-03-15 14:01:002.62.6NaNNaNNaNNaN
2177-03-15 21:42:001.81.8NaNNaNNaNNaN
1000162188-05-24 12:00:002.0NaNNaNNaNNaNNaN
1000172103-03-11 05:10:001.1NaN1.1NaNNaNNaN
1000182176-08-29 15:29:001.3NaNNaNNaNNaNNaN
2176-08-30 09:23:000.90.9NaNNaNNaNNaN
2176-08-30 10:19:001.01.0NaNNaNNaNNaN
2176-08-30 11:29:000.90.9NaNNaNNaNNaN
2176-08-30 12:40:001.11.1NaNNaNNaNNaN
1000202142-11-30 21:54:001.1NaNNaNNaNNaNNaN
2142-12-03 00:17:001.0NaNNaNNaNNaNNaN
1000242170-09-19 10:25:001.41.4NaNNaNNaNNaN
2170-09-19 16:33:002.62.6NaNNaNNaNNaN
2170-09-20 02:04:003.23.2NaNNaNNaNNaN
........................
1999762182-02-14 11:15:000.8NaN0.8NaNNaNNaN
2182-02-16 03:57:000.8NaN0.8NaNNaNNaN
2182-02-19 03:59:000.8NaN0.8NaNNaNNaN
2182-02-20 03:31:000.7NaN0.7NaNNaNNaN
2182-02-21 04:55:000.9NaN0.9NaNNaNNaN
1999792182-02-06 09:17:001.0NaNNaNNaNNaNNaN
2182-02-06 14:16:003.83.8NaNNaNNaNNaN
1999812110-09-24 16:34:001.1NaN1.11.1NaNNaN
2110-09-24 20:09:001.0NaN1.01.0NaNNaN
2110-09-25 06:10:001.0NaN1.01.0NaNNaN
1999872175-05-19 16:30:002.3NaNNaNNaNNaNNaN
1999882169-01-24 12:48:001.0NaNNaNNaNNaNNaN
2169-02-07 01:35:001.6NaN1.6NaNNaNNaN
2169-02-07 11:18:001.0NaN1.0NaNNaNNaN
2169-02-07 16:43:001.0NaN1.0NaNNaNNaN
2169-02-07 22:35:001.1NaN1.1NaNNaNNaN
2169-02-10 05:33:001.4NaNNaNNaNNaNNaN
1999932161-11-12 23:14:000.9NaN0.9NaNNaNNaN
2161-11-13 03:46:000.9NaN0.9NaNNaNNaN
1999942188-07-07 21:23:001.0NaN1.0NaNNaNNaN
2188-07-08 03:09:000.7NaN0.7NaNNaNNaN
2188-07-08 04:13:000.6NaN0.6NaNNaNNaN
2188-07-08 06:20:000.7NaN0.7NaNNaNNaN
1999982119-02-20 10:52:001.11.11.11.1NaNNaN
2119-02-20 12:36:001.91.91.91.9NaNNaN
2119-02-20 13:33:002.02.02.02.0NaNNaN
2119-02-20 13:59:002.62.62.62.6NaNNaN
2119-02-20 20:43:001.31.31.31.3NaNNaN
1999992136-04-04 20:55:001.9NaNNaNNaNNaNNaN
2136-04-06 15:29:001.8NaNNaNNaNNaNNaN
\n", "

177451 rows × 6 columns

\n", "
" ], "text/plain": [ "label lactate \n", "status known unknown \n", "variable_type qn qn nom \n", "units mmol/L no_units no_units \n", "description 50813 225668 818 1531 50813(mmol/L) 818\n", "id datetime \n", "100001 2117-09-11 09:32:00 1.9 NaN NaN NaN NaN NaN\n", "100003 2150-04-17 19:12:00 1.1 1.1 NaN NaN NaN NaN\n", "100006 2108-04-08 10:58:00 4.5 NaN 4.5 NaN NaN NaN\n", "100007 2145-03-31 00:44:00 3.1 NaN NaN NaN NaN NaN\n", " 2145-04-02 14:10:00 1.9 NaN NaN NaN NaN NaN\n", "100009 2162-05-17 13:19:00 1.1 1.1 NaN NaN NaN NaN\n", " 2162-05-17 17:14:00 1.5 1.5 NaN NaN NaN NaN\n", "100010 2109-12-10 10:25:00 0.6 NaN NaN NaN NaN NaN\n", " 2109-12-10 12:11:00 0.9 NaN NaN NaN NaN NaN\n", " 2109-12-10 13:05:00 1.0 NaN NaN NaN NaN NaN\n", " 2109-12-10 13:58:00 0.8 NaN NaN NaN NaN NaN\n", "100011 2177-08-29 04:44:00 3.8 NaN NaN NaN NaN NaN\n", " 2177-08-29 06:55:00 2.3 2.3 NaN NaN NaN NaN\n", "100012 2177-03-14 07:38:00 2.3 NaN NaN NaN NaN NaN\n", " 2177-03-14 11:42:00 2.5 2.5 NaN NaN NaN NaN\n", " 2177-03-15 08:05:00 2.1 2.1 NaN NaN NaN NaN\n", " 2177-03-15 14:01:00 2.6 2.6 NaN NaN NaN NaN\n", " 2177-03-15 21:42:00 1.8 1.8 NaN NaN NaN NaN\n", "100016 2188-05-24 12:00:00 2.0 NaN NaN NaN NaN NaN\n", "100017 2103-03-11 05:10:00 1.1 NaN 1.1 NaN NaN NaN\n", "100018 2176-08-29 15:29:00 1.3 NaN NaN NaN NaN NaN\n", " 2176-08-30 09:23:00 0.9 0.9 NaN NaN NaN NaN\n", " 2176-08-30 10:19:00 1.0 1.0 NaN NaN NaN NaN\n", " 2176-08-30 11:29:00 0.9 0.9 NaN NaN NaN NaN\n", " 2176-08-30 12:40:00 1.1 1.1 NaN NaN NaN NaN\n", "100020 2142-11-30 21:54:00 1.1 NaN NaN NaN NaN NaN\n", " 2142-12-03 00:17:00 1.0 NaN NaN NaN NaN NaN\n", "100024 2170-09-19 10:25:00 1.4 1.4 NaN NaN NaN NaN\n", " 2170-09-19 16:33:00 2.6 2.6 NaN NaN NaN NaN\n", " 2170-09-20 02:04:00 3.2 3.2 NaN NaN NaN NaN\n", "... ... ... ... ... ... ...\n", "199976 2182-02-14 11:15:00 0.8 NaN 0.8 NaN NaN NaN\n", " 2182-02-16 03:57:00 0.8 NaN 0.8 NaN NaN NaN\n", " 2182-02-19 03:59:00 0.8 NaN 0.8 NaN NaN NaN\n", " 2182-02-20 03:31:00 0.7 NaN 0.7 NaN NaN NaN\n", " 2182-02-21 04:55:00 0.9 NaN 0.9 NaN NaN NaN\n", "199979 2182-02-06 09:17:00 1.0 NaN NaN NaN NaN NaN\n", " 2182-02-06 14:16:00 3.8 3.8 NaN NaN NaN NaN\n", "199981 2110-09-24 16:34:00 1.1 NaN 1.1 1.1 NaN NaN\n", " 2110-09-24 20:09:00 1.0 NaN 1.0 1.0 NaN NaN\n", " 2110-09-25 06:10:00 1.0 NaN 1.0 1.0 NaN NaN\n", "199987 2175-05-19 16:30:00 2.3 NaN NaN NaN NaN NaN\n", "199988 2169-01-24 12:48:00 1.0 NaN NaN NaN NaN NaN\n", " 2169-02-07 01:35:00 1.6 NaN 1.6 NaN NaN NaN\n", " 2169-02-07 11:18:00 1.0 NaN 1.0 NaN NaN NaN\n", " 2169-02-07 16:43:00 1.0 NaN 1.0 NaN NaN NaN\n", " 2169-02-07 22:35:00 1.1 NaN 1.1 NaN NaN NaN\n", " 2169-02-10 05:33:00 1.4 NaN NaN NaN NaN NaN\n", "199993 2161-11-12 23:14:00 0.9 NaN 0.9 NaN NaN NaN\n", " 2161-11-13 03:46:00 0.9 NaN 0.9 NaN NaN NaN\n", "199994 2188-07-07 21:23:00 1.0 NaN 1.0 NaN NaN NaN\n", " 2188-07-08 03:09:00 0.7 NaN 0.7 NaN NaN NaN\n", " 2188-07-08 04:13:00 0.6 NaN 0.6 NaN NaN NaN\n", " 2188-07-08 06:20:00 0.7 NaN 0.7 NaN NaN NaN\n", "199998 2119-02-20 10:52:00 1.1 1.1 1.1 1.1 NaN NaN\n", " 2119-02-20 12:36:00 1.9 1.9 1.9 1.9 NaN NaN\n", " 2119-02-20 13:33:00 2.0 2.0 2.0 2.0 NaN NaN\n", " 2119-02-20 13:59:00 2.6 2.6 2.6 2.6 NaN NaN\n", " 2119-02-20 20:43:00 1.3 1.3 1.3 1.3 NaN NaN\n", "199999 2136-04-04 20:55:00 1.9 NaN NaN NaN NaN NaN\n", " 2136-04-06 15:29:00 1.8 NaN NaN NaN NaN NaN\n", "\n", "[177451 rows x 6 columns]" ] }, "execution_count": 216, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_cleaned" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df_invalid = df_cleaned.loc[:,df_cleaned.columns.get_level_values('status') == 'unknown']" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df_invalid.columns.set_labels([0]*df_invalid.columns.size,level='status',inplace=True)\n", "df_invalid.columns.set_levels(['known'],level='status',inplace=True)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labellactate
statusknownunknown
variable_typeqnnomqnqnnom
unitsno_unitsno_unitsmmol/Lno_unitsno_units
description818153150813(mmol/L)81850813225668818153150813(mmol/L)818
iddatetime
1000012117-09-11 09:32:00NaNNaNNaNNaN1.9NaNNaNNaNNaNNaN
1000032150-04-17 19:12:00NaNNaNNaNNaN1.11.1NaNNaNNaNNaN
1000062108-04-08 10:58:004.5NaNNaNNaN4.5NaN4.5NaNNaNNaN
1000072145-03-31 00:44:00NaNNaNNaNNaN3.1NaNNaNNaNNaNNaN
2145-04-02 14:10:00NaNNaNNaNNaN1.9NaNNaNNaNNaNNaN
1000092162-05-17 13:19:00NaNNaNNaNNaN1.11.1NaNNaNNaNNaN
2162-05-17 17:14:00NaNNaNNaNNaN1.51.5NaNNaNNaNNaN
1000102109-12-10 10:25:00NaNNaNNaNNaN0.6NaNNaNNaNNaNNaN
2109-12-10 12:11:00NaNNaNNaNNaN0.9NaNNaNNaNNaNNaN
2109-12-10 13:05:00NaNNaNNaNNaN1.0NaNNaNNaNNaNNaN
2109-12-10 13:58:00NaNNaNNaNNaN0.8NaNNaNNaNNaNNaN
1000112177-08-29 04:44:00NaNNaNNaNNaN3.8NaNNaNNaNNaNNaN
2177-08-29 06:55:00NaNNaNNaNNaN2.32.3NaNNaNNaNNaN
1000122177-03-14 07:38:00NaNNaNNaNNaN2.3NaNNaNNaNNaNNaN
2177-03-14 11:42:00NaNNaNNaNNaN2.52.5NaNNaNNaNNaN
2177-03-15 08:05:00NaNNaNNaNNaN2.12.1NaNNaNNaNNaN
2177-03-15 14:01:00NaNNaNNaNNaN2.62.6NaNNaNNaNNaN
2177-03-15 21:42:00NaNNaNNaNNaN1.81.8NaNNaNNaNNaN
1000162188-05-24 12:00:00NaNNaNNaNNaN2.0NaNNaNNaNNaNNaN
1000172103-03-11 05:10:001.1NaNNaNNaN1.1NaN1.1NaNNaNNaN
1000182176-08-29 15:29:00NaNNaNNaNNaN1.3NaNNaNNaNNaNNaN
2176-08-30 09:23:00NaNNaNNaNNaN0.90.9NaNNaNNaNNaN
2176-08-30 10:19:00NaNNaNNaNNaN1.01.0NaNNaNNaNNaN
2176-08-30 11:29:00NaNNaNNaNNaN0.90.9NaNNaNNaNNaN
2176-08-30 12:40:00NaNNaNNaNNaN1.11.1NaNNaNNaNNaN
1000202142-11-30 21:54:00NaNNaNNaNNaN1.1NaNNaNNaNNaNNaN
2142-12-03 00:17:00NaNNaNNaNNaN1.0NaNNaNNaNNaNNaN
1000242170-09-19 10:25:00NaNNaNNaNNaN1.41.4NaNNaNNaNNaN
2170-09-19 16:33:00NaNNaNNaNNaN2.62.6NaNNaNNaNNaN
2170-09-20 02:04:00NaNNaNNaNNaN3.23.2NaNNaNNaNNaN
....................................
1999762182-02-14 11:15:000.8NaNNaNNaN0.8NaN0.8NaNNaNNaN
2182-02-16 03:57:000.8NaNNaNNaN0.8NaN0.8NaNNaNNaN
2182-02-19 03:59:000.8NaNNaNNaN0.8NaN0.8NaNNaNNaN
2182-02-20 03:31:000.7NaNNaNNaN0.7NaN0.7NaNNaNNaN
2182-02-21 04:55:000.9NaNNaNNaN0.9NaN0.9NaNNaNNaN
1999792182-02-06 09:17:00NaNNaNNaNNaN1.0NaNNaNNaNNaNNaN
2182-02-06 14:16:00NaNNaNNaNNaN3.83.8NaNNaNNaNNaN
1999812110-09-24 16:34:001.11.1NaNNaN1.1NaN1.11.1NaNNaN
2110-09-24 20:09:001.01.0NaNNaN1.0NaN1.01.0NaNNaN
2110-09-25 06:10:001.01.0NaNNaN1.0NaN1.01.0NaNNaN
1999872175-05-19 16:30:00NaNNaNNaNNaN2.3NaNNaNNaNNaNNaN
1999882169-01-24 12:48:00NaNNaNNaNNaN1.0NaNNaNNaNNaNNaN
2169-02-07 01:35:001.6NaNNaNNaN1.6NaN1.6NaNNaNNaN
2169-02-07 11:18:001.0NaNNaNNaN1.0NaN1.0NaNNaNNaN
2169-02-07 16:43:001.0NaNNaNNaN1.0NaN1.0NaNNaNNaN
2169-02-07 22:35:001.1NaNNaNNaN1.1NaN1.1NaNNaNNaN
2169-02-10 05:33:00NaNNaNNaNNaN1.4NaNNaNNaNNaNNaN
1999932161-11-12 23:14:000.9NaNNaNNaN0.9NaN0.9NaNNaNNaN
2161-11-13 03:46:000.9NaNNaNNaN0.9NaN0.9NaNNaNNaN
1999942188-07-07 21:23:001.0NaNNaNNaN1.0NaN1.0NaNNaNNaN
2188-07-08 03:09:000.7NaNNaNNaN0.7NaN0.7NaNNaNNaN
2188-07-08 04:13:000.6NaNNaNNaN0.6NaN0.6NaNNaNNaN
2188-07-08 06:20:000.7NaNNaNNaN0.7NaN0.7NaNNaNNaN
1999982119-02-20 10:52:001.11.1NaNNaN1.11.11.11.1NaNNaN
2119-02-20 12:36:001.91.9NaNNaN1.91.91.91.9NaNNaN
2119-02-20 13:33:002.02.0NaNNaN2.02.02.02.0NaNNaN
2119-02-20 13:59:002.62.6NaNNaN2.62.62.62.6NaNNaN
2119-02-20 20:43:001.31.3NaNNaN1.31.31.31.3NaNNaN
1999992136-04-04 20:55:00NaNNaNNaNNaN1.9NaNNaNNaNNaNNaN
2136-04-06 15:29:00NaNNaNNaNNaN1.8NaNNaNNaNNaNNaN
\n", "

177451 rows × 10 columns

\n", "
" ], "text/plain": [ "label lactate \\\n", "status known \n", "variable_type qn nom qn \n", "units no_units no_units mmol/L \n", "description 818 1531 50813(mmol/L) 818 50813 225668 \n", "id datetime \n", "100001 2117-09-11 09:32:00 NaN NaN NaN NaN 1.9 NaN \n", "100003 2150-04-17 19:12:00 NaN NaN NaN NaN 1.1 1.1 \n", "100006 2108-04-08 10:58:00 4.5 NaN NaN NaN 4.5 NaN \n", "100007 2145-03-31 00:44:00 NaN NaN NaN NaN 3.1 NaN \n", " 2145-04-02 14:10:00 NaN NaN NaN NaN 1.9 NaN \n", "100009 2162-05-17 13:19:00 NaN NaN NaN NaN 1.1 1.1 \n", " 2162-05-17 17:14:00 NaN NaN NaN NaN 1.5 1.5 \n", "100010 2109-12-10 10:25:00 NaN NaN NaN NaN 0.6 NaN \n", " 2109-12-10 12:11:00 NaN NaN NaN NaN 0.9 NaN \n", " 2109-12-10 13:05:00 NaN NaN NaN NaN 1.0 NaN \n", " 2109-12-10 13:58:00 NaN NaN NaN NaN 0.8 NaN \n", "100011 2177-08-29 04:44:00 NaN NaN NaN NaN 3.8 NaN \n", " 2177-08-29 06:55:00 NaN NaN NaN NaN 2.3 2.3 \n", "100012 2177-03-14 07:38:00 NaN NaN NaN NaN 2.3 NaN \n", " 2177-03-14 11:42:00 NaN NaN NaN NaN 2.5 2.5 \n", " 2177-03-15 08:05:00 NaN NaN NaN NaN 2.1 2.1 \n", " 2177-03-15 14:01:00 NaN NaN NaN NaN 2.6 2.6 \n", " 2177-03-15 21:42:00 NaN NaN NaN NaN 1.8 1.8 \n", "100016 2188-05-24 12:00:00 NaN NaN NaN NaN 2.0 NaN \n", "100017 2103-03-11 05:10:00 1.1 NaN NaN NaN 1.1 NaN \n", "100018 2176-08-29 15:29:00 NaN NaN NaN NaN 1.3 NaN \n", " 2176-08-30 09:23:00 NaN NaN NaN NaN 0.9 0.9 \n", " 2176-08-30 10:19:00 NaN NaN NaN NaN 1.0 1.0 \n", " 2176-08-30 11:29:00 NaN NaN NaN NaN 0.9 0.9 \n", " 2176-08-30 12:40:00 NaN NaN NaN NaN 1.1 1.1 \n", "100020 2142-11-30 21:54:00 NaN NaN NaN NaN 1.1 NaN \n", " 2142-12-03 00:17:00 NaN NaN NaN NaN 1.0 NaN \n", "100024 2170-09-19 10:25:00 NaN NaN NaN NaN 1.4 1.4 \n", " 2170-09-19 16:33:00 NaN NaN NaN NaN 2.6 2.6 \n", " 2170-09-20 02:04:00 NaN NaN NaN NaN 3.2 3.2 \n", "... ... ... ... ... ... ... \n", "199976 2182-02-14 11:15:00 0.8 NaN NaN NaN 0.8 NaN \n", " 2182-02-16 03:57:00 0.8 NaN NaN NaN 0.8 NaN \n", " 2182-02-19 03:59:00 0.8 NaN NaN NaN 0.8 NaN \n", " 2182-02-20 03:31:00 0.7 NaN NaN NaN 0.7 NaN \n", " 2182-02-21 04:55:00 0.9 NaN NaN NaN 0.9 NaN \n", "199979 2182-02-06 09:17:00 NaN NaN NaN NaN 1.0 NaN \n", " 2182-02-06 14:16:00 NaN NaN NaN NaN 3.8 3.8 \n", "199981 2110-09-24 16:34:00 1.1 1.1 NaN NaN 1.1 NaN \n", " 2110-09-24 20:09:00 1.0 1.0 NaN NaN 1.0 NaN \n", " 2110-09-25 06:10:00 1.0 1.0 NaN NaN 1.0 NaN \n", "199987 2175-05-19 16:30:00 NaN NaN NaN NaN 2.3 NaN \n", "199988 2169-01-24 12:48:00 NaN NaN NaN NaN 1.0 NaN \n", " 2169-02-07 01:35:00 1.6 NaN NaN NaN 1.6 NaN \n", " 2169-02-07 11:18:00 1.0 NaN NaN NaN 1.0 NaN \n", " 2169-02-07 16:43:00 1.0 NaN NaN NaN 1.0 NaN \n", " 2169-02-07 22:35:00 1.1 NaN NaN NaN 1.1 NaN \n", " 2169-02-10 05:33:00 NaN NaN NaN NaN 1.4 NaN \n", "199993 2161-11-12 23:14:00 0.9 NaN NaN NaN 0.9 NaN \n", " 2161-11-13 03:46:00 0.9 NaN NaN NaN 0.9 NaN \n", "199994 2188-07-07 21:23:00 1.0 NaN NaN NaN 1.0 NaN \n", " 2188-07-08 03:09:00 0.7 NaN NaN NaN 0.7 NaN \n", " 2188-07-08 04:13:00 0.6 NaN NaN NaN 0.6 NaN \n", " 2188-07-08 06:20:00 0.7 NaN NaN NaN 0.7 NaN \n", "199998 2119-02-20 10:52:00 1.1 1.1 NaN NaN 1.1 1.1 \n", " 2119-02-20 12:36:00 1.9 1.9 NaN NaN 1.9 1.9 \n", " 2119-02-20 13:33:00 2.0 2.0 NaN NaN 2.0 2.0 \n", " 2119-02-20 13:59:00 2.6 2.6 NaN NaN 2.6 2.6 \n", " 2119-02-20 20:43:00 1.3 1.3 NaN NaN 1.3 1.3 \n", "199999 2136-04-04 20:55:00 NaN NaN NaN NaN 1.9 NaN \n", " 2136-04-06 15:29:00 NaN NaN NaN NaN 1.8 NaN \n", "\n", "label \n", "status unknown \n", "variable_type qn nom \n", "units no_units no_units \n", "description 818 1531 50813(mmol/L) 818 \n", "id datetime \n", "100001 2117-09-11 09:32:00 NaN NaN NaN NaN \n", "100003 2150-04-17 19:12:00 NaN NaN NaN NaN \n", "100006 2108-04-08 10:58:00 4.5 NaN NaN NaN \n", "100007 2145-03-31 00:44:00 NaN NaN NaN NaN \n", " 2145-04-02 14:10:00 NaN NaN NaN NaN \n", "100009 2162-05-17 13:19:00 NaN NaN NaN NaN \n", " 2162-05-17 17:14:00 NaN NaN NaN NaN \n", "100010 2109-12-10 10:25:00 NaN NaN NaN NaN \n", " 2109-12-10 12:11:00 NaN NaN NaN NaN \n", " 2109-12-10 13:05:00 NaN NaN NaN NaN \n", " 2109-12-10 13:58:00 NaN NaN NaN NaN \n", "100011 2177-08-29 04:44:00 NaN NaN NaN NaN \n", " 2177-08-29 06:55:00 NaN NaN NaN NaN \n", "100012 2177-03-14 07:38:00 NaN NaN NaN NaN \n", " 2177-03-14 11:42:00 NaN NaN NaN NaN \n", " 2177-03-15 08:05:00 NaN NaN NaN NaN \n", " 2177-03-15 14:01:00 NaN NaN NaN NaN \n", " 2177-03-15 21:42:00 NaN NaN NaN NaN \n", "100016 2188-05-24 12:00:00 NaN NaN NaN NaN \n", "100017 2103-03-11 05:10:00 1.1 NaN NaN NaN \n", "100018 2176-08-29 15:29:00 NaN NaN NaN NaN \n", " 2176-08-30 09:23:00 NaN NaN NaN NaN \n", " 2176-08-30 10:19:00 NaN NaN NaN NaN \n", " 2176-08-30 11:29:00 NaN NaN NaN NaN \n", " 2176-08-30 12:40:00 NaN NaN NaN NaN \n", "100020 2142-11-30 21:54:00 NaN NaN NaN NaN \n", " 2142-12-03 00:17:00 NaN NaN NaN NaN \n", "100024 2170-09-19 10:25:00 NaN NaN NaN NaN \n", " 2170-09-19 16:33:00 NaN NaN NaN NaN \n", " 2170-09-20 02:04:00 NaN NaN NaN NaN \n", "... ... ... ... ... \n", "199976 2182-02-14 11:15:00 0.8 NaN NaN NaN \n", " 2182-02-16 03:57:00 0.8 NaN NaN NaN \n", " 2182-02-19 03:59:00 0.8 NaN NaN NaN \n", " 2182-02-20 03:31:00 0.7 NaN NaN NaN \n", " 2182-02-21 04:55:00 0.9 NaN NaN NaN \n", "199979 2182-02-06 09:17:00 NaN NaN NaN NaN \n", " 2182-02-06 14:16:00 NaN NaN NaN NaN \n", "199981 2110-09-24 16:34:00 1.1 1.1 NaN NaN \n", " 2110-09-24 20:09:00 1.0 1.0 NaN NaN \n", " 2110-09-25 06:10:00 1.0 1.0 NaN NaN \n", "199987 2175-05-19 16:30:00 NaN NaN NaN NaN \n", "199988 2169-01-24 12:48:00 NaN NaN NaN NaN \n", " 2169-02-07 01:35:00 1.6 NaN NaN NaN \n", " 2169-02-07 11:18:00 1.0 NaN NaN NaN \n", " 2169-02-07 16:43:00 1.0 NaN NaN NaN \n", " 2169-02-07 22:35:00 1.1 NaN NaN NaN \n", " 2169-02-10 05:33:00 NaN NaN NaN NaN \n", "199993 2161-11-12 23:14:00 0.9 NaN NaN NaN \n", " 2161-11-13 03:46:00 0.9 NaN NaN NaN \n", "199994 2188-07-07 21:23:00 1.0 NaN NaN NaN \n", " 2188-07-08 03:09:00 0.7 NaN NaN NaN \n", " 2188-07-08 04:13:00 0.6 NaN NaN NaN \n", " 2188-07-08 06:20:00 0.7 NaN NaN NaN \n", "199998 2119-02-20 10:52:00 1.1 1.1 NaN NaN \n", " 2119-02-20 12:36:00 1.9 1.9 NaN NaN \n", " 2119-02-20 13:33:00 2.0 2.0 NaN NaN \n", " 2119-02-20 13:59:00 2.6 2.6 NaN NaN \n", " 2119-02-20 20:43:00 1.3 1.3 NaN NaN \n", "199999 2136-04-04 20:55:00 NaN NaN NaN NaN \n", " 2136-04-06 15:29:00 NaN NaN NaN NaN \n", "\n", "[177451 rows x 10 columns]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_invalid.join(df_cleaned, how='outer')" ] }, { "cell_type": "code", "execution_count": 71, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df = utils.open_df('data/mimic_data','transformed/{}'.format(label))\n", "df_cleaned = pipeline.transform(df)\n", "utils.data_loss(df,df_cleaned)" ] }, { "cell_type": "code", "execution_count": 72, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(113L, 0, '0.0%')" ] }, "execution_count": 72, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Secondary/optional transformation" ] }, { "cell_type": "code", "execution_count": 99, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload (transformers)\n", "pipeline1 = Pipeline([\n", " ('combine_like_columns',transformers.combine_like_cols()),\n", " ('quantitative_vales_only',transformers.quantitative_only()),\n", " ('known_col_only',transformers.known_col_only())\n", " ])\n", "\n", "pipeline2 = Pipeline([\n", " ('combine_like_columns',transformers.combine_like_cols()),\n", " ('max_col',transformers.max_col_only()),\n", " ])" ] }, { "cell_type": "code", "execution_count": 74, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(205349L, 20, '0.0583%')" ] }, "execution_count": 74, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_final = pipeline1.transform(df_cleaned)\n", "utils.data_loss(df,df_final)" ] }, { "cell_type": "code", "execution_count": 75, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(205349L, 20, '0.0583%')" ] }, "execution_count": 75, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_final = pipeline2.transform(df_cleaned)\n", "utils.data_loss(df,df_final)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Handle Categorical columns\n", "\n", "0. Standardize category lists\n", "1. Make sure ordinal & nominal within category list?\n", "2. Explode nominal\n", "3. Ordinal to numeric" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def make_cat_dict(label,cat_codes):\n", " df = utils.open_df(hdf5_fname, 'transformed/{}'.format(label))\n", " mimic_cats = df.iloc[:,0].value_counts().sort_index().index.astype(str).tolist()\n", " return dict(zip(mimic_cats,cat_codes))" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "category_map={}" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
val_numericval_text
category_id
01No motor response
12Extension to pain
23Flexion to pain
34Withdrawal from pain
45Localizing pain
56Obeys commands
61No eye opening
72Eye opening to pain
83Eye opening to verbal command
94Eyes open spontaneously
101No verbal response (>2 yrs); no vocal response...
112Incomprehensible sounds
123Inappropriate words
134Confused
145Oriented
\n", "
" ], "text/plain": [ " val_numeric val_text\n", "category_id \n", "0 1 No motor response \n", "1 2 Extension to pain\n", "2 3 Flexion to pain\n", "3 4 Withdrawal from pain   \n", "4 5 Localizing pain\n", "5 6 Obeys commands\n", "6 1 No eye opening\n", "7 2 Eye opening to pain\n", "8 3 Eye opening to verbal command\n", "9 4 Eyes open spontaneously\n", "10 1 No verbal response (>2 yrs); no vocal response...\n", "11 2 Incomprehensible sounds\n", "12 3 Inappropriate words\n", "13 4 Confused\n", "14 5 Oriented" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_dict.tables.categories" ] }, { "cell_type": "code", "execution_count": 62, "metadata": { "collapsed": false }, "outputs": [], "source": [ "label = data_dict.labels.GLASGOW_COMA_SCALE_MOTOR\n", "category_map[label] = make_cat_dict(label,range(0,6))\n", "label = data_dict.labels.GLASGOW_COMA_SCALE_EYE_OPENING\n", "category_map[label] = make_cat_dict(label,range(6,10))\n", "label = data_dict.labels.GLASGOW_COMA_SCALE_VERBAL\n", "category_map[label] = make_cat_dict(label,[10] + range(10,15))" ] }, { "cell_type": "code", "execution_count": 63, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "{'glasgow coma scale eye opening': {'1 No Response': 6,\n", " '2 To pain': 7,\n", " '3 To speech': 8,\n", " '4 Spontaneously': 9},\n", " 'glasgow coma scale motor': {'1 No Response': 0,\n", " '2 Abnorm extensn': 1,\n", " '3 Abnorm flexion': 2,\n", " '4 Flex-withdraws': 3,\n", " '5 Localizes Pain': 4,\n", " '6 Obeys Commands': 5},\n", " 'glasgow coma scale verbal': {'1 No Response': 10,\n", " '1.0 ET/Trach': 10,\n", " '2 Incomp sounds': 11,\n", " '3 Inapprop words': 12,\n", " '4 Confused': 13,\n", " '5 Oriented': 14}}" ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "category_map" ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "reload(utils)\n", "reload(transformers)" ] }, { "cell_type": "code", "execution_count": 58, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelglasgow coma scale verbal
statusknown
variable_typeord
unitsno_units
description723
1.0 ET/Trach444357
5 Oriented379668
4 Confused82434
1 No Response20836
2 Incomp sounds18007
3 Inapprop words5611
\n", "
" ], "text/plain": [ "label glasgow coma scale verbal\n", "status known\n", "variable_type ord\n", "units no_units\n", "description 723\n", "1.0 ET/Trach 444357\n", "5 Oriented 379668\n", "4 Confused 82434\n", "1 No Response 20836\n", "2 Incomp sounds 18007\n", "3 Inapprop words 5611" ] }, "execution_count": 58, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_gcs = utils.open_df(hdf5_fname, 'transformed/{}'.format(data_dict.labels.GLASGOW_COMA_SCALE_VERBAL))\n", "df_gcs = standard_pipeline.transform(df_gcs)\n", "df_gcs.apply(lambda x: x.value_counts())" ] }, { "cell_type": "code", "execution_count": 59, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelglasgow coma scale verbal
statusknown
variable_typeord
unitsno_units
description723
1465193
5 Oriented379668
482434
218007
35611
\n", "
" ], "text/plain": [ "label glasgow coma scale verbal\n", "status known\n", "variable_type ord\n", "units no_units\n", "description 723\n", "1 465193\n", "5 Oriented 379668\n", "4 82434\n", "2 18007\n", "3 5611" ] }, "execution_count": 59, "metadata": {}, "output_type": "execute_result" } ], "source": [ "transformer = transformers.standardize_categories(data_dict,category_map,use_numeric=True)\n", "df_gcs = transformer.transform(df_gcs)\n", "df_gcs.apply(lambda x: x.value_counts())" ] }, { "cell_type": "code", "execution_count": 60, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelglasgow coma scale verbal
statusknownunknown
variable_typeordnom
unitsno_unitsno_units
description723723
1465193.0NaN
218007.0NaN
35611.0NaN
482434.0NaN
5 OrientedNaN379668.0
\n", "
" ], "text/plain": [ "label glasgow coma scale verbal \n", "status known unknown\n", "variable_type ord nom\n", "units no_units no_units\n", "description 723 723\n", "1 465193.0 NaN\n", "2 18007.0 NaN\n", "3 5611.0 NaN\n", "4 82434.0 NaN\n", "5 Oriented NaN 379668.0" ] }, "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ "transformer2 = transformers.split_bad_categories(data_dict,use_numeric=True)\n", "transformer2.transform(df_gcs).apply(lambda x: x.value_counts())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Unified pipeline" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import icu_data_defs\n", "from sklearn.pipeline import Pipeline\n", "import transformers\n", "from units import MedicalUreg\n", "from constants import variable_type\n", "import utils\n", "import mimic\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(transformers)\n", "reload(utils)\n", "reload(mimic)\n", "hdf5_fname = 'data/mimic_data'\n", "\n", "#get all labels\n", "data_dict = icu_data_defs.data_dictionary('config/data_definitions.xlsx')\n", "ureg = MedicalUreg('config/medical_units.txt')\n", "\n", "agg_func = lambda x:x.iloc[0]\n", "var_types_to_keep = [variable_type.QUANTITATIVE,variable_type.ORDINAL]\n", "category_map = {\n", " data_dict.labels.GLASGOW_COMA_SCALE_EYE_OPENING: {\n", " '1 No Response': 6,\n", " '2 To pain': 7,\n", " '3 To speech': 8,\n", " '4 Spontaneously': 9\n", " },\n", " data_dict.labels.GLASGOW_COMA_SCALE_MOTOR: {\n", " '1 No Response': 0,\n", " '2 Abnorm extensn': 1,\n", " '3 Abnorm flexion': 2,\n", " '4 Flex-withdraws': 3,\n", " '5 Localizes Pain': 4,\n", " '6 Obeys Commands': 5\n", " },\n", " data_dict.labels.GLASGOW_COMA_SCALE_VERBAL: {\n", " '1 No Response': 10,\n", " '1.0 ET/Trach': 10,\n", " '2 Incomp sounds': 11,\n", " '3 Inapprop words': 12,\n", " '4 Confused': 13,\n", " '5 Oriented':14\n", " }\n", "}\n", "\n", "mimic_transform = Pipeline([\n", " ('clean',mimic.clean_extract()),\n", " ('unstack',mimic.unstacker()),\n", " ('add_level',transformers.add_level(None,'label',axis=1)),\n", "])\n", "\n", "standard_pipeline = Pipeline([\n", " ('drop_small_columns',transformers.remove_small_columns(threshold=5)),\n", " ('aggregate_same_datetime',transformers.same_index_aggregator(agg_func)),\n", " ('split_dtype',transformers.split_dtype()),\n", " ('standardize_columns',transformers.standardize_columns(data_dict,ureg)),\n", " ('standardize_categories',transformers.standardize_categories(data_dict,category_map)),\n", " ('split_bad_categories',transformers.split_bad_categories(data_dict))\n", " ])\n", "\n", "cleaning_pipeline = Pipeline([\n", " ('drop_small_columns',transformers.remove_small_columns(threshold=50)),\n", " ('combine_like_columns',transformers.combine_like_cols()),\n", " ('quantitative_only',transformers.filter_var_type(var_types_to_keep)),\n", " ('known_col_only',transformers.known_col_only()),\n", "# ('max_col',transformers.max_col_only()) \n", " ])" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
count817373.000000817373771272.000000817373817373.000000
uniqueNaN536890446614.0000006NaN
topNaN2159-09-30 01:00:0099.999996mL/hourNaN
freqNaN925385.000000719922NaN
firstNaN2100-06-08 04:23:00NaNNaNNaN
lastNaN2209-08-07 14:27:00NaNNaNNaN
mean150437.603989NaNNaNNaN222346.358593
std28720.645234NaNNaNNaN23242.543767
min100001.000000NaNNaNNaN4647.000000
25%125824.000000NaNNaNNaN225158.000000
50%150715.000000NaNNaNNaN225158.000000
75%175466.000000NaNNaNNaN225158.000000
max199984.000000NaNNaNNaN225158.000000
\n", "
" ], "text/plain": [ " id datetime value units \\\n", "count 817373.000000 817373 771272.000000 817373 \n", "unique NaN 536890 446614.000000 6 \n", "top NaN 2159-09-30 01:00:00 99.999996 mL/hour \n", "freq NaN 9 25385.000000 719922 \n", "first NaN 2100-06-08 04:23:00 NaN NaN \n", "last NaN 2209-08-07 14:27:00 NaN NaN \n", "mean 150437.603989 NaN NaN NaN \n", "std 28720.645234 NaN NaN NaN \n", "min 100001.000000 NaN NaN NaN \n", "25% 125824.000000 NaN NaN NaN \n", "50% 150715.000000 NaN NaN NaN \n", "75% 175466.000000 NaN NaN NaN \n", "max 199984.000000 NaN NaN NaN \n", "\n", " itemid \n", "count 817373.000000 \n", "unique NaN \n", "top NaN \n", "freq NaN \n", "first NaN \n", "last NaN \n", "mean 222346.358593 \n", "std 23242.543767 \n", "min 4647.000000 \n", "25% 225158.000000 \n", "50% 225158.000000 \n", "75% 225158.000000 \n", "max 225158.000000 " ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "label = 'normal saline'\n", "df = utils.open_df(hdf5_fname,'extract/{}'.format(label))\n", "df.describe(include='all')" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "collapsed": false, "scrolled": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelnormal saline
unitsmL/hourmLmlmL/hourno_unitsmlmLmlmL/minLmlno_unitsmlmL/hourmlmL/hour
description2251582251583019030190301902251583019046472251582251586190440534405344440444404191341913
count719400.00000039402.05167.0259.0160.06730.00000050.0880.01.013.02.05.01.02.01.0
unique443511.000000147.0243.01.01.04233.0000001.016.01.012.01.02.01.01.01.0
top99.999996500.01.00.00.049.9999990.0given100.01000.0given17.00.037.00.020.00.0
freq25193.00000011589.0651.0259.0160.0352.00000050.0860.01.012.02.03.01.02.01.0
\n", "
" ], "text/plain": [ "label normal saline \\\n", "units mL/hour mL ml mL/hour no_units ml \n", "description 225158 225158 30190 30190 30190 225158 \n", "count 719400.000000 39402.0 5167.0 259.0 160.0 6730.000000 \n", "unique 443511.000000 147.0 243.0 1.0 1.0 4233.000000 \n", "top 99.999996 500.0 1.0 0.0 0.0 49.999999 \n", "freq 25193.000000 11589.0 651.0 259.0 160.0 352.000000 \n", "\n", "label \\\n", "units mL ml mL/min L ml no_units ml \n", "description 30190 4647 225158 225158 6190 44053 44053 44440 \n", "count 50.0 8 80.0 1.0 1 3.0 2.0 5.0 \n", "unique 1.0 1 6.0 1.0 1 2.0 1.0 2.0 \n", "top 0.0 given 100.0 1000.0 given 17.0 0.0 37.0 \n", "freq 50.0 8 60.0 1.0 1 2.0 2.0 3.0 \n", "\n", "label \n", "units mL/hour ml mL/hour \n", "description 44440 41913 41913 \n", "count 1.0 2.0 1.0 \n", "unique 1.0 1.0 1.0 \n", "top 0.0 20.0 0.0 \n", "freq 1.0 2.0 1.0 " ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mimic_transform.set_params(add_level__level_val=label)\n", "df_tr = mimic_transform.transform(df)\n", "df_tr.describe()" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "((770238, 17), (504923, 9), 265320L, 4, '0.0202% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelnormal saline
statusknownunknownknownunknown
variable_typeqnqnqnqnnom
unitsmL/hrmLmL/hrno_unitsmLmL/minno_units
description225158(mL/hour)22515830190(ml)30190(mL/hour)30190225158(ml)301902251584647(ml)
count456226.00000039261.0000005068.000000258.0157.04844.00000050.080.0000008
uniqueNaNNaNNaNNaNNaNNaNNaNNaN1
topNaNNaNNaNNaNNaNNaNNaNNaNgiven
freqNaNNaNNaNNaNNaNNaNNaNNaN8
mean75.669716471.5604394.2550240.00.0128.0948180.0103.312500NaN
std217.566524372.0848618.4189810.00.0260.1865340.055.885358NaN
min-906.0000000.0000000.0000000.00.00.0000000.00.500000NaN
25%6.000000200.0000000.8500000.00.09.3102940.0100.000000NaN
50%15.000000500.0000001.1000000.00.025.0000000.0100.000000NaN
75%62.255889500.0000002.4000000.00.0100.0000010.0100.000000NaN
max51947.99940011000.000000117.0000000.00.05000.0001600.0300.000000NaN
\n", "
" ], "text/plain": [ "label normal saline \\\n", "status known \n", "variable_type qn \n", "units mL/hr mL mL/hr \n", "description 225158(mL/hour) 225158 30190(ml) 30190(mL/hour) \n", "count 456226.000000 39261.000000 5068.000000 258.0 \n", "unique NaN NaN NaN NaN \n", "top NaN NaN NaN NaN \n", "freq NaN NaN NaN NaN \n", "mean 75.669716 471.560439 4.255024 0.0 \n", "std 217.566524 372.084861 8.418981 0.0 \n", "min -906.000000 0.000000 0.000000 0.0 \n", "25% 6.000000 200.000000 0.850000 0.0 \n", "50% 15.000000 500.000000 1.100000 0.0 \n", "75% 62.255889 500.000000 2.400000 0.0 \n", "max 51947.999400 11000.000000 117.000000 0.0 \n", "\n", "label \n", "status unknown known unknown \n", "variable_type qn qn qn nom \n", "units no_units mL mL/min no_units \n", "description 30190 225158(ml) 30190 225158 4647(ml) \n", "count 157.0 4844.000000 50.0 80.000000 8 \n", "unique NaN NaN NaN NaN 1 \n", "top NaN NaN NaN NaN given \n", "freq NaN NaN NaN NaN 8 \n", "mean 0.0 128.094818 0.0 103.312500 NaN \n", "std 0.0 260.186534 0.0 55.885358 NaN \n", "min 0.0 0.000000 0.0 0.500000 NaN \n", "25% 0.0 9.310294 0.0 100.000000 NaN \n", "50% 0.0 25.000000 0.0 100.000000 NaN \n", "75% 0.0 100.000001 0.0 100.000000 NaN \n", "max 0.0 5000.000160 0.0 300.000000 NaN " ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_cln1 = standard_pipeline.transform(df_tr)\n", "print utils.data_loss(df_tr,df_cln1)\n", "df_cln1.describe(include='all')" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "this step: ((504923, 9), (504874, 2), 295L, 1, '0.0051% records')\n", "overall: ((770238, 17), (504874, 2), 265615L, 5, '0.0253% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelnormal saline
statusknown
variable_typeqn
unitsmLmL/hr
count49173.000000456484.000000
mean389.56317875.626948
std380.287765217.512467
min0.000000-906.000000
25%100.0000006.000000
50%250.00000015.000000
75%500.00000062.204886
max11000.00000051947.999400
\n", "
" ], "text/plain": [ "label normal saline \n", "status known \n", "variable_type qn \n", "units mL mL/hr\n", "count 49173.000000 456484.000000\n", "mean 389.563178 75.626948\n", "std 380.287765 217.512467\n", "min 0.000000 -906.000000\n", "25% 100.000000 6.000000\n", "50% 250.000000 15.000000\n", "75% 500.000000 62.204886\n", "max 11000.000000 51947.999400" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_cln2 = cleaning_pipeline.transform(df_cln1)\n", "print 'this step:',utils.data_loss(df_cln1,df_cln2)\n", "print 'overall:',utils.data_loss(df_tr,df_cln2)\n", "df_cln2.describe(include='all')" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "array([[,\n", " ]], dtype=object)" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAFeCAYAAADzFKfgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9clfX9//Hn+QGiHPBHYbd0DoyJpTMT0Fn+GE1Z2Nxn\nlVDg1Jrmr1Xb1HlDWop+0sA2P61ltvZpy88HW2K/1ubWKtJwMpdKN1xSWl/b1KwZoSUcCQ6c9/cP\nP5yJggeBA9r7cb/dvN0813W9r9f7fTjviyfnXOe6HMYYIwAAYBVnV3cAAAB0PgIAAAAWIgAAAGAh\nAgAAABYiAAAAYCECAAAAFrpoA8CaNWtUUlKitWvXavr06V3dnSauvPJKffrpp+3ezwsvvKB58+ZJ\nku677z7t2LGjXX3atWvXObeZPn261q5dqyNHjmjEiBFtrtWRvvGNb+h3v/tdl7TfuXOnrrzyyjbX\nbkv7tWvXauXKlefc12uvvaZHH320zf26UDGnz79PzOnzcyHN6UZPPfWUnnrqKeXk5OjJJ588rz60\n91hwUQaAPXv26MCBAxozZowiIiIUGRnZ1V1qwuFwdPg+V65cqWuvvbbN7bt3767u3bsH3aZHjx6S\nQjOGtmhNv0PVPiIiol2129u+pX1NmDBBpaWl2rdvX4fs+0LAnD5/zOnzdyHN6UZbtmzRxIkT29SH\n9h4L3G1q1cUeeeSRwF8IQ4cOVeO1jL7xjW/olltu0Y4dO/TRRx9p0qRJWrx4sSSpsLBQGzZskMvl\n0iWXXKJly5YpNjZWOTk5+vTTT/XBBx8oJSVFn3zyibp166a33npLlZWVSktLU58+fbRlyxZVVlZq\n5cqV+trXvqZ//vOf+s///E+dPHlSH3/8sa666io99NBDCg8PV0vXVvrFL36h1157TWFhYerVq5fy\n8/N16aWX6tlnn9WmTZtUX1+vTz/9VHPmzFFmZmaTttOnT9f06dM1dOhQ3XHHHfr617+uPXv26MSJ\nE/rhD3+oG2+8UZL0y1/+Uq+88oqMMerfv79yc3MVExOj6667TgkJCed8XpOSkjRs2LAmyw4cOKA5\nc+YoJydHV111VYu16+vrlZ+frx07dsjlcmn48OFasmSJnnvuOb311lv66U9/qvr6en3ta1/TT37y\nE91yyy168803lZeXp8WLF+uhhx7SgAED9N5778nn82nZsmUaNWqURo0apSFDhpyz3y+88IJefvll\n/fKXvzzr8entH374YW3evFl9+vRRYmKi9u7dq4KCAuXk5CgyMlLvvvuu/vWvf+mKK67QQw89pK98\n5SsaO3bsOWuf/rP55je/2eTx2LFjA+1zcnJa9bpqyZl9SU9P19q1a7V27dqg/bsYMKeZ06ezYU5L\nUlVVlbxery677DJJ0ptvvqmXX35ZlZWVGjRokP7rv/5LERERGjZsmCZMmKD9+/dr1apVHXcsMBeZ\nEydOmGuuucb4fL6z1l1//fVm9erVxhhj/vWvf5mrr77afPDBB+avf/2r+eY3v2mOHz9ujDHm+eef\nNzfeeKMxxpglS5aY733ve4F9LFmyxNx2222moaHBVFRUmMGDB5sNGzYYY4z5n//5HzNz5kxjjDGr\nV682v//9740xxvh8PvPtb3/bvPLKK8YYYwYPHhyo1eijjz4ySUlJpq6uzhhjzJNPPmmKioqM1+s1\nt912m/n000+NMcaUlZWZESNGBPo5d+5cY4wx06ZNMy+//LL54IMPzODBg83rr79ujDHm5ZdfNtdf\nf70xxpgXXnjBLFiwwDQ0NBhjjCksLDSzZ88+7+f4gw8+MCNGjDDvvvuuSU1NNX/7298Cy1uq/fDD\nD5t77rknUDsnJ8fk5uaaI0eOmDFjxhhjjHnjjTfM2LFjzaJFi4wxxjz44IPmiSeeMG+88YYZOnSo\n2bdvnzHGmN/85jdm2rRpre7v6c9Tc48b+zp58mRz8uRJU19fb2bPnm2mT59ujDn1M8/KyjI+n8/4\nfD5z8803m+eff77V9Rt/Ni09bqzRmtfVI488Yu6///6gNaurq83w4cNNbW1tq/t5oWJOM6fPZMuc\n/sMf/mDWrVsX2N+tt95qamtrTUNDg7n55pvNiy++aIw59fprfG2eqT3HgovuI4CDBw+qb9++crub\nf/NiwoQJkqTLLrtMl1xyiT777DNt375dkyZNUq9evSRJN998sz7++GMdOXJEkpSYmNhkH9dff72c\nTqcuvfRSde/eXePGjZMkffnLX9Znn30mSVq8eLF69+6tJ554QsuXL1dFRYW8Xm+L/b7ssst01VVX\n6eabb9bq1as1ePBgTZgwQT169NAvf/lLbd26VQ8//LAee+wx1dTUnPM5CAsL09e//nVJ0pAhQwJ9\nev3117Vnzx7dcsstuummm7RhwwYdPHjwnPtqSV1dnW6//XZdddVVTVJsS7X/8pe/KCsrS07nqZfU\n9OnTtW3bNvXr10+XXXaZ3nrrLf3lL3/R3LlztXPnTkmnPr9qTNj9+vXT4MGDz9pvR3njjTeUmpqq\n7t27y+Vy6dZbb22yfty4cXK73XK73UpISOjw+lLrXletFRkZKY/HE3gNX8yY08zptvgizOmioqLA\n61s69VoPDw+X0+lUQkKCjh07FliXlJTU7D7acyy46AKA0+lUQ0NDi+sjIiKaPDbGyO/3n7Wd3+9X\nfX29JJ31eWN4eHiTx80dmBYsWKBNmzapf//++t73vhf0LS2Hw6GCggLl5+erd+/eysvL0wMPPKCj\nR4/qpptu0kcffaTk5GT96Ec/Oud+pFMT9vT9mv97e9Lv92v27Nn63e9+p9/97nd67rnn9NRTTwXd\nX0seffRRvf3223r11VdbVft0DQ0Ngef3m9/8poqLi1VSUqK0tDT169dPf/rTn9S9e3cNGDBAktSt\nW7dm99saZ3626fP5ztqmW7duTfZ5+jikpq+bttQ/ffvm6kute12dj4aGBrlcrnbt40LAnGZOn8mG\nOe3z+XTw4MEmH+O09LOQFDiXozltPRZcdAFgwIABqqysVF1dXavbjBs3Ti+99FIgTT333HPq3bu3\nYmNj29yPkpIS3XXXXZo0aZKMMdqzZ885D2L79u3T5MmTFR8frzlz5uiOO+7Qvn379NZbb6lPnz6a\nP3++xowZo61bt0rSOV+sLa0bO3asnnnmGVVXV0uSfv7znys7O7tN4wsLC9OIESO0atUq5ebmqrKy\nMmjtp59+WvX19fL7/frtb3+rMWPGSJImTpyozZs3y+/369JLL9V1112nn/70p0pLS2tT387Uu3dv\nvfvuu6qrq1N9fb22bNly1jYpKSl66aWXVFVVJb/frxdffLFDaktSnz59tHfvXknSoUOHtH///nbt\nrzUHqurqatXW1uryyy9vV60LAXOaOX0mG+b0jh07gp4j0BrtORZcdCcBRkVFKTk5WW+88Ubg7ZZG\nZ6bGxsfXXXedbr/9dt1+++2STr24Hn/88VbVa+nM2QULFuiuu+5Sr1691L17d40aNUqHDh1qsc2V\nV16pSZMm6ZZbblGPHj3UvXt33XfffYqLi9Nzzz2nG264QZGRkRo2bJj69Olz1tt8p++zpT5lZGTo\n448/1m233San06nLL79ceXl5Z2130003adWqVRo6dGjQcY8aNUrf+ta3dO+992rZsmUt1v7+97+v\nBx98UDfddJMaGhp09dVXa+nSpZKk+Ph4ORyOwBnPY8eO1WOPPabU1NQW6zdnzpw5ysrK0vXXX99k\n+dixYzVq1CilpaWpb9+++trXvnbWhB01apRmzJihrKwsRUREqF+/fudVe+/evbrvvvua/frR/Pnz\ntWTJEr3++uu64oorNGrUqKD7O9cZ2Zs2bdILL7wQeDx48GA9/fTTTbbZvn27rr/++rP+6rkYMaeZ\n0zbO6cGDB2vy5Mmt6uu59t2uY8F5nzVwAXjzzTfNnDlzurobF62HHnrIvPPOO13djfO2adMm8+qr\nr3bIvv785z8HThhqrTNPQupKM2bMMPv37+/qbnQY5nT7MKcv/jndVu05FgT9CMAYo9zcXGVmZmrG\njBk6fPhwk/VbtmxRenq6MjMz9cwzz7SqTV5engoLCwOPN23apClTpigzM1Ovv/560NAyYsQIXXHF\nFdq+fXtrMg7O0L9//3ZdDKOruN3us/5S6CxHjx5VVlZWl9Q+U1FRkUaOHBn0K2Dn0pHz+tChQ5o6\ndaqmTZumFStWBPaxfv163XrrrbrtttsCFyupra3VD37wA333u9/V3Llzdfz4cUnM6fZiTp+/C2lO\nt1W7jwXBEsIrr7xilixZYow59XWW+fPnB9b5fD6TmppqqqqqTF1dnZkyZYqprKxssU1lZaW58847\nTWpqqtm4caMxxpiKigozefJk4/P5TFVVlZk8eXLgazUAQqMj5/W8efPMrl27jDHGLFu2zLz66qvm\n0KFDZsqUKYF9ZmZmmv3795snn3zSPPLII8YYY/74xz+alStXdsp4AZwt6DsApaWlgc/lhg8fHjgx\nQjp1QYnY2Fh5PB6FhYUpOTlZO3fuPKtNeXm5JOnkyZO655579B//8R+Bffz9739XUlKS3G63PB6P\n4uLi2n3CBYBz68h5XV5eruTkZEnS+PHjtWPHDvXr109PPPFEYJ8NDQ3q1q2bSktLNX78+CbbAuga\nQQNAdXW1oqKiAo/dbnfg6yFnruvRo0fgykanL3e5XPL7/frSl76kq6+++pz7b9wHgNDpqHnd0NDQ\n5AznyMhIVVVVyeVyBb6jv3r1ag0ZMkSxsbGqrq6Wx+MJbNt4djuAzhc0AHg8niYXw/D7/YELQ3g8\nniYT2Ov1qmfPnuds09z+z9xHdHT0OftkzuP7nADO1lHz2uVyNZnbp8/furo6LVq0SDU1NcrNzT2r\n7pmBoiXMdyA0gn4NMDExUVu3blVaWprKysqanGwQHx+vgwcP6sSJE4qIiNDu3bs1a9YsSWqxzZmu\nvvpq/fznP1ddXZ1qa2v1/vvva9CgQefsk8PhUEVF171LEBMT1aX1L4Q+2F7/QuhDTEzwX54t6ch5\nPWTIEO3atUsjR47Utm3bNHr0aEmnvkp17bXX6s4772xSt7i4WMOGDVNxcXHgo4Nz6ez53tk/1y9y\nvS/y2Dq7Xnvme0uCBoDU1FSVlJQEbmSRl5enzZs3q6amRhkZGcrJydHMmTNljFF6err69u3bbJuW\nXHrppZo+fbqmTp0qY4wWLlx41tWVAHSsjpzX2dnZWrp0qXw+n+Lj45WWlqaioiLt3r1bPp9PxcXF\ncjgcWrRokbKyspSdna2pU6cqPDxca9as6bLnALCdw1yk76919V9e/PVpd/0LoQ+h+IvgQvVF/avu\ni17vizy2zq4Xivl+0V0KGAAAtB8BAAAACxEAAACwEAEAAAALEQAAALAQAQAAAAsRAAAAsBABAAAA\nCxEAAACwEAEAAAALEQAAALAQAQAAAAsFvRsgAHzRPfviS3r7/X9JkrpFhKn2c1+r245JHKQJXx8b\nqq4BIUMAAGC9f35YqX/Wxp56UHt+bfsd+rDjOwR0Aj4CAADAQgQAAAAsRAAAAMBCBAAAACxEAAAA\nwEIEAAAALEQAAADAQgQAAAAsRAAAAMBCBAAAACxEAAAAwEIX5b0AMmf/RGHde7VrHyc/O6pHV98n\nl8vVQb0CAODicVEGgEpfb0VED2rXPj7318oY00E9AgDg4sJHAAAAWIgAAACAhQgAAABYiAAAAICF\nCAAAAFiIAAAAgIUIAAAAWIgAAACAhQgAAABYiAAAAICFCAAAAFiIAAAAgIUIAAAAWIgAAACAhQgA\nAABYiAAAAICFCAAAAFiIAAAAgIUIAAAAWIgAAACAhQgAAABYiAAAAICFCAAAAFiIAAAAgIUIAAAA\nWIgAAACAhdzBNjDGaPny5dq/f7/Cw8O1atUqDRgwILB+y5YtWrdundxut6ZMmaKMjIwW2xw6dEhL\nliyR0+nUoEGDlJubK0n6zW9+o82bN8vlcmnu3LmaOHFi6EYMAACCvwNQVFSkuro6bdy4UYsWLVJe\nXl5gXX19vfLz87V+/XoVFBSosLBQx44da7FNXl6eFi5cqA0bNsjv96uoqEhVVVUqKCjQM888o1//\n+td64IEHQjdaAAAgqRUBoLS0VOPGjZMkDR8+XHv37g2sO3DggGJjY+XxeBQWFqbk5GTt3LnzrDbl\n5eWSpPLyciUnJ0uSxo8frx07dqh79+7q37+/vF6vTp48KaeTTyUAAAi1oB8BVFdXKyoq6t8N3G75\n/X45nc6z1vXo0UNVVVXyer1NlrtcLjU0NMgYE1gWGRmpqqoqSdJll12mG2+8UcYYzZkzp0MGBgAA\nWhY0AHg8Hnm93sDjxl/+jeuqq6sD67xer3r27NlsG5fL1eSve6/Xq+joaG3btk2ffPKJtm7dKmOM\nZs2apcTERA0bNqxDBtgSp9OhmJgoud1Bn4JmxcREBd8oxLq6D7bXv1D6AABtEfS3X2JiorZu3aq0\ntDSVlZUpISEhsC4+Pl4HDx7UiRMnFBERod27d2vWrFmS1GybIUOGaNeuXRo5cqS2bdum0aNHKzo6\nWhEREQoLC5MkRUVFBd4ZCCW/36iioqpNASAmJkoVFaHv44XcB9vrXwh9IHwAaI+gv/1SU1NVUlKi\nzMxMSadO5Nu8ebNqamqUkZGhnJwczZw5U8YYpaenq2/fvs22kaTs7GwtXbpUPp9P8fHxSktLk8Ph\n0I4dO3TrrbfK6XQqKSlJ1113XQiHDAAAggYAh8OhFStWNFk2cODAwP9TUlKUkpIStI0kxcXFqaCg\n4Kzl99xzj+65557W9hkAALQTp9wDAGAhAgAAABYiAAAAYCECAAAAFiIAAABgIQIAAAAWIgAAAGCh\ntl0HF8BFrTNu8y1Jx44dU1ZWlv7whz8oPDxc0qkbgcXFxUmSRowYoQULFnTq2AGcQgAALHT6Lbv3\n7NmjvLw8rVu3TtK/b/P9/PPPq1u3bsrKytKECRNUWlrabJvG23wnJycrNzdXRUVFmjhxorZv3641\na9aosrIyUPfQoUMaOnSoHnvssa4aOoD/w0cAgIVCfZtv6dRdQNevX6+ePXsG9r13714dPXpUM2bM\n0Ny5c/WPf/yjU8YL4Gy8AwBYqDNu833ttddKUpP1ffv21dy5c3XDDTeotLRUixcv1rPPPhuycQJo\nGQEAsFCob/N9OofDEfj/V7/6VblcLklSUlKSKioqOnZgAFqNAABYKNS3+T7d6e8ArF27Vr169dKd\nd96pffv26fLLL29Vf0N96+Pu3cOkz9rWNjKyW7v719m3du7Mel/ksXVFvY5EAAAsFOrbfJ/u9HcA\n5syZo8WLF6u4uFhutzuwj2AqKqo6Ytgtqqnxtbmt11vbrv7FxESFfHxdVe+LPLbOrheKoEEAACzU\nGbf5bvTaa68F/h8dHa3HH3+8jb0G0JH4FgAAABYiAAAAYCECAAAAFiIAAABgIQIAAAAWIgAAAGAh\nAgAAABYiAAAAYCECAAAAFiIAAABgIQIAAAAWIgAAAGAhAgAAABYiAAAAYCECAAAAFiIAAABgIQIA\nAAAWIgAAAGAhAgAAABYiAAAAYCECAAAAFiIAAABgIQIAAAAWIgAAAGAhAgAAABYiAAAAYCECAAAA\nFiIAAABgIQIAAAAWIgAAAGAhAgAAABYiAAAAYCECAAAAFiIAAABgIQIAAAAWIgAAAGAhAgAAABYi\nAAAAYCF3sA2MMVq+fLn279+v8PBwrVq1SgMGDAis37Jli9atWye3260pU6YoIyOjxTaHDh3SkiVL\n5HQ6NWjQIOXm5kqSiouLtW7dOknS0KFDtWzZshANFwAASK14B6CoqEh1dXXauHGjFi1apLy8vMC6\n+vp65efna/369SooKFBhYaGOHTvWYpu8vDwtXLhQGzZskN/vV1FRkbxer372s5/p8ccfV2Fhofr3\n76/jx4+HbsQAACD4OwClpaUaN26cJGn48OHau3dvYN2BAwcUGxsrj8cjSUpOTtbOnTtVVlbWpE15\nebkkqby8XMnJyZKk8ePHq6SkRBEREUpISFB+fr4OHz6sjIwM9e7du2NHCQAAmggaAKqrqxUVFfXv\nBm63/H6/nE7nWet69Oihqqoqeb3eJstdLpcaGhpkjAksi4yMVHV1tY4fP6433nhDv//97xUREaHv\nfve7GjFihGJjYztqjAAA4AxBPwLweDzyer2Bx42//BvXVVdXB9Z5vV717Nmz2TYulyvQrnHb6Oho\n9erVS8OGDVOfPn3Uo0cPJScn65133umQwQEAgOYFfQcgMTFRW7duVVpamsrKypSQkBBYFx8fr4MH\nD+rEiROKiIjQ7t27NWvWLElqts2QIUO0a9cujRw5Utu2bdPo0aM1dOhQvffee/r000/l8Xi0Z88e\n3XbbbSEa7r85nQ7FxETJ7Q76FDQrJiYq+EYh1tV9sL3+hdIHAGiLoL/9UlNTVVJSoszMTEmnTuTb\nvHmzampqlJGRoZycHM2cOVPGGKWnp6tv377NtpGk7OxsLV26VD6fT/Hx8UpLS5PD4dDChQs1c+ZM\nORwO3XjjjfrKV74SwiGf4vcbVVRUtSkAxMREqaKiKgS9unj6YHv9C6EPhA8A7RH0t5/D4dCKFSua\nLBs4cGDg/ykpKUpJSQnaRpLi4uJUUFBw1vIbb7xRN954Y2v7DAAA2okLAQEAYCECAAAAFiIAAABg\nIQIAAAAWIgAAAGAhAgAAABYiAAAAYCECAAAAFiIAAABgIQIAAAAWIgAAAGAhAgAAABYiAAAAYCEC\nAAAAFiIAAABgIQIAAAAWIgAAAGAhAgAAABYiAAAAYCECAAAAFiIAAABgIQIAAAAWIgAAAGAhAgBg\nIWOMcnNzlZmZqRkzZujw4cNN1m/ZskXp6enKzMzUM888c842hw4d0tSpUzVt2jStWLGiyX6OHTum\nG264QXV1dZKk2tpa/eAHP9B3v/tdzZ07V8ePH++E0QJoDgEAsFBRUZHq6uq0ceNGLVq0SHl5eYF1\n9fX1ys/P1/r161VQUKDCwkIdO3asxTZ5eXlauHChNmzYIL/fr6KiIknS9u3bNWvWLFVWVgb2/fTT\nTyshIUFPPfWUvvOd72jdunWdO3AAAQQAwEKlpaUaN26cJGn48OHau3dvYN2BAwcUGxsrj8ejsLAw\nJScna+fOnWe1KS8vlySVl5crOTlZkjR+/Hjt2LFDkuRyubR+/Xr17NmzSd3x48eftS2Azufu6g4A\n6HzV1dWKiooKPHa73fL7/XI6nWet69Gjh6qqquT1epssd7lcamhokDEmsCwyMlJVVVWSpGuvvVaS\nmqyvrq6Wx+MJbFtdXR2aAQIIigAAWMjj8cjr9QYeN/7yb1x3+i9mr9ernj17NtvG5XIF2jVuGx0d\n3aSWw+Fotu6ZgeJcYmJat11bde8eJn3WtraRkd3a3b9Qj68r632Rx9YV9ToSAQCwUGJiorZu3aq0\ntDSVlZUpISEhsC4+Pl4HDx7UiRMnFBERod27d2vWrFmS1GybIUOGaNeuXRo5cqS2bdum0aNHN6l1\n+jsAiYmJKi4u1rBhw1RcXBz46CCYioqq9g75nGpqfG1u6/XWtqt/MTFRIR9fV9X7Io+ts+uFImgQ\nAAALpaamqqSkRJmZmZJOnci3efNm1dTUKCMjQzk5OZo5c6aMMUpPT1ffvn2bbSNJ2dnZWrp0qXw+\nn+Lj45WWltak1unvAGRlZSk7O1tTp05VeHi41qxZ00kjBnAmAgBgIYfDcdZX9gYOHBj4f0pKilJS\nUoK2kaS4uDgVFBS0WOu1114L/D8iIkIPP/xwG3sNoCPxLQAAACxEAAAAwEIEAAAALEQAAADAQgQA\nAAAsRAAAAMBCBAAAACxEAAAAwEIEAAAALEQAAADAQgQAAAAsRAAAAMBCBAAAACxEAAAAwEIEAAAA\nLEQAAADAQgQAAAAsRAAAAMBCBAAAACxEAAAAwEIEAAAALEQAAADAQgQAAAAsFDQAGGOUm5urzMxM\nzZgxQ4cPH26yfsuWLUpPT1dmZqaeeeaZc7Y5dOiQpk6dqmnTpmnFihVn1Zk9e7YKCws7amwAAKAF\nQQNAUVGR6urqtHHjRi1atEh5eXmBdfX19crPz9f69etVUFCgwsJCHTt2rMU2eXl5WrhwoTZs2CC/\n36+ioqLAvn7+85+rqqoqBEMEAABnChoASktLNW7cOEnS8OHDtXfv3sC6AwcOKDY2Vh6PR2FhYUpO\nTtbOnTvPalNeXi5JKi8vV3JysiRp/Pjx2rFjhyTp5ZdfltPp1NixYzt2dAAAoFlBA0B1dbWioqIC\nj91ut/x+f7PrevTooaqqKnm93ibLXS6XGhoaZIwJLIuMjFRVVZXee+89bd68WT/4wQ86ZEAAACA4\nd7ANPB6PvF5v4LHf75fT6Qysq66uDqzzer3q2bNns21cLlegXeO20dHRevHFF/Xxxx9rxowZOnLk\niMLDw9W/f/+QvxvgdDoUExMltzvoU9CsmJio4BuFWFf3wfb6F0ofAKAtgv72S0xM1NatW5WWlqay\nsjIlJCQE1sXHx+vgwYM6ceKEIiIitHv3bs2aNUuSmm0zZMgQ7dq1SyNHjtS2bds0evRoTZo0KbC/\ntWvXKiYmplM+CvD7jSoqqtoUAGJiolRR0bXnK3R1H2yvfyH0gfABoD2C/vZLTU1VSUmJMjMzJZ06\nkW/z5s2qqalRRkaGcnJyNHPmTBljlJ6err59+zbbRpKys7O1dOlS+Xw+xcfHKy0tLYRDAwAALQka\nABwOx1lf2Rs4cGDg/ykpKUpJSQnaRpLi4uJUUFDQYq277747WHcAAEAH4EJAAABYiAAAAICFCAAA\nAFiIAAAAgIUIAAAAWIgAAACAhQgAAABYiAAAAICFCAAAAFiIAAAAgIUIAAAAWIgAAACAhQgAAABY\niAAAAICFCAAAAFiIAAAAgIUIAAAAWIgAAACAhQgAAABYiAAAAICFCAAAAFiIAAAAgIUIAAAAWIgA\nAACAhQgAAABYiAAAAICFCAAAAFiIAAAAgIUIAAAAWIgAAACAhQgAAABYiAAAAICFCAAAAFiIAAAA\ngIUIAADpki9qAAARWUlEQVQAWIgAAACAhQgAAABYiAAAAICF3F3dAQCdzxij5cuXa//+/QoPD9eq\nVas0YMCAwPotW7Zo3bp1crvdmjJlijIyMlpsc+jQIS1ZskROp1ODBg1Sbm6uJGnTpk0qLCxUWFiY\n5s2bp5SUFEnS+PHjFRcXJ0kaMWKEFixY0NnDByACAGCloqIi1dXVaePGjdqzZ4/y8vK0bt06SVJ9\nfb3y8/P1/PPPq1u3bsrKytKECRNUWlrabJu8vDwtXLhQycnJys3NVVFRka655hoVFBTohRde0Oef\nf66srCyNGTNGH330kYYOHarHHnusi58BAAQAwEKlpaUaN26cJGn48OHau3dvYN2BAwcUGxsrj8cj\nSUpOTtbOnTtVVlbWpE15ebkkqby8XMnJyZJO/XVfUlIip9OppKQkud1ueTwexcXFaf/+/Tp06JCO\nHj2qGTNmqHv37lqyZIkGDhzYmUMH8H84BwCwUHV1taKiogKP3W63/H5/s+t69Oihqqoqeb3eJstd\nLpcaGhpkjAksi4yMVHV19VnbNu6jb9++mjt3rv73f/9Xc+bM0eLFi0M5TADnwDsAgIU8Ho+8Xm/g\nsd/vl9PpDKyrrq4OrPN6verZs2ezbVwuV6Bd47bR0dHN7iM6Olrx8fFyuVySpKSkJFVUVLSqvzEx\nUcE3aofu3cOkz9rWNjKyW7v7F+rxdWW9L/LYuqJeRyIAABZKTEzU1q1blZaWprKyMiUkJATWxcfH\n6+DBgzpx4oQiIiK0e/duzZo1S5KabTNkyBDt2rVLI0eO1LZt2zR69GgNGzZMDz30kOrq6lRbW6v3\n339fgwYN0i9+8Qv16tVLd955p/bt26fLL7+8Vf2tqKjq+CfhNDU1vja39Xpr29W/mJiokI+vq+p9\nkcfW2fVCETQIAICFUlNTVVJSoszMTElSXl6eNm/erJqaGmVkZCgnJ0czZ86UMUbp6enq27dvs20k\nKTs7W0uXLpXP51N8fLzS0tLkcDg0ffp0TZ06VcYYLVy4UOHh4YG3/YuLi+V2uwP7AND5CACAhRwO\nh1asWNFk2ekn46WkpAS+tneuNpIUFxengoKCs5ZnZGQoIyOjybLo6Gg9/vjj7eg5gI7CSYAAAFiI\nAAAAgIUIAAAAWIgAAACAhQgAAABYiAAAAICFCAAAAFgo6HUAOuO2oevXr9ef/vQnORwOjR8/Xnfd\ndVfoRgwAAIK/A3D6bUMXLVrU5MpdjbcNXb9+vQoKClRYWKhjx4612KbxtqEbNmyQ3+9XUVGRDh8+\nrM2bNwfuHb59+3a9++67oRsxAAAIHgBae9vQsLCwwG1Dz2zT0m1Dd+zYoX79+umJJ54I7LO+vl7d\nunXruBECAICzBA0AobxtaFVVlVwul3r16iVJWr16tYYMGaLY2Nj2jwwAALQo6DkAob5tqCTV1dUp\nJydHUVFRWr58ebsH1RpOp0MxMVFyu9t2O4QL4RaQXd0H2+tfKH0AgLYI+tsv1LcNlaT58+fr2muv\n1Z133hmKMTbL7zeqqKhqUwDo7FtOXoh9sL3+hdAHwgeA9gj62y/Utw0tKirS7t275fP5VFxcLIfD\noUWLFmn48OEhHDYAAHYLGgBCfdvQiRMnas+ePefTZwAA0E5cCAgAAAsRAAAAsBABAAAACxEAAACw\nEAEAAAALEQAAALAQAQAAAAsRAAAAsBABAAAACxEAAACwEAEAAAALEQAAALAQAQAAAAsRAAAAsBAB\nAAAACxEAAACwEAEAAAALEQAAALAQAQAAAAsRAAAAsBABAAAACxEAAACwEAEAAAALEQAAALAQAQAA\nAAsRAAAAsBABAAAACxEAAACwEAEAAAALEQAAALAQAQAAAAsRAAAAsBABAAAACxEAAACwEAEAAAAL\nEQAAALAQAQAAAAsRAAAAsBABAAAACxEAAACwEAEAAAALEQAAALAQAQAAAAsRAAAAsBABAAAACxEA\nAACwEAEAAAALEQAAALAQAQAAAAsRAAAAsBABAAAACxEAAACwEAEAAAALEQAAALCQO9gGxhgtX75c\n+/fvV3h4uFatWqUBAwYE1m/ZskXr1q2T2+3WlClTlJGR0WKbQ4cOacmSJXI6nRo0aJByc3MlSZs2\nbVJhYaHCwsI0b948paSkhGzAALpuXtfW1mrx4sWqrKyUx+NRfn6+evfu3VVPA2C1oO8AFBUVqa6u\nThs3btSiRYuUl5cXWFdfX6/8/HytX79eBQUFKiws1LFjx1psk5eXp4ULF2rDhg3y+/0qKirSJ598\nEmj7xBNPaM2aNfL5fKEbMYAum9dPP/20EhIS9NRTT+k73/mO1q1b11VPAWC9oO8AlJaWaty4cZKk\n4cOHa+/evYF1Bw4cUGxsrDwejyQpOTlZO3fuVFlZWZM25eXlkqTy8nIlJydLksaPH6+SkhI5nU4l\nJSXJ7XbL4/EoLi5O+/fv11e/+tWOHekZjN+vAwf+n9xu13m3PX7co2PHqgOP4+KukMt1/vsBukpX\nzOt9+/aptLRUs2fPDmxLAAC6TtAAUF1draioqH83cLvl9/vldDrPWtejRw9VVVXJ6/U2We5yudTQ\n0CBjTGBZZGSkqqurz9q2cR/nYqoPyq/PWzfCFlR9tFffX/EvRXj6tGs/n1cf032zU/XlL8e2az/n\n68wQ0tlsr98RfYiPH9SBvTk/XTGvG5c3BovGbS8IDbXyV74lSXK5nWqo97e66fEIpw4ceK/NpTv7\ntdyZ9b7IY2upXlfO6/MVNAB4PB55vd7A48aDROO60yew1+tVz549m23jcrkC7Rq3jY6ObnYf0dHR\n5+xT0bOPtGJoAFrSVfP69H2cGRLOJSamddu11eoVPwrp/oELUdBzABITE1VcXCxJKisrU0JCQmBd\nfHy8Dh48qBMnTqiurk67d+/WNddcoxEjRjTbZsiQIdq1a5ckadu2bUpKStKwYcNUWlqquro6VVVV\n6f3339egQRdPggIuRl01r0/fR3FxceCjAwCdz2FOf/+uGaef+SudOuGnvLxcNTU1ysjI0Ouvv661\na9fKGKP09HRlZWU122bgwIH65z//qaVLl8rn8yk+Pl4rV66Uw+HQM888o8LCQhljNH/+fE2cODH0\nIwcs1lXz+vPPP1d2drYqKioUHh6uNWvW6JJLLunKpwKwVtAAAAAAvni4EBAAABYiAAAAYCECAAAA\nFgr6NcALSbDLl7ZHfX297r33Xh05ckQ+n0/z5s3TV77ylS65xGllZaWmTJmiJ598Ui6Xq1P78Ktf\n/UpbtmyRz+fT1KlTNXLkyE6tX19fr+zsbB05ckRut1v3339/pz0He/bs0c9+9jMVFBR0yOVty8rK\n9MADD8jtduu6667T3Xff3er677zzjlauXCmXy6Xw8HA9+OCD6tOnT0jrd4VXX31Vf/7zn7VmzRpJ\np56DVatWndXntWvXqri4WG63Wzk5Obr66qt1/Phx/fjHP1Ztba369u2rvLw8devWrdnLGLdGRx1f\nOut1FKpjVkv1/H6/7rvvPv3jH/+Q0+nUihUrFB4eHtJ50tHHwnPVuuWWWwLXqPjSl76kefPmhbRe\nKI615z3nzUXklVdeMUuWLDHGGFNWVmbmz5/fYft+7rnnzAMPPGCMMeazzz4zKSkpZt68eWbXrl3G\nGGOWLVtmXn31VVNRUWEmT55sfD6fqaqqMpMnTzZ1dXXmySefNI888ogxxpg//vGPZuXKlW3qh8/n\nM3fddZe54YYbzPvvv9+pfXjjjTfMvHnzjDHGeL1e88gjj3T6c1BUVGR+9KMfGWOMKSkpMffcc0+n\n9OG///u/zeTJk81tt91mjDEdUvM73/mOOXz4sDHGmNmzZ5t33nmn1fWnTZtm9u3bZ4wxZuPGjSY/\nPz+k9bvCypUrzaRJk8zChQsDy5rrc3l5ubn99tuNMcZ8+OGHZsqUKcYYY+6//37zwgsvGGOMefzx\nx8369euNz+czqamppqqqytTV1ZkpU6aYysrKVvWnI44vnfk6CtUxq6V6r776qrn33nuNMaeOFfPn\nzw9pvVAcC1uqVVtba26++eYmP8tQ1gvVsfZ85/xF9RHAuS5f2l6TJk3SD3/4Q0lSQ0ODXC6X3n77\n7SaXOP3rX/+qv//97y1e4nT8+PGBbXfs2NGmfqxevVpZWVnq27evjDGd2oft27crISFB3//+9zV/\n/nylpKR0+nMQFxcXuLpcVVWV3G53p/QhNjZWjz76aODxmZe3PZ+af/vb31RdXS2fz6cvfelLkqSx\nY8fqr3/9a6vrP/TQQxo8eLCkU3/phYeHh7R+V0hMTNTy5csDj5vrc0lJiUpLSzVmzBhJ0uWXXy6/\n369jx47pzTffDBwPGn/Wp1/GOCwsTElJSYFrFATTEceXznwddfQxK1i9iRMn6v7775ckffjhh+rZ\ns2dI63XksTBYrX379unkyZOaNWuW7rjjDu3Zsyek9Tr6WNvWOX9RBYCWLl/aEbp37x64XOkPf/hD\nLViwoNMvcfr888/rkksu0ZgxYwK1Tx9fqPtw/Phx7d27V7/4xS+0fPly/fjHP+7U+o3tPvjgA6Wl\npWnZsmWaPn16p/wcUlNTm9zPoT01Gy+b27js9OWtrX/ppZdKkt5880399re/1R133NHsJXo7qn4o\nPfvss/r2t7/d5N/evXs1adKkJtu11Oczn/fmfh7n2ra14+6I40tnvo46+pjVmteN0+nUkiVLtHLl\nSk2ePDlk9Tr6WBhsbBEREZo1a5Z+/etfB459oXwuO/pY29Y5f1GdA3Cuy5d2hI8++kh33323pk2b\npm9961v66U9/GlgXqkucnu7555+Xw+FQSUmJ9u/fr+zsbB0/frzT+tCrVy/Fx8fL7XZr4MCB6tat\nm44ePdqpz8H69es1btw4LViwQEePHtX06dOb3B2yM/ogqd2Xtz0zfLTmEtdn+tOf/qTHH39cv/rV\nr9S7d+9Or99R0tPTlZ6eHnS75vrcs2dPhYWFNZn31dXVio6ODmzfp0+fwLjbcmnxRqE4voT6ddTR\nx6zWvG7y8/NVWVmp9PR01dbWhqReKI6F5xpbXFycYmNjA//v1auX3n777ZDVC8Wxti1z/qJ6B+Bc\nly9tr08++USzZs3S4sWLdfPNN0uSrrrqqk69xOmGDRtUUFCggoICXXnllXrwwQc1bty4TutDUlKS\n/vKXv0iSjh49qpqaGo0ePVo7d+7stOeg8ZrzkhQVFaX6+noNGTKkU/sgtf/yth6PR+Hh4Tp8+LCM\nMdq+fbuSkpJaXf/FF1/UU089pYKCAvXv31+SdPXVV3da/a7QUp9HjBih7du3yxijDz/8UMYY9erV\nS4mJidq2bZukUz+j5ORkXXHFFU0uY7xr1y5dc801raofiuNLKF9HoThmnaveiy++qF/96leSpG7d\nusnpdOqrX/1qu+ZmS/VCcSw819iee+455efnSzp17KuurtaYMWNCMjYpNMfatsz5i+pKgKaFS5F2\nhFWrVumll17SFVdcIWOMHA6HfvKTn2jlypVdconTGTNmaMWKFXI4HJ16mdWf/exn+tvf/iZjjBYt\nWqT+/fvrvvvu67T6J0+e1L333quKigrV19fr9ttv19ChQzulD0eOHNGiRYu0cePGDrm87d///net\nWrVKfr9fY8aM0Y9+dO4bzjTW/+1vf6trr71W/fr1k8fjkcPh0KhRo3T33XeHtH5X2LlzpwoLCwPf\nAmipz2vXrtW2bdtkjFFOTo4SExNVWVmp7OxsnTx5Ur1799aaNWsUERHR7GWMW6Ojji+d9ToK1TGr\npXo1NTXKycnRJ598ovr6es2dO1dXXHFFu+dmsNdpRx4LW6rl8/mUk5OjDz/8UE6nU4sXL1avXr1C\nOrZQHGvPd85fVAEAAAB0jIvqIwAAANAxCAAAAFiIAAAAgIUIAAAAWIgAAACAhQgAAABYiAAAAICF\nCAAAAFjo/wOY67O5O3hY0wAAAABJRU5ErkJggg==\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import seaborn as sns\n", "df_cln2.hist(normed=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Single pipeline" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [], "source": [ "all_pipeline = Pipeline([\n", " ('transform',mimic_transform),\n", " ('format',standard_pipeline),\n", " ('clean',cleaning_pipeline)\n", " ])" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
count382993.000000393608393608393608393608.000000
uniqueNaN1873236162NaN
topNaN2140-07-14 03:59:001.2mmol/LNaN
freqNaN820704393592NaN
firstNaN2096-08-25 16:32:00NaNNaNNaN
lastNaN2210-08-19 04:56:00NaNNaNNaN
mean150112.612539NaNNaNNaN64364.242439
std28874.716612NaNNaNNaN77900.842790
min100001.000000NaNNaNNaN818.000000
25%125190.000000NaNNaNNaN1531.000000
50%149789.000000NaNNaNNaN50813.000000
75%175567.000000NaNNaNNaN50813.000000
max199999.000000NaNNaNNaN225668.000000
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "count 382993.000000 393608 393608 393608 393608.000000\n", "unique NaN 187323 616 2 NaN\n", "top NaN 2140-07-14 03:59:00 1.2 mmol/L NaN\n", "freq NaN 8 20704 393592 NaN\n", "first NaN 2096-08-25 16:32:00 NaN NaN NaN\n", "last NaN 2210-08-19 04:56:00 NaN NaN NaN\n", "mean 150112.612539 NaN NaN NaN 64364.242439\n", "std 28874.716612 NaN NaN NaN 77900.842790\n", "min 100001.000000 NaN NaN NaN 818.000000\n", "25% 125190.000000 NaN NaN NaN 1531.000000\n", "50% 149789.000000 NaN NaN NaN 50813.000000\n", "75% 175567.000000 NaN NaN NaN 50813.000000\n", "max 199999.000000 NaN NaN NaN 225668.000000" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "label = 'lactate'\n", "df = utils.open_df(hdf5_fname,'extract/{}'.format(label))\n", "df.describe(include='all')" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labellactate
statusknown
variable_typeqn
unitsmmol/L
iddatetime
1000012117-09-11 09:32:001.9
1000032150-04-17 19:12:001.1
1000062108-04-08 10:58:004.5
1000072145-03-31 00:44:003.1
2145-04-02 14:10:001.9
1000092162-05-17 13:19:001.1
2162-05-17 17:14:001.5
1000102109-12-10 10:25:000.6
2109-12-10 12:11:000.9
2109-12-10 13:05:001.0
2109-12-10 13:58:000.8
1000112177-08-29 04:44:003.8
2177-08-29 06:55:002.3
1000122177-03-14 07:38:002.3
2177-03-14 11:42:002.5
2177-03-15 08:05:002.1
2177-03-15 14:01:002.6
2177-03-15 21:42:001.8
1000162188-05-24 12:00:002.0
1000172103-03-11 05:10:001.1
1000182176-08-29 15:29:001.3
2176-08-30 09:23:000.9
2176-08-30 10:19:001.0
2176-08-30 11:29:000.9
2176-08-30 12:40:001.1
1000202142-11-30 21:54:001.1
2142-12-03 00:17:001.0
1000242170-09-19 10:25:001.4
2170-09-19 16:33:002.6
2170-09-20 02:04:003.2
.........
1999762182-02-14 11:15:000.8
2182-02-16 03:57:000.8
2182-02-19 03:59:000.8
2182-02-20 03:31:000.7
2182-02-21 04:55:000.9
1999792182-02-06 09:17:001.0
2182-02-06 14:16:003.8
1999812110-09-24 16:34:001.1
2110-09-24 20:09:001.0
2110-09-25 06:10:001.0
1999872175-05-19 16:30:002.3
1999882169-01-24 12:48:001.0
2169-02-07 01:35:001.6
2169-02-07 11:18:001.0
2169-02-07 16:43:001.0
2169-02-07 22:35:001.1
2169-02-10 05:33:001.4
1999932161-11-12 23:14:000.9
2161-11-13 03:46:000.9
1999942188-07-07 21:23:001.0
2188-07-08 03:09:000.7
2188-07-08 04:13:000.6
2188-07-08 06:20:000.7
1999982119-02-20 10:52:001.1
2119-02-20 12:36:001.9
2119-02-20 13:33:002.0
2119-02-20 13:59:002.6
2119-02-20 20:43:001.3
1999992136-04-04 20:55:001.9
2136-04-06 15:29:001.8
\n", "

177439 rows × 1 columns

\n", "
" ], "text/plain": [ "label lactate\n", "status known\n", "variable_type qn\n", "units mmol/L\n", "id datetime \n", "100001 2117-09-11 09:32:00 1.9\n", "100003 2150-04-17 19:12:00 1.1\n", "100006 2108-04-08 10:58:00 4.5\n", "100007 2145-03-31 00:44:00 3.1\n", " 2145-04-02 14:10:00 1.9\n", "100009 2162-05-17 13:19:00 1.1\n", " 2162-05-17 17:14:00 1.5\n", "100010 2109-12-10 10:25:00 0.6\n", " 2109-12-10 12:11:00 0.9\n", " 2109-12-10 13:05:00 1.0\n", " 2109-12-10 13:58:00 0.8\n", "100011 2177-08-29 04:44:00 3.8\n", " 2177-08-29 06:55:00 2.3\n", "100012 2177-03-14 07:38:00 2.3\n", " 2177-03-14 11:42:00 2.5\n", " 2177-03-15 08:05:00 2.1\n", " 2177-03-15 14:01:00 2.6\n", " 2177-03-15 21:42:00 1.8\n", "100016 2188-05-24 12:00:00 2.0\n", "100017 2103-03-11 05:10:00 1.1\n", "100018 2176-08-29 15:29:00 1.3\n", " 2176-08-30 09:23:00 0.9\n", " 2176-08-30 10:19:00 1.0\n", " 2176-08-30 11:29:00 0.9\n", " 2176-08-30 12:40:00 1.1\n", "100020 2142-11-30 21:54:00 1.1\n", " 2142-12-03 00:17:00 1.0\n", "100024 2170-09-19 10:25:00 1.4\n", " 2170-09-19 16:33:00 2.6\n", " 2170-09-20 02:04:00 3.2\n", "... ...\n", "199976 2182-02-14 11:15:00 0.8\n", " 2182-02-16 03:57:00 0.8\n", " 2182-02-19 03:59:00 0.8\n", " 2182-02-20 03:31:00 0.7\n", " 2182-02-21 04:55:00 0.9\n", "199979 2182-02-06 09:17:00 1.0\n", " 2182-02-06 14:16:00 3.8\n", "199981 2110-09-24 16:34:00 1.1\n", " 2110-09-24 20:09:00 1.0\n", " 2110-09-25 06:10:00 1.0\n", "199987 2175-05-19 16:30:00 2.3\n", "199988 2169-01-24 12:48:00 1.0\n", " 2169-02-07 01:35:00 1.6\n", " 2169-02-07 11:18:00 1.0\n", " 2169-02-07 16:43:00 1.0\n", " 2169-02-07 22:35:00 1.1\n", " 2169-02-10 05:33:00 1.4\n", "199993 2161-11-12 23:14:00 0.9\n", " 2161-11-13 03:46:00 0.9\n", "199994 2188-07-07 21:23:00 1.0\n", " 2188-07-08 03:09:00 0.7\n", " 2188-07-08 04:13:00 0.6\n", " 2188-07-08 06:20:00 0.7\n", "199998 2119-02-20 10:52:00 1.1\n", " 2119-02-20 12:36:00 1.9\n", " 2119-02-20 13:33:00 2.0\n", " 2119-02-20 13:59:00 2.6\n", " 2119-02-20 20:43:00 1.3\n", "199999 2136-04-04 20:55:00 1.9\n", " 2136-04-06 15:29:00 1.8\n", "\n", "[177439 rows x 1 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_pipeline.set_params(transform__add_level__level_val=label)\n", "all_pipeline.transform(df)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Loop over everything" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import logger" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:18:23) heart rate\n", "(2017-06-03 07:18:23)>> Open Extract\n", "(2017-06-03 07:18:26)<< DONE (3.0s)\n", "(2017-06-03 07:18:26)>> Run Pipeline\n", "(2017-06-03 07:21:19)<< DONE (173.0s)\n", "(2017-06-03 07:21:19)>> Analyze...\n", "((7952939, 1), (7922961, 1), 29978L, 171, '0.3015% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelheart rate
statusknown
variable_typeqn
unitsbeats/min
count7.922961e+06
mean1.026618e+02
std3.552932e+03
min-8.800000e+01
25%7.700000e+01
50%9.200000e+01
75%1.190000e+02
max9.999999e+06
\n", "
" ], "text/plain": [ "label heart rate\n", "status known\n", "variable_type qn\n", "units beats/min\n", "count 7.922961e+06\n", "mean 1.026618e+02\n", "std 3.552932e+03\n", "min -8.800000e+01\n", "25% 7.700000e+01\n", "50% 9.200000e+01\n", "75% 1.190000e+02\n", "max 9.999999e+06" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:21:20)<< DONE (1.0s)\n", "(2017-06-03 07:21:20)>> Joining!\n", "(2017-06-03 07:21:20)<< DONE (0.0s)\n", "(2017-06-03 07:21:20) DONE (177.0s)\n", "(2017-06-03 07:21:20) blood pressure systolic\n", "(2017-06-03 07:21:20)>> Open Extract\n", "(2017-06-03 07:21:23)<< DONE (3.0s)\n", "(2017-06-03 07:21:23)>> Run Pipeline\n", "(2017-06-03 07:25:39)<< DONE (256.0s)\n", "(2017-06-03 07:25:39)>> Analyze...\n", "((6374824, 1), (5974186, 1), 548979L, 174, '0.307% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelblood pressure systolic
statusknown
variable_typeqn
unitsmmHg
count5.825845e+06
mean1.214682e+02
std1.311542e+02
min-6.900000e+01
25%1.040000e+02
50%1.190000e+02
75%1.370000e+02
max1.411460e+05
\n", "
" ], "text/plain": [ "label blood pressure systolic\n", "status known\n", "variable_type qn\n", "units mmHg\n", "count 5.825845e+06\n", "mean 1.214682e+02\n", "std 1.311542e+02\n", "min -6.900000e+01\n", "25% 1.040000e+02\n", "50% 1.190000e+02\n", "75% 1.370000e+02\n", "max 1.411460e+05" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:25:40)<< DONE (1.0s)\n", "(2017-06-03 07:25:40)>> Joining!\n", "(2017-06-03 07:26:28)<< DONE (48.0s)\n", "(2017-06-03 07:26:28) DONE (308.0s)\n", "(2017-06-03 07:26:28) blood pressure diastolic\n", "(2017-06-03 07:26:28)>> Open Extract\n", "(2017-06-03 07:26:31)<< DONE (3.0s)\n", "(2017-06-03 07:26:31)>> Run Pipeline\n", "(2017-06-03 07:30:32)<< DONE (241.0s)\n", "(2017-06-03 07:30:32)>> Analyze...\n", "((6371249, 1), (5976313, 1), 543269L, 170, '0.2999% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelblood pressure diastolic
statusknown
variable_typeqn
unitsmmHg
count5.827980e+06
mean6.115824e+01
std2.350768e+02
min-1.600000e+01
25%5.000000e+01
50%5.900000e+01
75%6.900000e+01
max1.141090e+05
\n", "
" ], "text/plain": [ "label blood pressure diastolic\n", "status known\n", "variable_type qn\n", "units mmHg\n", "count 5.827980e+06\n", "mean 6.115824e+01\n", "std 2.350768e+02\n", "min -1.600000e+01\n", "25% 5.000000e+01\n", "50% 5.900000e+01\n", "75% 6.900000e+01\n", "max 1.141090e+05" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:30:34)<< DONE (2.0s)\n", "(2017-06-03 07:30:34)>> Joining!\n", "(2017-06-03 07:31:25)<< DONE (51.0s)\n", "(2017-06-03 07:31:25) DONE (297.0s)\n", "(2017-06-03 07:31:25) blood pressure mean\n", "(2017-06-03 07:31:25)>> Open Extract\n", "(2017-06-03 07:31:26)<< DONE (1.0s)\n", "(2017-06-03 07:31:26)>> Run Pipeline\n", "(2017-06-03 07:32:39)<< DONE (73.0s)\n", "(2017-06-03 07:32:39)>> Analyze...\n", "((2536271, 1), (2415995, 1), 120276L, 0, '0.0% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelblood pressure mean
statusknown
variable_typeqn
unitsmmHg
count2.415995e+06
mean7.879668e+01
std1.413279e+02
min-1.350000e+02
25%6.700000e+01
50%7.700000e+01
75%8.800000e+01
max1.201300e+05
\n", "
" ], "text/plain": [ "label blood pressure mean\n", "status known\n", "variable_type qn\n", "units mmHg\n", "count 2.415995e+06\n", "mean 7.879668e+01\n", "std 1.413279e+02\n", "min -1.350000e+02\n", "25% 6.700000e+01\n", "50% 7.700000e+01\n", "75% 8.800000e+01\n", "max 1.201300e+05" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:32:40)<< DONE (1.0s)\n", "(2017-06-03 07:32:40)>> Joining!\n", "(2017-06-03 07:33:20)<< DONE (40.0s)\n", "(2017-06-03 07:33:20) DONE (115.0s)\n", "(2017-06-03 07:33:20) respiratory rate\n", "(2017-06-03 07:33:20)>> Open Extract\n", "(2017-06-03 07:33:24)<< DONE (4.0s)\n", "(2017-06-03 07:33:24)>> Run Pipeline\n", "(2017-06-03 07:37:33)<< DONE (249.0s)\n", "(2017-06-03 07:37:33)>> Analyze...\n", "((7810019, 1), (7780015, 1), 5072936L, 172, '0.3035% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelrespiratory rate
statusknown
variable_typeqn
unitsinsp/min
count2.737083e+06
mean2.092347e+01
std1.423811e+03
min0.000000e+00
25%1.600000e+01
50%2.000000e+01
75%2.400000e+01
max2.355555e+06
\n", "
" ], "text/plain": [ "label respiratory rate\n", "status known\n", "variable_type qn\n", "units insp/min\n", "count 2.737083e+06\n", "mean 2.092347e+01\n", "std 1.423811e+03\n", "min 0.000000e+00\n", "25% 1.600000e+01\n", "50% 2.000000e+01\n", "75% 2.400000e+01\n", "max 2.355555e+06" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:37:35)<< DONE (2.0s)\n", "(2017-06-03 07:37:35)>> Joining!\n", "(2017-06-03 07:38:29)<< DONE (54.0s)\n", "(2017-06-03 07:38:29) DONE (309.0s)\n", "(2017-06-03 07:38:29) temperature body\n", "(2017-06-03 07:38:29)>> Open Extract\n", "(2017-06-03 07:38:31)<< DONE (2.0s)\n", "(2017-06-03 07:38:31)>> Run Pipeline\n", "(2017-06-03 07:39:35)<< DONE (64.0s)\n", "(2017-06-03 07:39:35)>> Analyze...\n", "((1751447, 1), (1731794, 1), 461152L, 156, '0.3189% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
statusknown
variable_typeqn
unitsdegF
count1.290295e+06
mean9.850479e+01
std9.563484e+00
min-9.990000e+01
25%9.760000e+01
50%9.850000e+01
75%9.950000e+01
max9.637000e+03
\n", "
" ], "text/plain": [ "label temperature body\n", "status known\n", "variable_type qn\n", "units degF\n", "count 1.290295e+06\n", "mean 9.850479e+01\n", "std 9.563484e+00\n", "min -9.990000e+01\n", "25% 9.760000e+01\n", "50% 9.850000e+01\n", "75% 9.950000e+01\n", "max 9.637000e+03" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:39:36)<< DONE (1.0s)\n", "(2017-06-03 07:39:36)>> Joining!\n", "(2017-06-03 07:40:14)<< DONE (38.0s)\n", "(2017-06-03 07:40:14) DONE (105.0s)\n", "(2017-06-03 07:40:14) oxygen saturation pulse oximetry\n", "(2017-06-03 07:40:14)>> Open Extract\n", "(2017-06-03 07:40:17)<< DONE (3.0s)\n", "(2017-06-03 07:40:17)>> Run Pipeline\n", "(2017-06-03 07:42:42)<< DONE (145.0s)\n", "(2017-06-03 07:42:42)>> Analyze...\n", "((6099827, 1), (6073019, 1), 26808L, 163, '0.3326% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeloxygen saturation pulse oximetry
statusknown
variable_typeqn
unitspercent
count6.073019e+06
mean9.885942e+01
std2.942035e+03
min0.000000e+00
25%9.600000e+01
50%9.800000e+01
75%9.900000e+01
max6.363333e+06
\n", "
" ], "text/plain": [ "label oxygen saturation pulse oximetry\n", "status known\n", "variable_type qn\n", "units percent\n", "count 6.073019e+06\n", "mean 9.885942e+01\n", "std 2.942035e+03\n", "min 0.000000e+00\n", "25% 9.600000e+01\n", "50% 9.800000e+01\n", "75% 9.900000e+01\n", "max 6.363333e+06" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:42:43)<< DONE (1.0s)\n", "(2017-06-03 07:42:43)>> Joining!\n", "(2017-06-03 07:43:35)<< DONE (52.0s)\n", "(2017-06-03 07:43:35) DONE (201.0s)\n", "(2017-06-03 07:43:35) weight body\n", "(2017-06-03 07:43:35)>> Open Extract\n", "(2017-06-03 07:43:36)<< DONE (1.0s)\n", "(2017-06-03 07:43:36)>> Run Pipeline\n", "(2017-06-03 07:43:40)<< DONE (4.0s)\n", "(2017-06-03 07:43:40)>> Analyze...\n", "((95425, 1), (94457, 1), 1956L, 158, '0.4958% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelweight body
statusknown
variable_typeqn
unitskg
count9.346900e+04
mean2.676828e+02
std3.908673e+04
min0.000000e+00
25%6.980000e+01
50%8.340000e+01
75%9.890000e+01
max8.654765e+06
\n", "
" ], "text/plain": [ "label weight body\n", "status known\n", "variable_type qn\n", "units kg\n", "count 9.346900e+04\n", "mean 2.676828e+02\n", "std 3.908673e+04\n", "min 0.000000e+00\n", "25% 6.980000e+01\n", "50% 8.340000e+01\n", "75% 9.890000e+01\n", "max 8.654765e+06" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:43:40)<< DONE (0.0s)\n", "(2017-06-03 07:43:40)>> Joining!\n", "(2017-06-03 07:44:15)<< DONE (35.0s)\n", "(2017-06-03 07:44:15) DONE (40.0s)\n", "(2017-06-03 07:44:15) output urine\n", "(2017-06-03 07:44:15)>> Open Extract\n", "(2017-06-03 07:44:16)<< DONE (1.0s)\n", "(2017-06-03 07:44:16)>> Run Pipeline\n", "(2017-06-03 07:47:57)<< DONE (221.0s)\n", "(2017-06-03 07:47:57)>> Analyze...\n", "((3644639, 1), (3624029, 1), 417242L, 92, '0.1758% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeloutput urine
statusknown
variable_typeqn
unitsmL
count3.216363e+06
mean1.265574e+02
std2.577291e+03
min-1.500000e+03
25%4.200000e+01
50%8.000000e+01
75%1.600000e+02
max4.555555e+06
\n", "
" ], "text/plain": [ "label output urine\n", "status known\n", "variable_type qn\n", "units mL\n", "count 3.216363e+06\n", "mean 1.265574e+02\n", "std 2.577291e+03\n", "min -1.500000e+03\n", "25% 4.200000e+01\n", "50% 8.000000e+01\n", "75% 1.600000e+02\n", "max 4.555555e+06" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:47:58)<< DONE (1.0s)\n", "(2017-06-03 07:47:58)>> Joining!\n", "(2017-06-03 07:48:43)<< DONE (45.0s)\n", "(2017-06-03 07:48:43) DONE (268.0s)\n", "(2017-06-03 07:48:43) glasgow coma scale motor\n", "(2017-06-03 07:48:43)>> Open Extract\n", "(2017-06-03 07:48:43)<< DONE (0.0s)\n", "(2017-06-03 07:48:43)>> Run Pipeline\n", "(2017-06-03 07:49:29)<< DONE (46.0s)\n", "(2017-06-03 07:49:29)>> Analyze...\n", "((952565, 1), (949198, 1), 3367L, 1, '0.0037% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelglasgow coma scale motor
statusknown
variable_typeord
unitsno_units
count949198
unique6
topObeys commands
freq642879
\n", "
" ], "text/plain": [ "label glasgow coma scale motor\n", "status known\n", "variable_type ord\n", "units no_units\n", "count 949198\n", "unique 6\n", "top Obeys commands\n", "freq 642879" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:49:29)<< DONE (0.0s)\n", "(2017-06-03 07:49:29)>> Joining!\n", "(2017-06-03 07:50:06)<< DONE (37.0s)\n", "(2017-06-03 07:50:06) DONE (83.0s)\n", "(2017-06-03 07:50:06) glasgow coma scale eye opening\n", "(2017-06-03 07:50:06)>> Open Extract\n", "(2017-06-03 07:50:07)<< DONE (1.0s)\n", "(2017-06-03 07:50:07)>> Run Pipeline\n", "(2017-06-03 07:50:50)<< DONE (43.0s)\n", "(2017-06-03 07:50:50)>> Analyze...\n", "((956672, 1), (953595, 1), 3077L, 1, '0.0037% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelglasgow coma scale eye opening
statusknown
variable_typeord
unitsno_units
count953595
unique4
topEyes open spontaneously
freq588057
\n", "
" ], "text/plain": [ "label glasgow coma scale eye opening\n", "status known\n", "variable_type ord\n", "units no_units\n", "count 953595\n", "unique 4\n", "top Eyes open spontaneously\n", "freq 588057" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:50:50)<< DONE (0.0s)\n", "(2017-06-03 07:50:50)>> Joining!\n", "(2017-06-03 07:51:28)<< DONE (38.0s)\n", "(2017-06-03 07:51:28) DONE (82.0s)\n", "(2017-06-03 07:51:28) glasgow coma scale verbal\n", "(2017-06-03 07:51:28)>> Open Extract\n", "(2017-06-03 07:51:29)<< DONE (1.0s)\n", "(2017-06-03 07:51:29)>> Run Pipeline\n", "(2017-06-03 07:52:14)<< DONE (45.0s)\n", "(2017-06-03 07:52:14)>> Analyze...\n", "((954700, 1), (950913, 1), 3787L, 2, '0.0074% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelglasgow coma scale verbal
statusknown
variable_typeord
unitsno_units
count950913
unique5
topNo verbal response (>2 yrs); no vocal response...
freq465193
\n", "
" ], "text/plain": [ "label glasgow coma scale verbal\n", "status known\n", "variable_type ord\n", "units no_units\n", "count 950913\n", "unique 5\n", "top No verbal response (>2 yrs); no vocal response...\n", "freq 465193" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 07:52:14)<< DONE (0.0s)\n", "(2017-06-03 07:52:14)>> Joining!\n", "(2017-06-03 07:52:51)<< DONE (37.0s)\n", "(2017-06-03 07:52:51) DONE (83.0s)\n", "(2017-06-03 07:52:51) normal saline\n", "(2017-06-03 07:52:51)>> Open Extract\n", "(2017-06-03 07:52:51)<< DONE (0.0s)\n", "(2017-06-03 07:52:51)>> Run Pipeline\n", "(2017-06-03 08:01:38)<< DONE (527.0s)\n", "(2017-06-03 08:01:38)>> Analyze...\n", "((817373, 1), (504874, 2), 265615L, 8, '0.0405% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelnormal saline
statusknown
variable_typeqn
unitsmLmL/hr
count49173.000000456484.000000
mean389.56317875.626948
std380.287765217.512467
min0.000000-906.000000
25%100.0000006.000000
50%250.00000015.000000
75%500.00000062.204886
max11000.00000051947.999400
\n", "
" ], "text/plain": [ "label normal saline \n", "status known \n", "variable_type qn \n", "units mL mL/hr\n", "count 49173.000000 456484.000000\n", "mean 389.563178 75.626948\n", "std 380.287765 217.512467\n", "min 0.000000 -906.000000\n", "25% 100.000000 6.000000\n", "50% 250.000000 15.000000\n", "75% 500.000000 62.204886\n", "max 11000.000000 51947.999400" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 08:01:38)<< DONE (0.0s)\n", "(2017-06-03 08:01:38)>> Joining!\n", "(2017-06-03 08:02:15)<< DONE (37.0s)\n", "(2017-06-03 08:02:15) DONE (564.0s)\n", "(2017-06-03 08:02:15) lactated ringers\n", "(2017-06-03 08:02:15)>> Open Extract\n", "(2017-06-03 08:02:15)<< DONE (0.0s)\n", "(2017-06-03 08:02:15)>> Run Pipeline\n", "(2017-06-03 08:02:57)<< DONE (42.0s)\n", "(2017-06-03 08:02:57)>> Analyze...\n", "((504306, 1), (254175, 2), 19193L, 18, '0.1085% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labellactated ringers
statusknown
variable_typeqn
unitsmLmL/hr
count248510.0000002161.000000
mean204.109607289.877760
std338.277593418.153805
min0.0000000.000000
25%15.0000000.000000
50%100.00000099.994818
75%200.000000499.999980
max60000.0000003923.333176
\n", "
" ], "text/plain": [ "label lactated ringers \n", "status known \n", "variable_type qn \n", "units mL mL/hr\n", "count 248510.000000 2161.000000\n", "mean 204.109607 289.877760\n", "std 338.277593 418.153805\n", "min 0.000000 0.000000\n", "25% 15.000000 0.000000\n", "50% 100.000000 99.994818\n", "75% 200.000000 499.999980\n", "max 60000.000000 3923.333176" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 08:02:57)<< DONE (0.0s)\n", "(2017-06-03 08:02:57)>> Joining!\n", "(2017-06-03 08:03:34)<< DONE (37.0s)\n", "(2017-06-03 08:03:34) DONE (79.0s)\n", "(2017-06-03 08:03:34) norepinephrine\n", "(2017-06-03 08:03:34)>> Open Extract\n", "(2017-06-03 08:03:34)<< DONE (0.0s)\n", "(2017-06-03 08:03:34)>> Run Pipeline\n", "(2017-06-03 08:05:19)<< DONE (105.0s)\n", "(2017-06-03 08:05:19)>> Analyze...\n", "((1136938, 1), (389986, 2), 331666L, 17, '0.231% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelnorepinephrine
statusknown
variable_typeqn
unitsmcg/kg/minmcg/min
count286781.00000014226.000000
mean0.1412079.687762
std0.34321214.889317
min0.0000000.000000
25%0.0350002.666667
50%0.0800006.000000
75%0.18000012.000000
max50.000000250.000000
\n", "
" ], "text/plain": [ "label norepinephrine \n", "status known \n", "variable_type qn \n", "units mcg/kg/min mcg/min\n", "count 286781.000000 14226.000000\n", "mean 0.141207 9.687762\n", "std 0.343212 14.889317\n", "min 0.000000 0.000000\n", "25% 0.035000 2.666667\n", "50% 0.080000 6.000000\n", "75% 0.180000 12.000000\n", "max 50.000000 250.000000" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 08:05:20)<< DONE (1.0s)\n", "(2017-06-03 08:05:20)>> Joining!\n", "(2017-06-03 08:05:58)<< DONE (38.0s)\n", "(2017-06-03 08:05:58) DONE (144.0s)\n", "(2017-06-03 08:05:58) vasopressin\n", "(2017-06-03 08:05:58)>> Open Extract\n", "(2017-06-03 08:05:58)<< DONE (0.0s)\n", "(2017-06-03 08:05:58)>> Run Pipeline\n", "(2017-06-03 08:06:20)<< DONE (22.0s)\n", "(2017-06-03 08:06:20)>> Analyze...\n", "((339184, 1), (110333, 2), 16608L, 8, '0.3406% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelvasopressin
statusknown
variable_typeqn
unitsunitsunits/min
count57371.00000098872.000000
mean2.5173660.992521
std4.8428541.291858
min0.0000000.000000
25%1.8000000.040000
50%2.4000000.044000
75%2.4000002.400000
max722.40000050.000000
\n", "
" ], "text/plain": [ "label vasopressin \n", "status known \n", "variable_type qn \n", "units units units/min\n", "count 57371.000000 98872.000000\n", "mean 2.517366 0.992521\n", "std 4.842854 1.291858\n", "min 0.000000 0.000000\n", "25% 1.800000 0.040000\n", "50% 2.400000 0.044000\n", "75% 2.400000 2.400000\n", "max 722.400000 50.000000" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 08:06:20)<< DONE (0.0s)\n", "(2017-06-03 08:06:20)>> Joining!\n", "(2017-06-03 08:06:56)<< DONE (36.0s)\n", "(2017-06-03 08:06:56) DONE (58.0s)\n", "(2017-06-03 08:06:56) hemoglobin\n", "(2017-06-03 08:06:56)>> Open Extract\n", "(2017-06-03 08:06:56)<< DONE (0.0s)\n", "(2017-06-03 08:06:56)>> Run Pipeline\n", "(2017-06-03 08:07:52)<< DONE (56.0s)\n", "(2017-06-03 08:07:52)>> Analyze...\n", "((1167921, 1), (671087, 1), 641393L, 7, '0.0123% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelhemoglobin
statusknown
variable_typeqn
unitsg/dL
count526528.000000
mean10.351784
std1.980592
min0.000000
25%9.000000
50%10.100000
75%11.400000
max130.000000
\n", "
" ], "text/plain": [ "label hemoglobin\n", "status known\n", "variable_type qn\n", "units g/dL\n", "count 526528.000000\n", "mean 10.351784\n", "std 1.980592\n", "min 0.000000\n", "25% 9.000000\n", "50% 10.100000\n", "75% 11.400000\n", "max 130.000000" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 08:07:52)<< DONE (0.0s)\n", "(2017-06-03 08:07:52)>> Joining!\n", "(2017-06-03 08:08:36)<< DONE (44.0s)\n", "(2017-06-03 08:08:36) DONE (100.0s)\n", "(2017-06-03 08:08:36) lactate\n", "(2017-06-03 08:08:36)>> Open Extract\n", "(2017-06-03 08:08:37)<< DONE (1.0s)\n", "(2017-06-03 08:08:37)>> Run Pipeline\n", "(2017-06-03 08:08:52)<< DONE (15.0s)\n", "(2017-06-03 08:08:52)>> Analyze...\n", "((393608, 1), (177439, 1), 216244L, 33, '0.0962% records')\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labellactate
statusknown
variable_typeqn
unitsmmol/L
count177364.000000
mean8.286749
std2374.466306
min0.000000
25%1.200000
50%1.800000
75%2.900000
max999999.000000
\n", "
" ], "text/plain": [ "label lactate\n", "status known\n", "variable_type qn\n", "units mmol/L\n", "count 177364.000000\n", "mean 8.286749\n", "std 2374.466306\n", "min 0.000000\n", "25% 1.200000\n", "50% 1.800000\n", "75% 2.900000\n", "max 999999.000000" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-03 08:08:52)<< DONE (0.0s)\n", "(2017-06-03 08:08:52)>> Joining!\n", "(2017-06-03 08:09:35)<< DONE (43.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelheart rateblood pressure systolicblood pressure diastolicblood pressure meanrespiratory ratetemperature bodyoxygen saturation pulse oximetryweight bodyoutput urineglasgow coma scale motor...normal salinelactated ringersnorepinephrinevasopressinhemoglobinlactate
statusknownknownknownknownknownknownknownknownknownknown...knownknownknownknownknownknown
variable_typeqnqnqnqnqnqnqnqnqnord...qnqnqnqnqnqn
unitsbeats/minmmHgmmHgmmHginsp/mindegFpercentkgmLno_units...mLmL/hrmLmL/hrmcg/kg/minmcg/minunitsunits/ming/dLmmol/L
iddatetime
1000012117-09-11 09:22:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaN13.0NaN
2117-09-11 09:32:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaN1.9
2117-09-11 12:50:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN5.000000NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 12:55:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN6.996487NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 12:57:00122.0NaNNaNNaN14.0NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:00:00118.0NaNNaNNaN22.0NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:01:00NaN192.0100.0122.0NaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:48:00NaNNaNNaNNaNNaN98.0NaNNaNNaNNaN...NaN499.999980NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:49:00NaNNaNNaNNaNNaNNaNNaNNaN300.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:50:00118.0NaNNaNNaN22.0NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", "

10 rows × 22 columns

\n", "
" ], "text/plain": [ "label heart rate blood pressure systolic \\\n", "status known known \n", "variable_type qn qn \n", "units beats/min mmHg \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 122.0 NaN \n", " 2117-09-11 13:00:00 118.0 NaN \n", " 2117-09-11 13:01:00 NaN 192.0 \n", " 2117-09-11 13:48:00 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 118.0 NaN \n", "\n", "label blood pressure diastolic blood pressure mean \\\n", "status known known \n", "variable_type qn qn \n", "units mmHg mmHg \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN \n", " 2117-09-11 13:01:00 100.0 122.0 \n", " 2117-09-11 13:48:00 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN \n", "\n", "label respiratory rate temperature body \\\n", "status known known \n", "variable_type qn qn \n", "units insp/min degF \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 14.0 NaN \n", " 2117-09-11 13:00:00 22.0 NaN \n", " 2117-09-11 13:01:00 NaN NaN \n", " 2117-09-11 13:48:00 NaN 98.0 \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 22.0 NaN \n", "\n", "label oxygen saturation pulse oximetry weight body \\\n", "status known known \n", "variable_type qn qn \n", "units percent kg \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN \n", " 2117-09-11 13:01:00 NaN NaN \n", " 2117-09-11 13:48:00 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN \n", "\n", "label output urine glasgow coma scale motor ... \\\n", "status known known ... \n", "variable_type qn ord ... \n", "units mL no_units ... \n", "id datetime ... \n", "100001 2117-09-11 09:22:00 NaN NaN ... \n", " 2117-09-11 09:32:00 NaN NaN ... \n", " 2117-09-11 12:50:00 NaN NaN ... \n", " 2117-09-11 12:55:00 NaN NaN ... \n", " 2117-09-11 12:57:00 NaN NaN ... \n", " 2117-09-11 13:00:00 NaN NaN ... \n", " 2117-09-11 13:01:00 NaN NaN ... \n", " 2117-09-11 13:48:00 NaN NaN ... \n", " 2117-09-11 13:49:00 300.0 NaN ... \n", " 2117-09-11 13:50:00 NaN NaN ... \n", "\n", "label normal saline lactated ringers \\\n", "status known known \n", "variable_type qn qn \n", "units mL mL/hr mL mL/hr \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN NaN NaN \n", " 2117-09-11 12:50:00 NaN 5.000000 NaN NaN \n", " 2117-09-11 12:55:00 NaN 6.996487 NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN NaN NaN \n", " 2117-09-11 13:01:00 NaN NaN NaN NaN \n", " 2117-09-11 13:48:00 NaN 499.999980 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN NaN NaN \n", "\n", "label norepinephrine vasopressin \\\n", "status known known \n", "variable_type qn qn \n", "units mcg/kg/min mcg/min units units/min \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN NaN NaN \n", " 2117-09-11 13:01:00 NaN NaN NaN NaN \n", " 2117-09-11 13:48:00 NaN NaN NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN NaN NaN \n", "\n", "label hemoglobin lactate \n", "status known known \n", "variable_type qn qn \n", "units g/dL mmol/L \n", "id datetime \n", "100001 2117-09-11 09:22:00 13.0 NaN \n", " 2117-09-11 09:32:00 NaN 1.9 \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN \n", " 2117-09-11 13:01:00 NaN NaN \n", " 2117-09-11 13:48:00 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN \n", "\n", "[10 rows x 22 columns]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "reload(logger)\n", "simple_data = data_dict.get_panel_defintions(12) #12 is \"simple data\"\n", "labels = simple_data.label.unique().tolist()\n", "\n", "df_final = None\n", "for label in labels:\n", " logger.log(label,new_level=True)\n", " logger.log('Open Extract')\n", " df_extract = utils.open_df(hdf5_fname,'extract/{}'.format(label))\n", "\n", " logger.log('Run Pipeline')\n", " all_pipeline.set_params(transform__add_level__level_val=label)\n", " df = all_pipeline.transform(df_extract)\n", " \n", " logger.log('Analyze...')\n", " print utils.data_loss(df_extract.set_index('id').value.to_frame(),df)\n", " display(df.describe())\n", "\n", " logger.log('Joining!')\n", "\n", " if df_final is None: df_final = df\n", " else: \n", " df_final = df_final.join(df,how='outer')\n", " del df\n", " logger.end_log_level()\n", "\n", "df_final.head(10)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelheart rateblood pressure systolicblood pressure diastolicblood pressure meanrespiratory ratetemperature bodyoxygen saturation pulse oximetryweight bodyoutput urineglasgow coma scale motor...normal salinelactated ringersnorepinephrinevasopressinhemoglobinlactate
statusknownknownknownknownknownknownknownknownknownknown...knownknownknownknownknownknown
variable_typeqnqnqnqnqnqnqnqnqnord...qnqnqnqnqnqn
unitsbeats/minmmHgmmHgmmHginsp/mindegFpercentkgmLno_units...mLmL/hrmLmL/hrmcg/kg/minmcg/minunitsunits/ming/dLmmol/L
iddatetime
1000012117-09-11 09:22:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaN13.0NaN
2117-09-11 09:32:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaN1.9
2117-09-11 12:50:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN5.000000NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 12:55:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN6.996487NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 12:57:00122.0NaNNaNNaN14.0NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:00:00118.0NaNNaNNaN22.0NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:01:00NaN192.0100.0122.0NaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:48:00NaNNaNNaNNaNNaN98.0NaNNaNNaNNaN...NaN499.999980NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:49:00NaNNaNNaNNaNNaNNaNNaNNaN300.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 13:50:00118.0NaNNaNNaN22.0NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 14:00:00118.0165.085.0103.015.0NaNNaNNaNNaNNaN...NaN8.990600NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 15:00:00110.0119.064.076.015.0NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 15:48:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN999.999960NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 15:59:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN199.999998NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 16:00:00104.0169.084.0105.016.097.9NaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 16:02:00NaNNaNNaNNaNNaNNaN100.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 16:11:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN40.000000NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 16:12:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN4.992476NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 17:00:00101.0110.062.075.015.0NaN97.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 18:00:00112.0170.086.0106.016.0NaN98.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 18:34:00NaNNaNNaNNaNNaNNaNNaNNaN400.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 19:00:00108.0179.088.0110.0NaN99.5NaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 19:31:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaN11.0NaN
2117-09-11 20:00:00116.0183.091.0114.0NaNNaN97.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 21:00:00117.0189.098.0121.0NaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 21:12:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN3.396080NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 21:16:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaN5.012718NaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 22:00:00124.0180.088.0110.0NaN100.0NaNNaN200.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 22:10:00NaNNaNNaNNaNNaNNaN97.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2117-09-11 22:25:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...500.0NaNNaNNaNNaNNaNNaNNaNNaNNaN
.....................................................................
1999992136-04-09 09:00:0079.0130.048.069.025.0NaN95.0NaN350.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 10:00:0081.0123.064.080.024.0NaN94.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 11:00:0089.0115.057.070.016.0NaN94.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 12:00:0072.0119.045.065.024.098.393.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 13:00:0082.0156.058.078.022.0NaN94.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 14:00:0074.0128.053.072.028.0NaN93.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 15:00:0069.0129.049.069.026.0NaN93.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 16:00:0071.0133.046.070.027.098.094.0NaN400.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 17:00:0068.0150.055.079.024.0NaN94.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 18:00:0090.0146.053.079.024.0NaN93.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 19:00:0078.0134.050.072.026.0NaN93.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 20:00:0082.0142.052.075.023.097.495.0NaN480.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 21:00:0081.0132.050.070.029.0NaN93.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 22:00:0072.0112.048.062.025.0NaN97.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-09 23:00:0065.0129.042.066.024.0NaN96.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 00:00:0080.0NaNNaNNaN25.097.494.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 00:01:00NaN128.059.075.0NaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 01:00:0080.0124.059.075.026.0NaN95.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 02:00:0076.0130.049.068.026.0NaN93.0NaN350.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 02:28:00NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaN12.6NaN
2136-04-10 03:00:0072.0126.037.061.027.097.491.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 04:00:0066.0126.031.052.026.0NaN95.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 05:00:0066.0129.039.063.027.0NaN96.0NaN330.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 06:00:0065.0143.047.071.019.0NaN92.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 07:00:0079.0NaNNaNNaN28.0NaN92.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 08:00:0069.0NaNNaNNaN24.098.196.0NaN200.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 09:00:0079.0NaNNaNNaN24.0NaN93.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 09:10:00NaN128.081.089.0NaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 10:00:0072.0NaNNaNNaN26.0NaN97.0NaN200.0NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2136-04-10 11:00:0072.0144.0123.0128.024.0NaN96.0NaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", "

10071464 rows × 22 columns

\n", "
" ], "text/plain": [ "label heart rate blood pressure systolic \\\n", "status known known \n", "variable_type qn qn \n", "units beats/min mmHg \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 122.0 NaN \n", " 2117-09-11 13:00:00 118.0 NaN \n", " 2117-09-11 13:01:00 NaN 192.0 \n", " 2117-09-11 13:48:00 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 118.0 NaN \n", " 2117-09-11 14:00:00 118.0 165.0 \n", " 2117-09-11 15:00:00 110.0 119.0 \n", " 2117-09-11 15:48:00 NaN NaN \n", " 2117-09-11 15:59:00 NaN NaN \n", " 2117-09-11 16:00:00 104.0 169.0 \n", " 2117-09-11 16:02:00 NaN NaN \n", " 2117-09-11 16:11:00 NaN NaN \n", " 2117-09-11 16:12:00 NaN NaN \n", " 2117-09-11 17:00:00 101.0 110.0 \n", " 2117-09-11 18:00:00 112.0 170.0 \n", " 2117-09-11 18:34:00 NaN NaN \n", " 2117-09-11 19:00:00 108.0 179.0 \n", " 2117-09-11 19:31:00 NaN NaN \n", " 2117-09-11 20:00:00 116.0 183.0 \n", " 2117-09-11 21:00:00 117.0 189.0 \n", " 2117-09-11 21:12:00 NaN NaN \n", " 2117-09-11 21:16:00 NaN NaN \n", " 2117-09-11 22:00:00 124.0 180.0 \n", " 2117-09-11 22:10:00 NaN NaN \n", " 2117-09-11 22:25:00 NaN NaN \n", "... ... ... \n", "199999 2136-04-09 09:00:00 79.0 130.0 \n", " 2136-04-09 10:00:00 81.0 123.0 \n", " 2136-04-09 11:00:00 89.0 115.0 \n", " 2136-04-09 12:00:00 72.0 119.0 \n", " 2136-04-09 13:00:00 82.0 156.0 \n", " 2136-04-09 14:00:00 74.0 128.0 \n", " 2136-04-09 15:00:00 69.0 129.0 \n", " 2136-04-09 16:00:00 71.0 133.0 \n", " 2136-04-09 17:00:00 68.0 150.0 \n", " 2136-04-09 18:00:00 90.0 146.0 \n", " 2136-04-09 19:00:00 78.0 134.0 \n", " 2136-04-09 20:00:00 82.0 142.0 \n", " 2136-04-09 21:00:00 81.0 132.0 \n", " 2136-04-09 22:00:00 72.0 112.0 \n", " 2136-04-09 23:00:00 65.0 129.0 \n", " 2136-04-10 00:00:00 80.0 NaN \n", " 2136-04-10 00:01:00 NaN 128.0 \n", " 2136-04-10 01:00:00 80.0 124.0 \n", " 2136-04-10 02:00:00 76.0 130.0 \n", " 2136-04-10 02:28:00 NaN NaN \n", " 2136-04-10 03:00:00 72.0 126.0 \n", " 2136-04-10 04:00:00 66.0 126.0 \n", " 2136-04-10 05:00:00 66.0 129.0 \n", " 2136-04-10 06:00:00 65.0 143.0 \n", " 2136-04-10 07:00:00 79.0 NaN \n", " 2136-04-10 08:00:00 69.0 NaN \n", " 2136-04-10 09:00:00 79.0 NaN \n", " 2136-04-10 09:10:00 NaN 128.0 \n", " 2136-04-10 10:00:00 72.0 NaN \n", " 2136-04-10 11:00:00 72.0 144.0 \n", "\n", "label blood pressure diastolic blood pressure mean \\\n", "status known known \n", "variable_type qn qn \n", "units mmHg mmHg \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN \n", " 2117-09-11 13:01:00 100.0 122.0 \n", " 2117-09-11 13:48:00 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN \n", " 2117-09-11 14:00:00 85.0 103.0 \n", " 2117-09-11 15:00:00 64.0 76.0 \n", " 2117-09-11 15:48:00 NaN NaN \n", " 2117-09-11 15:59:00 NaN NaN \n", " 2117-09-11 16:00:00 84.0 105.0 \n", " 2117-09-11 16:02:00 NaN NaN \n", " 2117-09-11 16:11:00 NaN NaN \n", " 2117-09-11 16:12:00 NaN NaN \n", " 2117-09-11 17:00:00 62.0 75.0 \n", " 2117-09-11 18:00:00 86.0 106.0 \n", " 2117-09-11 18:34:00 NaN NaN \n", " 2117-09-11 19:00:00 88.0 110.0 \n", " 2117-09-11 19:31:00 NaN NaN \n", " 2117-09-11 20:00:00 91.0 114.0 \n", " 2117-09-11 21:00:00 98.0 121.0 \n", " 2117-09-11 21:12:00 NaN NaN \n", " 2117-09-11 21:16:00 NaN NaN \n", " 2117-09-11 22:00:00 88.0 110.0 \n", " 2117-09-11 22:10:00 NaN NaN \n", " 2117-09-11 22:25:00 NaN NaN \n", "... ... ... \n", "199999 2136-04-09 09:00:00 48.0 69.0 \n", " 2136-04-09 10:00:00 64.0 80.0 \n", " 2136-04-09 11:00:00 57.0 70.0 \n", " 2136-04-09 12:00:00 45.0 65.0 \n", " 2136-04-09 13:00:00 58.0 78.0 \n", " 2136-04-09 14:00:00 53.0 72.0 \n", " 2136-04-09 15:00:00 49.0 69.0 \n", " 2136-04-09 16:00:00 46.0 70.0 \n", " 2136-04-09 17:00:00 55.0 79.0 \n", " 2136-04-09 18:00:00 53.0 79.0 \n", " 2136-04-09 19:00:00 50.0 72.0 \n", " 2136-04-09 20:00:00 52.0 75.0 \n", " 2136-04-09 21:00:00 50.0 70.0 \n", " 2136-04-09 22:00:00 48.0 62.0 \n", " 2136-04-09 23:00:00 42.0 66.0 \n", " 2136-04-10 00:00:00 NaN NaN \n", " 2136-04-10 00:01:00 59.0 75.0 \n", " 2136-04-10 01:00:00 59.0 75.0 \n", " 2136-04-10 02:00:00 49.0 68.0 \n", " 2136-04-10 02:28:00 NaN NaN \n", " 2136-04-10 03:00:00 37.0 61.0 \n", " 2136-04-10 04:00:00 31.0 52.0 \n", " 2136-04-10 05:00:00 39.0 63.0 \n", " 2136-04-10 06:00:00 47.0 71.0 \n", " 2136-04-10 07:00:00 NaN NaN \n", " 2136-04-10 08:00:00 NaN NaN \n", " 2136-04-10 09:00:00 NaN NaN \n", " 2136-04-10 09:10:00 81.0 89.0 \n", " 2136-04-10 10:00:00 NaN NaN \n", " 2136-04-10 11:00:00 123.0 128.0 \n", "\n", "label respiratory rate temperature body \\\n", "status known known \n", "variable_type qn qn \n", "units insp/min degF \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 14.0 NaN \n", " 2117-09-11 13:00:00 22.0 NaN \n", " 2117-09-11 13:01:00 NaN NaN \n", " 2117-09-11 13:48:00 NaN 98.0 \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 22.0 NaN \n", " 2117-09-11 14:00:00 15.0 NaN \n", " 2117-09-11 15:00:00 15.0 NaN \n", " 2117-09-11 15:48:00 NaN NaN \n", " 2117-09-11 15:59:00 NaN NaN \n", " 2117-09-11 16:00:00 16.0 97.9 \n", " 2117-09-11 16:02:00 NaN NaN \n", " 2117-09-11 16:11:00 NaN NaN \n", " 2117-09-11 16:12:00 NaN NaN \n", " 2117-09-11 17:00:00 15.0 NaN \n", " 2117-09-11 18:00:00 16.0 NaN \n", " 2117-09-11 18:34:00 NaN NaN \n", " 2117-09-11 19:00:00 NaN 99.5 \n", " 2117-09-11 19:31:00 NaN NaN \n", " 2117-09-11 20:00:00 NaN NaN \n", " 2117-09-11 21:00:00 NaN NaN \n", " 2117-09-11 21:12:00 NaN NaN \n", " 2117-09-11 21:16:00 NaN NaN \n", " 2117-09-11 22:00:00 NaN 100.0 \n", " 2117-09-11 22:10:00 NaN NaN \n", " 2117-09-11 22:25:00 NaN NaN \n", "... ... ... \n", "199999 2136-04-09 09:00:00 25.0 NaN \n", " 2136-04-09 10:00:00 24.0 NaN \n", " 2136-04-09 11:00:00 16.0 NaN \n", " 2136-04-09 12:00:00 24.0 98.3 \n", " 2136-04-09 13:00:00 22.0 NaN \n", " 2136-04-09 14:00:00 28.0 NaN \n", " 2136-04-09 15:00:00 26.0 NaN \n", " 2136-04-09 16:00:00 27.0 98.0 \n", " 2136-04-09 17:00:00 24.0 NaN \n", " 2136-04-09 18:00:00 24.0 NaN \n", " 2136-04-09 19:00:00 26.0 NaN \n", " 2136-04-09 20:00:00 23.0 97.4 \n", " 2136-04-09 21:00:00 29.0 NaN \n", " 2136-04-09 22:00:00 25.0 NaN \n", " 2136-04-09 23:00:00 24.0 NaN \n", " 2136-04-10 00:00:00 25.0 97.4 \n", " 2136-04-10 00:01:00 NaN NaN \n", " 2136-04-10 01:00:00 26.0 NaN \n", " 2136-04-10 02:00:00 26.0 NaN \n", " 2136-04-10 02:28:00 NaN NaN \n", " 2136-04-10 03:00:00 27.0 97.4 \n", " 2136-04-10 04:00:00 26.0 NaN \n", " 2136-04-10 05:00:00 27.0 NaN \n", " 2136-04-10 06:00:00 19.0 NaN \n", " 2136-04-10 07:00:00 28.0 NaN \n", " 2136-04-10 08:00:00 24.0 98.1 \n", " 2136-04-10 09:00:00 24.0 NaN \n", " 2136-04-10 09:10:00 NaN NaN \n", " 2136-04-10 10:00:00 26.0 NaN \n", " 2136-04-10 11:00:00 24.0 NaN \n", "\n", "label oxygen saturation pulse oximetry weight body \\\n", "status known known \n", "variable_type qn qn \n", "units percent kg \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN \n", " 2117-09-11 13:01:00 NaN NaN \n", " 2117-09-11 13:48:00 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN \n", " 2117-09-11 14:00:00 NaN NaN \n", " 2117-09-11 15:00:00 NaN NaN \n", " 2117-09-11 15:48:00 NaN NaN \n", " 2117-09-11 15:59:00 NaN NaN \n", " 2117-09-11 16:00:00 NaN NaN \n", " 2117-09-11 16:02:00 100.0 NaN \n", " 2117-09-11 16:11:00 NaN NaN \n", " 2117-09-11 16:12:00 NaN NaN \n", " 2117-09-11 17:00:00 97.0 NaN \n", " 2117-09-11 18:00:00 98.0 NaN \n", " 2117-09-11 18:34:00 NaN NaN \n", " 2117-09-11 19:00:00 NaN NaN \n", " 2117-09-11 19:31:00 NaN NaN \n", " 2117-09-11 20:00:00 97.0 NaN \n", " 2117-09-11 21:00:00 NaN NaN \n", " 2117-09-11 21:12:00 NaN NaN \n", " 2117-09-11 21:16:00 NaN NaN \n", " 2117-09-11 22:00:00 NaN NaN \n", " 2117-09-11 22:10:00 97.0 NaN \n", " 2117-09-11 22:25:00 NaN NaN \n", "... ... ... \n", "199999 2136-04-09 09:00:00 95.0 NaN \n", " 2136-04-09 10:00:00 94.0 NaN \n", " 2136-04-09 11:00:00 94.0 NaN \n", " 2136-04-09 12:00:00 93.0 NaN \n", " 2136-04-09 13:00:00 94.0 NaN \n", " 2136-04-09 14:00:00 93.0 NaN \n", " 2136-04-09 15:00:00 93.0 NaN \n", " 2136-04-09 16:00:00 94.0 NaN \n", " 2136-04-09 17:00:00 94.0 NaN \n", " 2136-04-09 18:00:00 93.0 NaN \n", " 2136-04-09 19:00:00 93.0 NaN \n", " 2136-04-09 20:00:00 95.0 NaN \n", " 2136-04-09 21:00:00 93.0 NaN \n", " 2136-04-09 22:00:00 97.0 NaN \n", " 2136-04-09 23:00:00 96.0 NaN \n", " 2136-04-10 00:00:00 94.0 NaN \n", " 2136-04-10 00:01:00 NaN NaN \n", " 2136-04-10 01:00:00 95.0 NaN \n", " 2136-04-10 02:00:00 93.0 NaN \n", " 2136-04-10 02:28:00 NaN NaN \n", " 2136-04-10 03:00:00 91.0 NaN \n", " 2136-04-10 04:00:00 95.0 NaN \n", " 2136-04-10 05:00:00 96.0 NaN \n", " 2136-04-10 06:00:00 92.0 NaN \n", " 2136-04-10 07:00:00 92.0 NaN \n", " 2136-04-10 08:00:00 96.0 NaN \n", " 2136-04-10 09:00:00 93.0 NaN \n", " 2136-04-10 09:10:00 NaN NaN \n", " 2136-04-10 10:00:00 97.0 NaN \n", " 2136-04-10 11:00:00 96.0 NaN \n", "\n", "label output urine glasgow coma scale motor ... \\\n", "status known known ... \n", "variable_type qn ord ... \n", "units mL no_units ... \n", "id datetime ... \n", "100001 2117-09-11 09:22:00 NaN NaN ... \n", " 2117-09-11 09:32:00 NaN NaN ... \n", " 2117-09-11 12:50:00 NaN NaN ... \n", " 2117-09-11 12:55:00 NaN NaN ... \n", " 2117-09-11 12:57:00 NaN NaN ... \n", " 2117-09-11 13:00:00 NaN NaN ... \n", " 2117-09-11 13:01:00 NaN NaN ... \n", " 2117-09-11 13:48:00 NaN NaN ... \n", " 2117-09-11 13:49:00 300.0 NaN ... \n", " 2117-09-11 13:50:00 NaN NaN ... \n", " 2117-09-11 14:00:00 NaN NaN ... \n", " 2117-09-11 15:00:00 NaN NaN ... \n", " 2117-09-11 15:48:00 NaN NaN ... \n", " 2117-09-11 15:59:00 NaN NaN ... \n", " 2117-09-11 16:00:00 NaN NaN ... \n", " 2117-09-11 16:02:00 NaN NaN ... \n", " 2117-09-11 16:11:00 NaN NaN ... \n", " 2117-09-11 16:12:00 NaN NaN ... \n", " 2117-09-11 17:00:00 NaN NaN ... \n", " 2117-09-11 18:00:00 NaN NaN ... \n", " 2117-09-11 18:34:00 400.0 NaN ... \n", " 2117-09-11 19:00:00 NaN NaN ... \n", " 2117-09-11 19:31:00 NaN NaN ... \n", " 2117-09-11 20:00:00 NaN NaN ... \n", " 2117-09-11 21:00:00 NaN NaN ... \n", " 2117-09-11 21:12:00 NaN NaN ... \n", " 2117-09-11 21:16:00 NaN NaN ... \n", " 2117-09-11 22:00:00 200.0 NaN ... \n", " 2117-09-11 22:10:00 NaN NaN ... \n", " 2117-09-11 22:25:00 NaN NaN ... \n", "... ... ... ... \n", "199999 2136-04-09 09:00:00 350.0 NaN ... \n", " 2136-04-09 10:00:00 NaN NaN ... \n", " 2136-04-09 11:00:00 NaN NaN ... \n", " 2136-04-09 12:00:00 NaN NaN ... \n", " 2136-04-09 13:00:00 NaN NaN ... \n", " 2136-04-09 14:00:00 NaN NaN ... \n", " 2136-04-09 15:00:00 NaN NaN ... \n", " 2136-04-09 16:00:00 400.0 NaN ... \n", " 2136-04-09 17:00:00 NaN NaN ... \n", " 2136-04-09 18:00:00 NaN NaN ... \n", " 2136-04-09 19:00:00 NaN NaN ... \n", " 2136-04-09 20:00:00 480.0 NaN ... \n", " 2136-04-09 21:00:00 NaN NaN ... \n", " 2136-04-09 22:00:00 NaN NaN ... \n", " 2136-04-09 23:00:00 NaN NaN ... \n", " 2136-04-10 00:00:00 NaN NaN ... \n", " 2136-04-10 00:01:00 NaN NaN ... \n", " 2136-04-10 01:00:00 NaN NaN ... \n", " 2136-04-10 02:00:00 350.0 NaN ... \n", " 2136-04-10 02:28:00 NaN NaN ... \n", " 2136-04-10 03:00:00 NaN NaN ... \n", " 2136-04-10 04:00:00 NaN NaN ... \n", " 2136-04-10 05:00:00 330.0 NaN ... \n", " 2136-04-10 06:00:00 NaN NaN ... \n", " 2136-04-10 07:00:00 NaN NaN ... \n", " 2136-04-10 08:00:00 200.0 NaN ... \n", " 2136-04-10 09:00:00 NaN NaN ... \n", " 2136-04-10 09:10:00 NaN NaN ... \n", " 2136-04-10 10:00:00 200.0 NaN ... \n", " 2136-04-10 11:00:00 NaN NaN ... \n", "\n", "label normal saline lactated ringers \\\n", "status known known \n", "variable_type qn qn \n", "units mL mL/hr mL mL/hr \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN NaN NaN \n", " 2117-09-11 12:50:00 NaN 5.000000 NaN NaN \n", " 2117-09-11 12:55:00 NaN 6.996487 NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN NaN NaN \n", " 2117-09-11 13:01:00 NaN NaN NaN NaN \n", " 2117-09-11 13:48:00 NaN 499.999980 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN NaN NaN \n", " 2117-09-11 14:00:00 NaN 8.990600 NaN NaN \n", " 2117-09-11 15:00:00 NaN NaN NaN NaN \n", " 2117-09-11 15:48:00 NaN 999.999960 NaN NaN \n", " 2117-09-11 15:59:00 NaN 199.999998 NaN NaN \n", " 2117-09-11 16:00:00 NaN NaN NaN NaN \n", " 2117-09-11 16:02:00 NaN NaN NaN NaN \n", " 2117-09-11 16:11:00 NaN 40.000000 NaN NaN \n", " 2117-09-11 16:12:00 NaN 4.992476 NaN NaN \n", " 2117-09-11 17:00:00 NaN NaN NaN NaN \n", " 2117-09-11 18:00:00 NaN NaN NaN NaN \n", " 2117-09-11 18:34:00 NaN NaN NaN NaN \n", " 2117-09-11 19:00:00 NaN NaN NaN NaN \n", " 2117-09-11 19:31:00 NaN NaN NaN NaN \n", " 2117-09-11 20:00:00 NaN NaN NaN NaN \n", " 2117-09-11 21:00:00 NaN NaN NaN NaN \n", " 2117-09-11 21:12:00 NaN 3.396080 NaN NaN \n", " 2117-09-11 21:16:00 NaN 5.012718 NaN NaN \n", " 2117-09-11 22:00:00 NaN NaN NaN NaN \n", " 2117-09-11 22:10:00 NaN NaN NaN NaN \n", " 2117-09-11 22:25:00 500.0 NaN NaN NaN \n", "... ... ... ... ... \n", "199999 2136-04-09 09:00:00 NaN NaN NaN NaN \n", " 2136-04-09 10:00:00 NaN NaN NaN NaN \n", " 2136-04-09 11:00:00 NaN NaN NaN NaN \n", " 2136-04-09 12:00:00 NaN NaN NaN NaN \n", " 2136-04-09 13:00:00 NaN NaN NaN NaN \n", " 2136-04-09 14:00:00 NaN NaN NaN NaN \n", " 2136-04-09 15:00:00 NaN NaN NaN NaN \n", " 2136-04-09 16:00:00 NaN NaN NaN NaN \n", " 2136-04-09 17:00:00 NaN NaN NaN NaN \n", " 2136-04-09 18:00:00 NaN NaN NaN NaN \n", " 2136-04-09 19:00:00 NaN NaN NaN NaN \n", " 2136-04-09 20:00:00 NaN NaN NaN NaN \n", " 2136-04-09 21:00:00 NaN NaN NaN NaN \n", " 2136-04-09 22:00:00 NaN NaN NaN NaN \n", " 2136-04-09 23:00:00 NaN NaN NaN NaN \n", " 2136-04-10 00:00:00 NaN NaN NaN NaN \n", " 2136-04-10 00:01:00 NaN NaN NaN NaN \n", " 2136-04-10 01:00:00 NaN NaN NaN NaN \n", " 2136-04-10 02:00:00 NaN NaN NaN NaN \n", " 2136-04-10 02:28:00 NaN NaN NaN NaN \n", " 2136-04-10 03:00:00 NaN NaN NaN NaN \n", " 2136-04-10 04:00:00 NaN NaN NaN NaN \n", " 2136-04-10 05:00:00 NaN NaN NaN NaN \n", " 2136-04-10 06:00:00 NaN NaN NaN NaN \n", " 2136-04-10 07:00:00 NaN NaN NaN NaN \n", " 2136-04-10 08:00:00 NaN NaN NaN NaN \n", " 2136-04-10 09:00:00 NaN NaN NaN NaN \n", " 2136-04-10 09:10:00 NaN NaN NaN NaN \n", " 2136-04-10 10:00:00 NaN NaN NaN NaN \n", " 2136-04-10 11:00:00 NaN NaN NaN NaN \n", "\n", "label norepinephrine vasopressin \\\n", "status known known \n", "variable_type qn qn \n", "units mcg/kg/min mcg/min units units/min \n", "id datetime \n", "100001 2117-09-11 09:22:00 NaN NaN NaN NaN \n", " 2117-09-11 09:32:00 NaN NaN NaN NaN \n", " 2117-09-11 12:50:00 NaN NaN NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN NaN NaN \n", " 2117-09-11 13:01:00 NaN NaN NaN NaN \n", " 2117-09-11 13:48:00 NaN NaN NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN NaN NaN \n", " 2117-09-11 14:00:00 NaN NaN NaN NaN \n", " 2117-09-11 15:00:00 NaN NaN NaN NaN \n", " 2117-09-11 15:48:00 NaN NaN NaN NaN \n", " 2117-09-11 15:59:00 NaN NaN NaN NaN \n", " 2117-09-11 16:00:00 NaN NaN NaN NaN \n", " 2117-09-11 16:02:00 NaN NaN NaN NaN \n", " 2117-09-11 16:11:00 NaN NaN NaN NaN \n", " 2117-09-11 16:12:00 NaN NaN NaN NaN \n", " 2117-09-11 17:00:00 NaN NaN NaN NaN \n", " 2117-09-11 18:00:00 NaN NaN NaN NaN \n", " 2117-09-11 18:34:00 NaN NaN NaN NaN \n", " 2117-09-11 19:00:00 NaN NaN NaN NaN \n", " 2117-09-11 19:31:00 NaN NaN NaN NaN \n", " 2117-09-11 20:00:00 NaN NaN NaN NaN \n", " 2117-09-11 21:00:00 NaN NaN NaN NaN \n", " 2117-09-11 21:12:00 NaN NaN NaN NaN \n", " 2117-09-11 21:16:00 NaN NaN NaN NaN \n", " 2117-09-11 22:00:00 NaN NaN NaN NaN \n", " 2117-09-11 22:10:00 NaN NaN NaN NaN \n", " 2117-09-11 22:25:00 NaN NaN NaN NaN \n", "... ... ... ... ... \n", "199999 2136-04-09 09:00:00 NaN NaN NaN NaN \n", " 2136-04-09 10:00:00 NaN NaN NaN NaN \n", " 2136-04-09 11:00:00 NaN NaN NaN NaN \n", " 2136-04-09 12:00:00 NaN NaN NaN NaN \n", " 2136-04-09 13:00:00 NaN NaN NaN NaN \n", " 2136-04-09 14:00:00 NaN NaN NaN NaN \n", " 2136-04-09 15:00:00 NaN NaN NaN NaN \n", " 2136-04-09 16:00:00 NaN NaN NaN NaN \n", " 2136-04-09 17:00:00 NaN NaN NaN NaN \n", " 2136-04-09 18:00:00 NaN NaN NaN NaN \n", " 2136-04-09 19:00:00 NaN NaN NaN NaN \n", " 2136-04-09 20:00:00 NaN NaN NaN NaN \n", " 2136-04-09 21:00:00 NaN NaN NaN NaN \n", " 2136-04-09 22:00:00 NaN NaN NaN NaN \n", " 2136-04-09 23:00:00 NaN NaN NaN NaN \n", " 2136-04-10 00:00:00 NaN NaN NaN NaN \n", " 2136-04-10 00:01:00 NaN NaN NaN NaN \n", " 2136-04-10 01:00:00 NaN NaN NaN NaN \n", " 2136-04-10 02:00:00 NaN NaN NaN NaN \n", " 2136-04-10 02:28:00 NaN NaN NaN NaN \n", " 2136-04-10 03:00:00 NaN NaN NaN NaN \n", " 2136-04-10 04:00:00 NaN NaN NaN NaN \n", " 2136-04-10 05:00:00 NaN NaN NaN NaN \n", " 2136-04-10 06:00:00 NaN NaN NaN NaN \n", " 2136-04-10 07:00:00 NaN NaN NaN NaN \n", " 2136-04-10 08:00:00 NaN NaN NaN NaN \n", " 2136-04-10 09:00:00 NaN NaN NaN NaN \n", " 2136-04-10 09:10:00 NaN NaN NaN NaN \n", " 2136-04-10 10:00:00 NaN NaN NaN NaN \n", " 2136-04-10 11:00:00 NaN NaN NaN NaN \n", "\n", "label hemoglobin lactate \n", "status known known \n", "variable_type qn qn \n", "units g/dL mmol/L \n", "id datetime \n", "100001 2117-09-11 09:22:00 13.0 NaN \n", " 2117-09-11 09:32:00 NaN 1.9 \n", " 2117-09-11 12:50:00 NaN NaN \n", " 2117-09-11 12:55:00 NaN NaN \n", " 2117-09-11 12:57:00 NaN NaN \n", " 2117-09-11 13:00:00 NaN NaN \n", " 2117-09-11 13:01:00 NaN NaN \n", " 2117-09-11 13:48:00 NaN NaN \n", " 2117-09-11 13:49:00 NaN NaN \n", " 2117-09-11 13:50:00 NaN NaN \n", " 2117-09-11 14:00:00 NaN NaN \n", " 2117-09-11 15:00:00 NaN NaN \n", " 2117-09-11 15:48:00 NaN NaN \n", " 2117-09-11 15:59:00 NaN NaN \n", " 2117-09-11 16:00:00 NaN NaN \n", " 2117-09-11 16:02:00 NaN NaN \n", " 2117-09-11 16:11:00 NaN NaN \n", " 2117-09-11 16:12:00 NaN NaN \n", " 2117-09-11 17:00:00 NaN NaN \n", " 2117-09-11 18:00:00 NaN NaN \n", " 2117-09-11 18:34:00 NaN NaN \n", " 2117-09-11 19:00:00 NaN NaN \n", " 2117-09-11 19:31:00 11.0 NaN \n", " 2117-09-11 20:00:00 NaN NaN \n", " 2117-09-11 21:00:00 NaN NaN \n", " 2117-09-11 21:12:00 NaN NaN \n", " 2117-09-11 21:16:00 NaN NaN \n", " 2117-09-11 22:00:00 NaN NaN \n", " 2117-09-11 22:10:00 NaN NaN \n", " 2117-09-11 22:25:00 NaN NaN \n", "... ... ... \n", "199999 2136-04-09 09:00:00 NaN NaN \n", " 2136-04-09 10:00:00 NaN NaN \n", " 2136-04-09 11:00:00 NaN NaN \n", " 2136-04-09 12:00:00 NaN NaN \n", " 2136-04-09 13:00:00 NaN NaN \n", " 2136-04-09 14:00:00 NaN NaN \n", " 2136-04-09 15:00:00 NaN NaN \n", " 2136-04-09 16:00:00 NaN NaN \n", " 2136-04-09 17:00:00 NaN NaN \n", " 2136-04-09 18:00:00 NaN NaN \n", " 2136-04-09 19:00:00 NaN NaN \n", " 2136-04-09 20:00:00 NaN NaN \n", " 2136-04-09 21:00:00 NaN NaN \n", " 2136-04-09 22:00:00 NaN NaN \n", " 2136-04-09 23:00:00 NaN NaN \n", " 2136-04-10 00:00:00 NaN NaN \n", " 2136-04-10 00:01:00 NaN NaN \n", " 2136-04-10 01:00:00 NaN NaN \n", " 2136-04-10 02:00:00 NaN NaN \n", " 2136-04-10 02:28:00 12.6 NaN \n", " 2136-04-10 03:00:00 NaN NaN \n", " 2136-04-10 04:00:00 NaN NaN \n", " 2136-04-10 05:00:00 NaN NaN \n", " 2136-04-10 06:00:00 NaN NaN \n", " 2136-04-10 07:00:00 NaN NaN \n", " 2136-04-10 08:00:00 NaN NaN \n", " 2136-04-10 09:00:00 NaN NaN \n", " 2136-04-10 09:10:00 NaN NaN \n", " 2136-04-10 10:00:00 NaN NaN \n", " 2136-04-10 11:00:00 NaN NaN \n", "\n", "[10071464 rows x 22 columns]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "utils.save_df(df_final,hdf5_fname,'cleaned/test1')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Prepare for ML\n", "\n", "\n", "1. All categories to numeric representations\n", "2. Segment, add to index\n", "3. Transform into features (FeatureUnion)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "collapsed": true }, "outputs": [], "source": [ "intro_pipeline = Pipeline([\n", " ('transform',mimic_transform),\n", " ('format',standard_pipeline),\n", " ])" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labellactate
statusknownunknown
variable_typeqnqnnom
unitsmmol/Lno_unitsno_units
description50813225668818153122566850813(mmol/L)818(mmol/L)1531(mmol/L)
iddatetime
1000012117-09-11 09:32:001.9NaNNaNNaNNaNNaNNaNNaN
1000032150-04-17 19:12:001.11.1NaNNaNNaNNaNNaNNaN
1000062108-04-08 10:58:004.5NaN4.5NaNNaNNaNNaNNaN
1000072145-03-31 00:44:003.1NaNNaNNaNNaNNaNNaNNaN
2145-04-02 14:10:001.9NaNNaNNaNNaNNaNNaNNaN
\n", "
" ], "text/plain": [ "label lactate \\\n", "status known unknown \n", "variable_type qn qn nom \n", "units mmol/L no_units no_units \n", "description 50813 225668 818 1531 225668 50813(mmol/L) \n", "id datetime \n", "100001 2117-09-11 09:32:00 1.9 NaN NaN NaN NaN NaN \n", "100003 2150-04-17 19:12:00 1.1 1.1 NaN NaN NaN NaN \n", "100006 2108-04-08 10:58:00 4.5 NaN 4.5 NaN NaN NaN \n", "100007 2145-03-31 00:44:00 3.1 NaN NaN NaN NaN NaN \n", " 2145-04-02 14:10:00 1.9 NaN NaN NaN NaN NaN \n", "\n", "label \n", "status \n", "variable_type \n", "units \n", "description 818(mmol/L) 1531(mmol/L) \n", "id datetime \n", "100001 2117-09-11 09:32:00 NaN NaN \n", "100003 2150-04-17 19:12:00 NaN NaN \n", "100006 2108-04-08 10:58:00 NaN NaN \n", "100007 2145-03-31 00:44:00 NaN NaN \n", " 2145-04-02 14:10:00 NaN NaN " ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "label = 'lactate'\n", "df_extract = utils.open_df(hdf5_fname,'extract/{}'.format(label))\n", "df_cleaned = intro_pipeline.transform(df_extract)\n", "df_cleaned.head()" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labellactate
statusknownunknown
variable_typeqnqnnom
unitsmmol/Lno_unitsno_units
description50813225668818153122566850813(mmol/L)818(mmol/L)1531(mmol/L)
count176767.00000069402.00000072916.00000063134.00000014.000000575723
uniqueNaNNaNNaNNaNNaN34148
topNaNNaNNaNNaNNaNERRORno datano data
freqNaNNaNNaNNaNNaN18379
mean2.63512016.9436072.8493002.78033852.835714NaNNaNNaN
std2.5902683795.8812333.1520313.04552782.567153NaNNaNNaN
min0.0000000.0500000.0000000.0000000.000000NaNNaNNaN
25%1.2000001.2000001.2000001.2000001.175000NaNNaNNaN
50%1.8000001.8000001.8000001.8000003.400000NaNNaNNaN
75%2.9000002.9000003.1000003.000000115.400000NaNNaNNaN
max36.000000999999.000000153.000000153.000000203.000000NaNNaNNaN
\n", "
" ], "text/plain": [ "label lactate \\\n", "status known \n", "variable_type qn \n", "units mmol/L \n", "description 50813 225668 818 1531 \n", "count 176767.000000 69402.000000 72916.000000 63134.000000 \n", "unique NaN NaN NaN NaN \n", "top NaN NaN NaN NaN \n", "freq NaN NaN NaN NaN \n", "mean 2.635120 16.943607 2.849300 2.780338 \n", "std 2.590268 3795.881233 3.152031 3.045527 \n", "min 0.000000 0.050000 0.000000 0.000000 \n", "25% 1.200000 1.200000 1.200000 1.200000 \n", "50% 1.800000 1.800000 1.800000 1.800000 \n", "75% 2.900000 2.900000 3.100000 3.000000 \n", "max 36.000000 999999.000000 153.000000 153.000000 \n", "\n", "label \n", "status unknown \n", "variable_type qn nom \n", "units no_units no_units \n", "description 225668 50813(mmol/L) 818(mmol/L) 1531(mmol/L) \n", "count 14.000000 57 57 23 \n", "unique NaN 34 14 8 \n", "top NaN ERROR no data no data \n", "freq NaN 18 37 9 \n", "mean 52.835714 NaN NaN NaN \n", "std 82.567153 NaN NaN NaN \n", "min 0.000000 NaN NaN NaN \n", "25% 1.175000 NaN NaN NaN \n", "50% 3.400000 NaN NaN NaN \n", "75% 115.400000 NaN NaN NaN \n", "max 203.000000 NaN NaN NaN " ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_cleaned.describe(include='all')" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labellactate
statusknownunknown
variable_typeqnqnnom
unitsmmol/Lno_unitsno_units
description50813225668818153122566850813(mmol/L)_.50813(mmol/L)_5,050813(mmol/L)_>3050813(mmol/L)_>30.050813(mmol/L)_CLOTTED...818(mmol/L)_VOIDED818(mmol/L)_no data1531(mmol/L)_.1531(mmol/L)_5,01531(mmol/L)_>301531(mmol/L)_>30.01531(mmol/L)_CLOTTED1531(mmol/L)_ERROR1531(mmol/L)_VOIDED1531(mmol/L)_no data
iddatetime
1000012117-09-11 09:32:001.9NaNNaNNaNNaN00000...0000000000
1000032150-04-17 19:12:001.11.1NaNNaNNaN00000...0000000000
1000062108-04-08 10:58:004.5NaN4.5NaNNaN00000...0000000000
1000072145-03-31 00:44:003.1NaNNaNNaNNaN00000...0000000000
2145-04-02 14:10:001.9NaNNaNNaNNaN00000...0000000000
1000092162-05-17 13:19:001.11.1NaNNaNNaN00000...0000000000
2162-05-17 17:14:001.51.5NaNNaNNaN00000...0000000000
1000102109-12-10 10:25:000.6NaNNaNNaNNaN00000...0000000000
2109-12-10 12:11:000.9NaNNaNNaNNaN00000...0000000000
2109-12-10 13:05:001.0NaNNaNNaNNaN00000...0000000000
2109-12-10 13:58:000.8NaNNaNNaNNaN00000...0000000000
1000112177-08-29 04:44:003.8NaNNaNNaNNaN00000...0000000000
2177-08-29 06:55:002.32.3NaNNaNNaN00000...0000000000
1000122177-03-14 07:38:002.3NaNNaNNaNNaN00000...0000000000
2177-03-14 11:42:002.52.5NaNNaNNaN00000...0000000000
2177-03-15 08:05:002.12.1NaNNaNNaN00000...0000000000
2177-03-15 14:01:002.62.6NaNNaNNaN00000...0000000000
2177-03-15 21:42:001.81.8NaNNaNNaN00000...0000000000
1000162188-05-24 12:00:002.0NaNNaNNaNNaN00000...0000000000
1000172103-03-11 05:10:001.1NaN1.1NaNNaN00000...0000000000
1000182176-08-29 15:29:001.3NaNNaNNaNNaN00000...0000000000
2176-08-30 09:23:000.90.9NaNNaNNaN00000...0000000000
2176-08-30 10:19:001.01.0NaNNaNNaN00000...0000000000
2176-08-30 11:29:000.90.9NaNNaNNaN00000...0000000000
2176-08-30 12:40:001.11.1NaNNaNNaN00000...0000000000
1000202142-11-30 21:54:001.1NaNNaNNaNNaN00000...0000000000
2142-12-03 00:17:001.0NaNNaNNaNNaN00000...0000000000
1000242170-09-19 10:25:001.41.4NaNNaNNaN00000...0000000000
2170-09-19 16:33:002.62.6NaNNaNNaN00000...0000000000
2170-09-20 02:04:003.23.2NaNNaNNaN00000...0000000000
.....................................................................
1999762182-02-14 11:15:000.8NaN0.8NaNNaN00000...0000000000
2182-02-16 03:57:000.8NaN0.8NaNNaN00000...0000000000
2182-02-19 03:59:000.8NaN0.8NaNNaN00000...0000000000
2182-02-20 03:31:000.7NaN0.7NaNNaN00000...0000000000
2182-02-21 04:55:000.9NaN0.9NaNNaN00000...0000000000
1999792182-02-06 09:17:001.0NaNNaNNaNNaN00000...0000000000
2182-02-06 14:16:003.83.8NaNNaNNaN00000...0000000000
1999812110-09-24 16:34:001.1NaN1.11.1NaN00000...0000000000
2110-09-24 20:09:001.0NaN1.01.0NaN00000...0000000000
2110-09-25 06:10:001.0NaN1.01.0NaN00000...0000000000
1999872175-05-19 16:30:002.3NaNNaNNaNNaN00000...0000000000
1999882169-01-24 12:48:001.0NaNNaNNaNNaN00000...0000000000
2169-02-07 01:35:001.6NaN1.6NaNNaN00000...0000000000
2169-02-07 11:18:001.0NaN1.0NaNNaN00000...0000000000
2169-02-07 16:43:001.0NaN1.0NaNNaN00000...0000000000
2169-02-07 22:35:001.1NaN1.1NaNNaN00000...0000000000
2169-02-10 05:33:001.4NaNNaNNaNNaN00000...0000000000
1999932161-11-12 23:14:000.9NaN0.9NaNNaN00000...0000000000
2161-11-13 03:46:000.9NaN0.9NaNNaN00000...0000000000
1999942188-07-07 21:23:001.0NaN1.0NaNNaN00000...0000000000
2188-07-08 03:09:000.7NaN0.7NaNNaN00000...0000000000
2188-07-08 04:13:000.6NaN0.6NaNNaN00000...0000000000
2188-07-08 06:20:000.7NaN0.7NaNNaN00000...0000000000
1999982119-02-20 10:52:001.11.11.11.1NaN00000...0000000000
2119-02-20 12:36:001.91.91.91.9NaN00000...0000000000
2119-02-20 13:33:002.02.02.02.0NaN00000...0000000000
2119-02-20 13:59:002.62.62.62.6NaN00000...0000000000
2119-02-20 20:43:001.31.31.31.3NaN00000...0000000000
1999992136-04-04 20:55:001.9NaNNaNNaNNaN00000...0000000000
2136-04-06 15:29:001.8NaNNaNNaNNaN00000...0000000000
\n", "

177450 rows × 29 columns

\n", "
" ], "text/plain": [ "label lactate \\\n", "status known unknown \n", "variable_type qn qn nom \n", "units mmol/L no_units no_units \n", "description 50813 225668 818 1531 225668 50813(mmol/L)_. \n", "id datetime \n", "100001 2117-09-11 09:32:00 1.9 NaN NaN NaN NaN 0 \n", "100003 2150-04-17 19:12:00 1.1 1.1 NaN NaN NaN 0 \n", "100006 2108-04-08 10:58:00 4.5 NaN 4.5 NaN NaN 0 \n", "100007 2145-03-31 00:44:00 3.1 NaN NaN NaN NaN 0 \n", " 2145-04-02 14:10:00 1.9 NaN NaN NaN NaN 0 \n", "100009 2162-05-17 13:19:00 1.1 1.1 NaN NaN NaN 0 \n", " 2162-05-17 17:14:00 1.5 1.5 NaN NaN NaN 0 \n", "100010 2109-12-10 10:25:00 0.6 NaN NaN NaN NaN 0 \n", " 2109-12-10 12:11:00 0.9 NaN NaN NaN NaN 0 \n", " 2109-12-10 13:05:00 1.0 NaN NaN NaN NaN 0 \n", " 2109-12-10 13:58:00 0.8 NaN NaN NaN NaN 0 \n", "100011 2177-08-29 04:44:00 3.8 NaN NaN NaN NaN 0 \n", " 2177-08-29 06:55:00 2.3 2.3 NaN NaN NaN 0 \n", "100012 2177-03-14 07:38:00 2.3 NaN NaN NaN NaN 0 \n", " 2177-03-14 11:42:00 2.5 2.5 NaN NaN NaN 0 \n", " 2177-03-15 08:05:00 2.1 2.1 NaN NaN NaN 0 \n", " 2177-03-15 14:01:00 2.6 2.6 NaN NaN NaN 0 \n", " 2177-03-15 21:42:00 1.8 1.8 NaN NaN NaN 0 \n", "100016 2188-05-24 12:00:00 2.0 NaN NaN NaN NaN 0 \n", "100017 2103-03-11 05:10:00 1.1 NaN 1.1 NaN NaN 0 \n", "100018 2176-08-29 15:29:00 1.3 NaN NaN NaN NaN 0 \n", " 2176-08-30 09:23:00 0.9 0.9 NaN NaN NaN 0 \n", " 2176-08-30 10:19:00 1.0 1.0 NaN NaN NaN 0 \n", " 2176-08-30 11:29:00 0.9 0.9 NaN NaN NaN 0 \n", " 2176-08-30 12:40:00 1.1 1.1 NaN NaN NaN 0 \n", "100020 2142-11-30 21:54:00 1.1 NaN NaN NaN NaN 0 \n", " 2142-12-03 00:17:00 1.0 NaN NaN NaN NaN 0 \n", "100024 2170-09-19 10:25:00 1.4 1.4 NaN NaN NaN 0 \n", " 2170-09-19 16:33:00 2.6 2.6 NaN NaN NaN 0 \n", " 2170-09-20 02:04:00 3.2 3.2 NaN NaN NaN 0 \n", "... ... ... ... ... ... ... \n", "199976 2182-02-14 11:15:00 0.8 NaN 0.8 NaN NaN 0 \n", " 2182-02-16 03:57:00 0.8 NaN 0.8 NaN NaN 0 \n", " 2182-02-19 03:59:00 0.8 NaN 0.8 NaN NaN 0 \n", " 2182-02-20 03:31:00 0.7 NaN 0.7 NaN NaN 0 \n", " 2182-02-21 04:55:00 0.9 NaN 0.9 NaN NaN 0 \n", "199979 2182-02-06 09:17:00 1.0 NaN NaN NaN NaN 0 \n", " 2182-02-06 14:16:00 3.8 3.8 NaN NaN NaN 0 \n", "199981 2110-09-24 16:34:00 1.1 NaN 1.1 1.1 NaN 0 \n", " 2110-09-24 20:09:00 1.0 NaN 1.0 1.0 NaN 0 \n", " 2110-09-25 06:10:00 1.0 NaN 1.0 1.0 NaN 0 \n", "199987 2175-05-19 16:30:00 2.3 NaN NaN NaN NaN 0 \n", "199988 2169-01-24 12:48:00 1.0 NaN NaN NaN NaN 0 \n", " 2169-02-07 01:35:00 1.6 NaN 1.6 NaN NaN 0 \n", " 2169-02-07 11:18:00 1.0 NaN 1.0 NaN NaN 0 \n", " 2169-02-07 16:43:00 1.0 NaN 1.0 NaN NaN 0 \n", " 2169-02-07 22:35:00 1.1 NaN 1.1 NaN NaN 0 \n", " 2169-02-10 05:33:00 1.4 NaN NaN NaN NaN 0 \n", "199993 2161-11-12 23:14:00 0.9 NaN 0.9 NaN NaN 0 \n", " 2161-11-13 03:46:00 0.9 NaN 0.9 NaN NaN 0 \n", "199994 2188-07-07 21:23:00 1.0 NaN 1.0 NaN NaN 0 \n", " 2188-07-08 03:09:00 0.7 NaN 0.7 NaN NaN 0 \n", " 2188-07-08 04:13:00 0.6 NaN 0.6 NaN NaN 0 \n", " 2188-07-08 06:20:00 0.7 NaN 0.7 NaN NaN 0 \n", "199998 2119-02-20 10:52:00 1.1 1.1 1.1 1.1 NaN 0 \n", " 2119-02-20 12:36:00 1.9 1.9 1.9 1.9 NaN 0 \n", " 2119-02-20 13:33:00 2.0 2.0 2.0 2.0 NaN 0 \n", " 2119-02-20 13:59:00 2.6 2.6 2.6 2.6 NaN 0 \n", " 2119-02-20 20:43:00 1.3 1.3 1.3 1.3 NaN 0 \n", "199999 2136-04-04 20:55:00 1.9 NaN NaN NaN NaN 0 \n", " 2136-04-06 15:29:00 1.8 NaN NaN NaN NaN 0 \n", "\n", "label \\\n", "status \n", "variable_type \n", "units \n", "description 50813(mmol/L)_5,0 50813(mmol/L)_>30 \n", "id datetime \n", "100001 2117-09-11 09:32:00 0 0 \n", "100003 2150-04-17 19:12:00 0 0 \n", "100006 2108-04-08 10:58:00 0 0 \n", "100007 2145-03-31 00:44:00 0 0 \n", " 2145-04-02 14:10:00 0 0 \n", "100009 2162-05-17 13:19:00 0 0 \n", " 2162-05-17 17:14:00 0 0 \n", "100010 2109-12-10 10:25:00 0 0 \n", " 2109-12-10 12:11:00 0 0 \n", " 2109-12-10 13:05:00 0 0 \n", " 2109-12-10 13:58:00 0 0 \n", "100011 2177-08-29 04:44:00 0 0 \n", " 2177-08-29 06:55:00 0 0 \n", "100012 2177-03-14 07:38:00 0 0 \n", " 2177-03-14 11:42:00 0 0 \n", " 2177-03-15 08:05:00 0 0 \n", " 2177-03-15 14:01:00 0 0 \n", " 2177-03-15 21:42:00 0 0 \n", "100016 2188-05-24 12:00:00 0 0 \n", "100017 2103-03-11 05:10:00 0 0 \n", "100018 2176-08-29 15:29:00 0 0 \n", " 2176-08-30 09:23:00 0 0 \n", " 2176-08-30 10:19:00 0 0 \n", " 2176-08-30 11:29:00 0 0 \n", " 2176-08-30 12:40:00 0 0 \n", "100020 2142-11-30 21:54:00 0 0 \n", " 2142-12-03 00:17:00 0 0 \n", "100024 2170-09-19 10:25:00 0 0 \n", " 2170-09-19 16:33:00 0 0 \n", " 2170-09-20 02:04:00 0 0 \n", "... ... ... \n", "199976 2182-02-14 11:15:00 0 0 \n", " 2182-02-16 03:57:00 0 0 \n", " 2182-02-19 03:59:00 0 0 \n", " 2182-02-20 03:31:00 0 0 \n", " 2182-02-21 04:55:00 0 0 \n", "199979 2182-02-06 09:17:00 0 0 \n", " 2182-02-06 14:16:00 0 0 \n", "199981 2110-09-24 16:34:00 0 0 \n", " 2110-09-24 20:09:00 0 0 \n", " 2110-09-25 06:10:00 0 0 \n", "199987 2175-05-19 16:30:00 0 0 \n", "199988 2169-01-24 12:48:00 0 0 \n", " 2169-02-07 01:35:00 0 0 \n", " 2169-02-07 11:18:00 0 0 \n", " 2169-02-07 16:43:00 0 0 \n", " 2169-02-07 22:35:00 0 0 \n", " 2169-02-10 05:33:00 0 0 \n", "199993 2161-11-12 23:14:00 0 0 \n", " 2161-11-13 03:46:00 0 0 \n", "199994 2188-07-07 21:23:00 0 0 \n", " 2188-07-08 03:09:00 0 0 \n", " 2188-07-08 04:13:00 0 0 \n", " 2188-07-08 06:20:00 0 0 \n", "199998 2119-02-20 10:52:00 0 0 \n", " 2119-02-20 12:36:00 0 0 \n", " 2119-02-20 13:33:00 0 0 \n", " 2119-02-20 13:59:00 0 0 \n", " 2119-02-20 20:43:00 0 0 \n", "199999 2136-04-04 20:55:00 0 0 \n", " 2136-04-06 15:29:00 0 0 \n", "\n", "label \\\n", "status \n", "variable_type \n", "units \n", "description 50813(mmol/L)_>30.0 50813(mmol/L)_CLOTTED \n", "id datetime \n", "100001 2117-09-11 09:32:00 0 0 \n", "100003 2150-04-17 19:12:00 0 0 \n", "100006 2108-04-08 10:58:00 0 0 \n", "100007 2145-03-31 00:44:00 0 0 \n", " 2145-04-02 14:10:00 0 0 \n", "100009 2162-05-17 13:19:00 0 0 \n", " 2162-05-17 17:14:00 0 0 \n", "100010 2109-12-10 10:25:00 0 0 \n", " 2109-12-10 12:11:00 0 0 \n", " 2109-12-10 13:05:00 0 0 \n", " 2109-12-10 13:58:00 0 0 \n", "100011 2177-08-29 04:44:00 0 0 \n", " 2177-08-29 06:55:00 0 0 \n", "100012 2177-03-14 07:38:00 0 0 \n", " 2177-03-14 11:42:00 0 0 \n", " 2177-03-15 08:05:00 0 0 \n", " 2177-03-15 14:01:00 0 0 \n", " 2177-03-15 21:42:00 0 0 \n", "100016 2188-05-24 12:00:00 0 0 \n", "100017 2103-03-11 05:10:00 0 0 \n", "100018 2176-08-29 15:29:00 0 0 \n", " 2176-08-30 09:23:00 0 0 \n", " 2176-08-30 10:19:00 0 0 \n", " 2176-08-30 11:29:00 0 0 \n", " 2176-08-30 12:40:00 0 0 \n", "100020 2142-11-30 21:54:00 0 0 \n", " 2142-12-03 00:17:00 0 0 \n", "100024 2170-09-19 10:25:00 0 0 \n", " 2170-09-19 16:33:00 0 0 \n", " 2170-09-20 02:04:00 0 0 \n", "... ... ... \n", "199976 2182-02-14 11:15:00 0 0 \n", " 2182-02-16 03:57:00 0 0 \n", " 2182-02-19 03:59:00 0 0 \n", " 2182-02-20 03:31:00 0 0 \n", " 2182-02-21 04:55:00 0 0 \n", "199979 2182-02-06 09:17:00 0 0 \n", " 2182-02-06 14:16:00 0 0 \n", "199981 2110-09-24 16:34:00 0 0 \n", " 2110-09-24 20:09:00 0 0 \n", " 2110-09-25 06:10:00 0 0 \n", "199987 2175-05-19 16:30:00 0 0 \n", "199988 2169-01-24 12:48:00 0 0 \n", " 2169-02-07 01:35:00 0 0 \n", " 2169-02-07 11:18:00 0 0 \n", " 2169-02-07 16:43:00 0 0 \n", " 2169-02-07 22:35:00 0 0 \n", " 2169-02-10 05:33:00 0 0 \n", "199993 2161-11-12 23:14:00 0 0 \n", " 2161-11-13 03:46:00 0 0 \n", "199994 2188-07-07 21:23:00 0 0 \n", " 2188-07-08 03:09:00 0 0 \n", " 2188-07-08 04:13:00 0 0 \n", " 2188-07-08 06:20:00 0 0 \n", "199998 2119-02-20 10:52:00 0 0 \n", " 2119-02-20 12:36:00 0 0 \n", " 2119-02-20 13:33:00 0 0 \n", " 2119-02-20 13:59:00 0 0 \n", " 2119-02-20 20:43:00 0 0 \n", "199999 2136-04-04 20:55:00 0 0 \n", " 2136-04-06 15:29:00 0 0 \n", "\n", "label ... \\\n", "status ... \n", "variable_type ... \n", "units ... \n", "description ... 818(mmol/L)_VOIDED \n", "id datetime ... \n", "100001 2117-09-11 09:32:00 ... 0 \n", "100003 2150-04-17 19:12:00 ... 0 \n", "100006 2108-04-08 10:58:00 ... 0 \n", "100007 2145-03-31 00:44:00 ... 0 \n", " 2145-04-02 14:10:00 ... 0 \n", "100009 2162-05-17 13:19:00 ... 0 \n", " 2162-05-17 17:14:00 ... 0 \n", "100010 2109-12-10 10:25:00 ... 0 \n", " 2109-12-10 12:11:00 ... 0 \n", " 2109-12-10 13:05:00 ... 0 \n", " 2109-12-10 13:58:00 ... 0 \n", "100011 2177-08-29 04:44:00 ... 0 \n", " 2177-08-29 06:55:00 ... 0 \n", "100012 2177-03-14 07:38:00 ... 0 \n", " 2177-03-14 11:42:00 ... 0 \n", " 2177-03-15 08:05:00 ... 0 \n", " 2177-03-15 14:01:00 ... 0 \n", " 2177-03-15 21:42:00 ... 0 \n", "100016 2188-05-24 12:00:00 ... 0 \n", "100017 2103-03-11 05:10:00 ... 0 \n", "100018 2176-08-29 15:29:00 ... 0 \n", " 2176-08-30 09:23:00 ... 0 \n", " 2176-08-30 10:19:00 ... 0 \n", " 2176-08-30 11:29:00 ... 0 \n", " 2176-08-30 12:40:00 ... 0 \n", "100020 2142-11-30 21:54:00 ... 0 \n", " 2142-12-03 00:17:00 ... 0 \n", "100024 2170-09-19 10:25:00 ... 0 \n", " 2170-09-19 16:33:00 ... 0 \n", " 2170-09-20 02:04:00 ... 0 \n", "... ... ... \n", "199976 2182-02-14 11:15:00 ... 0 \n", " 2182-02-16 03:57:00 ... 0 \n", " 2182-02-19 03:59:00 ... 0 \n", " 2182-02-20 03:31:00 ... 0 \n", " 2182-02-21 04:55:00 ... 0 \n", "199979 2182-02-06 09:17:00 ... 0 \n", " 2182-02-06 14:16:00 ... 0 \n", "199981 2110-09-24 16:34:00 ... 0 \n", " 2110-09-24 20:09:00 ... 0 \n", " 2110-09-25 06:10:00 ... 0 \n", "199987 2175-05-19 16:30:00 ... 0 \n", "199988 2169-01-24 12:48:00 ... 0 \n", " 2169-02-07 01:35:00 ... 0 \n", " 2169-02-07 11:18:00 ... 0 \n", " 2169-02-07 16:43:00 ... 0 \n", " 2169-02-07 22:35:00 ... 0 \n", " 2169-02-10 05:33:00 ... 0 \n", "199993 2161-11-12 23:14:00 ... 0 \n", " 2161-11-13 03:46:00 ... 0 \n", "199994 2188-07-07 21:23:00 ... 0 \n", " 2188-07-08 03:09:00 ... 0 \n", " 2188-07-08 04:13:00 ... 0 \n", " 2188-07-08 06:20:00 ... 0 \n", "199998 2119-02-20 10:52:00 ... 0 \n", " 2119-02-20 12:36:00 ... 0 \n", " 2119-02-20 13:33:00 ... 0 \n", " 2119-02-20 13:59:00 ... 0 \n", " 2119-02-20 20:43:00 ... 0 \n", "199999 2136-04-04 20:55:00 ... 0 \n", " 2136-04-06 15:29:00 ... 0 \n", "\n", "label \\\n", "status \n", "variable_type \n", "units \n", "description 818(mmol/L)_no data 1531(mmol/L)_. \n", "id datetime \n", "100001 2117-09-11 09:32:00 0 0 \n", "100003 2150-04-17 19:12:00 0 0 \n", "100006 2108-04-08 10:58:00 0 0 \n", "100007 2145-03-31 00:44:00 0 0 \n", " 2145-04-02 14:10:00 0 0 \n", "100009 2162-05-17 13:19:00 0 0 \n", " 2162-05-17 17:14:00 0 0 \n", "100010 2109-12-10 10:25:00 0 0 \n", " 2109-12-10 12:11:00 0 0 \n", " 2109-12-10 13:05:00 0 0 \n", " 2109-12-10 13:58:00 0 0 \n", "100011 2177-08-29 04:44:00 0 0 \n", " 2177-08-29 06:55:00 0 0 \n", "100012 2177-03-14 07:38:00 0 0 \n", " 2177-03-14 11:42:00 0 0 \n", " 2177-03-15 08:05:00 0 0 \n", " 2177-03-15 14:01:00 0 0 \n", " 2177-03-15 21:42:00 0 0 \n", "100016 2188-05-24 12:00:00 0 0 \n", "100017 2103-03-11 05:10:00 0 0 \n", "100018 2176-08-29 15:29:00 0 0 \n", " 2176-08-30 09:23:00 0 0 \n", " 2176-08-30 10:19:00 0 0 \n", " 2176-08-30 11:29:00 0 0 \n", " 2176-08-30 12:40:00 0 0 \n", "100020 2142-11-30 21:54:00 0 0 \n", " 2142-12-03 00:17:00 0 0 \n", "100024 2170-09-19 10:25:00 0 0 \n", " 2170-09-19 16:33:00 0 0 \n", " 2170-09-20 02:04:00 0 0 \n", "... ... ... \n", "199976 2182-02-14 11:15:00 0 0 \n", " 2182-02-16 03:57:00 0 0 \n", " 2182-02-19 03:59:00 0 0 \n", " 2182-02-20 03:31:00 0 0 \n", " 2182-02-21 04:55:00 0 0 \n", "199979 2182-02-06 09:17:00 0 0 \n", " 2182-02-06 14:16:00 0 0 \n", "199981 2110-09-24 16:34:00 0 0 \n", " 2110-09-24 20:09:00 0 0 \n", " 2110-09-25 06:10:00 0 0 \n", "199987 2175-05-19 16:30:00 0 0 \n", "199988 2169-01-24 12:48:00 0 0 \n", " 2169-02-07 01:35:00 0 0 \n", " 2169-02-07 11:18:00 0 0 \n", " 2169-02-07 16:43:00 0 0 \n", " 2169-02-07 22:35:00 0 0 \n", " 2169-02-10 05:33:00 0 0 \n", "199993 2161-11-12 23:14:00 0 0 \n", " 2161-11-13 03:46:00 0 0 \n", "199994 2188-07-07 21:23:00 0 0 \n", " 2188-07-08 03:09:00 0 0 \n", " 2188-07-08 04:13:00 0 0 \n", " 2188-07-08 06:20:00 0 0 \n", "199998 2119-02-20 10:52:00 0 0 \n", " 2119-02-20 12:36:00 0 0 \n", " 2119-02-20 13:33:00 0 0 \n", " 2119-02-20 13:59:00 0 0 \n", " 2119-02-20 20:43:00 0 0 \n", "199999 2136-04-04 20:55:00 0 0 \n", " 2136-04-06 15:29:00 0 0 \n", "\n", "label \\\n", "status \n", "variable_type \n", "units \n", "description 1531(mmol/L)_5,0 1531(mmol/L)_>30 \n", "id datetime \n", "100001 2117-09-11 09:32:00 0 0 \n", "100003 2150-04-17 19:12:00 0 0 \n", "100006 2108-04-08 10:58:00 0 0 \n", "100007 2145-03-31 00:44:00 0 0 \n", " 2145-04-02 14:10:00 0 0 \n", "100009 2162-05-17 13:19:00 0 0 \n", " 2162-05-17 17:14:00 0 0 \n", "100010 2109-12-10 10:25:00 0 0 \n", " 2109-12-10 12:11:00 0 0 \n", " 2109-12-10 13:05:00 0 0 \n", " 2109-12-10 13:58:00 0 0 \n", "100011 2177-08-29 04:44:00 0 0 \n", " 2177-08-29 06:55:00 0 0 \n", "100012 2177-03-14 07:38:00 0 0 \n", " 2177-03-14 11:42:00 0 0 \n", " 2177-03-15 08:05:00 0 0 \n", " 2177-03-15 14:01:00 0 0 \n", " 2177-03-15 21:42:00 0 0 \n", "100016 2188-05-24 12:00:00 0 0 \n", "100017 2103-03-11 05:10:00 0 0 \n", "100018 2176-08-29 15:29:00 0 0 \n", " 2176-08-30 09:23:00 0 0 \n", " 2176-08-30 10:19:00 0 0 \n", " 2176-08-30 11:29:00 0 0 \n", " 2176-08-30 12:40:00 0 0 \n", "100020 2142-11-30 21:54:00 0 0 \n", " 2142-12-03 00:17:00 0 0 \n", "100024 2170-09-19 10:25:00 0 0 \n", " 2170-09-19 16:33:00 0 0 \n", " 2170-09-20 02:04:00 0 0 \n", "... ... ... \n", "199976 2182-02-14 11:15:00 0 0 \n", " 2182-02-16 03:57:00 0 0 \n", " 2182-02-19 03:59:00 0 0 \n", " 2182-02-20 03:31:00 0 0 \n", " 2182-02-21 04:55:00 0 0 \n", "199979 2182-02-06 09:17:00 0 0 \n", " 2182-02-06 14:16:00 0 0 \n", "199981 2110-09-24 16:34:00 0 0 \n", " 2110-09-24 20:09:00 0 0 \n", " 2110-09-25 06:10:00 0 0 \n", "199987 2175-05-19 16:30:00 0 0 \n", "199988 2169-01-24 12:48:00 0 0 \n", " 2169-02-07 01:35:00 0 0 \n", " 2169-02-07 11:18:00 0 0 \n", " 2169-02-07 16:43:00 0 0 \n", " 2169-02-07 22:35:00 0 0 \n", " 2169-02-10 05:33:00 0 0 \n", "199993 2161-11-12 23:14:00 0 0 \n", " 2161-11-13 03:46:00 0 0 \n", "199994 2188-07-07 21:23:00 0 0 \n", " 2188-07-08 03:09:00 0 0 \n", " 2188-07-08 04:13:00 0 0 \n", " 2188-07-08 06:20:00 0 0 \n", "199998 2119-02-20 10:52:00 0 0 \n", " 2119-02-20 12:36:00 0 0 \n", " 2119-02-20 13:33:00 0 0 \n", " 2119-02-20 13:59:00 0 0 \n", " 2119-02-20 20:43:00 0 0 \n", "199999 2136-04-04 20:55:00 0 0 \n", " 2136-04-06 15:29:00 0 0 \n", "\n", "label \\\n", "status \n", "variable_type \n", "units \n", "description 1531(mmol/L)_>30.0 1531(mmol/L)_CLOTTED \n", "id datetime \n", "100001 2117-09-11 09:32:00 0 0 \n", "100003 2150-04-17 19:12:00 0 0 \n", "100006 2108-04-08 10:58:00 0 0 \n", "100007 2145-03-31 00:44:00 0 0 \n", " 2145-04-02 14:10:00 0 0 \n", "100009 2162-05-17 13:19:00 0 0 \n", " 2162-05-17 17:14:00 0 0 \n", "100010 2109-12-10 10:25:00 0 0 \n", " 2109-12-10 12:11:00 0 0 \n", " 2109-12-10 13:05:00 0 0 \n", " 2109-12-10 13:58:00 0 0 \n", "100011 2177-08-29 04:44:00 0 0 \n", " 2177-08-29 06:55:00 0 0 \n", "100012 2177-03-14 07:38:00 0 0 \n", " 2177-03-14 11:42:00 0 0 \n", " 2177-03-15 08:05:00 0 0 \n", " 2177-03-15 14:01:00 0 0 \n", " 2177-03-15 21:42:00 0 0 \n", "100016 2188-05-24 12:00:00 0 0 \n", "100017 2103-03-11 05:10:00 0 0 \n", "100018 2176-08-29 15:29:00 0 0 \n", " 2176-08-30 09:23:00 0 0 \n", " 2176-08-30 10:19:00 0 0 \n", " 2176-08-30 11:29:00 0 0 \n", " 2176-08-30 12:40:00 0 0 \n", "100020 2142-11-30 21:54:00 0 0 \n", " 2142-12-03 00:17:00 0 0 \n", "100024 2170-09-19 10:25:00 0 0 \n", " 2170-09-19 16:33:00 0 0 \n", " 2170-09-20 02:04:00 0 0 \n", "... ... ... \n", "199976 2182-02-14 11:15:00 0 0 \n", " 2182-02-16 03:57:00 0 0 \n", " 2182-02-19 03:59:00 0 0 \n", " 2182-02-20 03:31:00 0 0 \n", " 2182-02-21 04:55:00 0 0 \n", "199979 2182-02-06 09:17:00 0 0 \n", " 2182-02-06 14:16:00 0 0 \n", "199981 2110-09-24 16:34:00 0 0 \n", " 2110-09-24 20:09:00 0 0 \n", " 2110-09-25 06:10:00 0 0 \n", "199987 2175-05-19 16:30:00 0 0 \n", "199988 2169-01-24 12:48:00 0 0 \n", " 2169-02-07 01:35:00 0 0 \n", " 2169-02-07 11:18:00 0 0 \n", " 2169-02-07 16:43:00 0 0 \n", " 2169-02-07 22:35:00 0 0 \n", " 2169-02-10 05:33:00 0 0 \n", "199993 2161-11-12 23:14:00 0 0 \n", " 2161-11-13 03:46:00 0 0 \n", "199994 2188-07-07 21:23:00 0 0 \n", " 2188-07-08 03:09:00 0 0 \n", " 2188-07-08 04:13:00 0 0 \n", " 2188-07-08 06:20:00 0 0 \n", "199998 2119-02-20 10:52:00 0 0 \n", " 2119-02-20 12:36:00 0 0 \n", " 2119-02-20 13:33:00 0 0 \n", " 2119-02-20 13:59:00 0 0 \n", " 2119-02-20 20:43:00 0 0 \n", "199999 2136-04-04 20:55:00 0 0 \n", " 2136-04-06 15:29:00 0 0 \n", "\n", "label \\\n", "status \n", "variable_type \n", "units \n", "description 1531(mmol/L)_ERROR 1531(mmol/L)_VOIDED \n", "id datetime \n", "100001 2117-09-11 09:32:00 0 0 \n", "100003 2150-04-17 19:12:00 0 0 \n", "100006 2108-04-08 10:58:00 0 0 \n", "100007 2145-03-31 00:44:00 0 0 \n", " 2145-04-02 14:10:00 0 0 \n", "100009 2162-05-17 13:19:00 0 0 \n", " 2162-05-17 17:14:00 0 0 \n", "100010 2109-12-10 10:25:00 0 0 \n", " 2109-12-10 12:11:00 0 0 \n", " 2109-12-10 13:05:00 0 0 \n", " 2109-12-10 13:58:00 0 0 \n", "100011 2177-08-29 04:44:00 0 0 \n", " 2177-08-29 06:55:00 0 0 \n", "100012 2177-03-14 07:38:00 0 0 \n", " 2177-03-14 11:42:00 0 0 \n", " 2177-03-15 08:05:00 0 0 \n", " 2177-03-15 14:01:00 0 0 \n", " 2177-03-15 21:42:00 0 0 \n", "100016 2188-05-24 12:00:00 0 0 \n", "100017 2103-03-11 05:10:00 0 0 \n", "100018 2176-08-29 15:29:00 0 0 \n", " 2176-08-30 09:23:00 0 0 \n", " 2176-08-30 10:19:00 0 0 \n", " 2176-08-30 11:29:00 0 0 \n", " 2176-08-30 12:40:00 0 0 \n", "100020 2142-11-30 21:54:00 0 0 \n", " 2142-12-03 00:17:00 0 0 \n", "100024 2170-09-19 10:25:00 0 0 \n", " 2170-09-19 16:33:00 0 0 \n", " 2170-09-20 02:04:00 0 0 \n", "... ... ... \n", "199976 2182-02-14 11:15:00 0 0 \n", " 2182-02-16 03:57:00 0 0 \n", " 2182-02-19 03:59:00 0 0 \n", " 2182-02-20 03:31:00 0 0 \n", " 2182-02-21 04:55:00 0 0 \n", "199979 2182-02-06 09:17:00 0 0 \n", " 2182-02-06 14:16:00 0 0 \n", "199981 2110-09-24 16:34:00 0 0 \n", " 2110-09-24 20:09:00 0 0 \n", " 2110-09-25 06:10:00 0 0 \n", "199987 2175-05-19 16:30:00 0 0 \n", "199988 2169-01-24 12:48:00 0 0 \n", " 2169-02-07 01:35:00 0 0 \n", " 2169-02-07 11:18:00 0 0 \n", " 2169-02-07 16:43:00 0 0 \n", " 2169-02-07 22:35:00 0 0 \n", " 2169-02-10 05:33:00 0 0 \n", "199993 2161-11-12 23:14:00 0 0 \n", " 2161-11-13 03:46:00 0 0 \n", "199994 2188-07-07 21:23:00 0 0 \n", " 2188-07-08 03:09:00 0 0 \n", " 2188-07-08 04:13:00 0 0 \n", " 2188-07-08 06:20:00 0 0 \n", "199998 2119-02-20 10:52:00 0 0 \n", " 2119-02-20 12:36:00 0 0 \n", " 2119-02-20 13:33:00 0 0 \n", " 2119-02-20 13:59:00 0 0 \n", " 2119-02-20 20:43:00 0 0 \n", "199999 2136-04-04 20:55:00 0 0 \n", " 2136-04-06 15:29:00 0 0 \n", "\n", "label \n", "status \n", "variable_type \n", "units \n", "description 1531(mmol/L)_no data \n", "id datetime \n", "100001 2117-09-11 09:32:00 0 \n", "100003 2150-04-17 19:12:00 0 \n", "100006 2108-04-08 10:58:00 0 \n", "100007 2145-03-31 00:44:00 0 \n", " 2145-04-02 14:10:00 0 \n", "100009 2162-05-17 13:19:00 0 \n", " 2162-05-17 17:14:00 0 \n", "100010 2109-12-10 10:25:00 0 \n", " 2109-12-10 12:11:00 0 \n", " 2109-12-10 13:05:00 0 \n", " 2109-12-10 13:58:00 0 \n", "100011 2177-08-29 04:44:00 0 \n", " 2177-08-29 06:55:00 0 \n", "100012 2177-03-14 07:38:00 0 \n", " 2177-03-14 11:42:00 0 \n", " 2177-03-15 08:05:00 0 \n", " 2177-03-15 14:01:00 0 \n", " 2177-03-15 21:42:00 0 \n", "100016 2188-05-24 12:00:00 0 \n", "100017 2103-03-11 05:10:00 0 \n", "100018 2176-08-29 15:29:00 0 \n", " 2176-08-30 09:23:00 0 \n", " 2176-08-30 10:19:00 0 \n", " 2176-08-30 11:29:00 0 \n", " 2176-08-30 12:40:00 0 \n", "100020 2142-11-30 21:54:00 0 \n", " 2142-12-03 00:17:00 0 \n", "100024 2170-09-19 10:25:00 0 \n", " 2170-09-19 16:33:00 0 \n", " 2170-09-20 02:04:00 0 \n", "... ... \n", "199976 2182-02-14 11:15:00 0 \n", " 2182-02-16 03:57:00 0 \n", " 2182-02-19 03:59:00 0 \n", " 2182-02-20 03:31:00 0 \n", " 2182-02-21 04:55:00 0 \n", "199979 2182-02-06 09:17:00 0 \n", " 2182-02-06 14:16:00 0 \n", "199981 2110-09-24 16:34:00 0 \n", " 2110-09-24 20:09:00 0 \n", " 2110-09-25 06:10:00 0 \n", "199987 2175-05-19 16:30:00 0 \n", "199988 2169-01-24 12:48:00 0 \n", " 2169-02-07 01:35:00 0 \n", " 2169-02-07 11:18:00 0 \n", " 2169-02-07 16:43:00 0 \n", " 2169-02-07 22:35:00 0 \n", " 2169-02-10 05:33:00 0 \n", "199993 2161-11-12 23:14:00 0 \n", " 2161-11-13 03:46:00 0 \n", "199994 2188-07-07 21:23:00 0 \n", " 2188-07-08 03:09:00 0 \n", " 2188-07-08 04:13:00 0 \n", " 2188-07-08 06:20:00 0 \n", "199998 2119-02-20 10:52:00 0 \n", " 2119-02-20 12:36:00 0 \n", " 2119-02-20 13:33:00 0 \n", " 2119-02-20 13:59:00 0 \n", " 2119-02-20 20:43:00 0 \n", "199999 2136-04-04 20:55:00 0 \n", " 2136-04-06 15:29:00 0 \n", "\n", "[177450 rows x 29 columns]" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "nominal_cols = df_cleaned.columns.get_level_values('variable_type') == variable_type.NOMINAL\n", "\n", "for col_name in df_cleaned.loc[:,nominal_cols]:\n", " column = df_cleaned[col_nam]\n", " df_dummies = pd.get_dummies(column)\n", " dummy_col_names = [col_name[:-1] + ('{}_{}'.format(col_name[-1],text),) for text in df_dummies.columns]\n", " df_dummies.columns = pd.MultiIndex.from_tuples(dummy_col_names,names=df_cleaned.columns.names)\n", " \n", " df_cleaned.drop(col_name,axis=1,inplace=True)\n", " df_cleaned = df_cleaned.join(df_dummies,how='outer')\n", "\n", "df_cleaned" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Possible additional cleaning:\n", "\n", "1. Infer UOM \n", "2. Remove extreme values [DONE]" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import utils\n", "import mimic\n", "import transformers\n", "from sklearn.pipeline import Pipeline\n", "import icu_data_defs\n", "import units" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "df_temp = utils.open_df('data/mimic_data','extract/temperature body')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(mimic)\n", "pipeline = mimic.transform_pipeline('temperature body')" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
0185910.02166-08-25 00:00:0098.400001525878906degF678
1188670.02183-08-23 20:00:0099.300003051757812degF678
2185910.02166-08-12 08:00:0098.699996948242188degF678
3185910.02166-08-13 17:00:0099.900001525878906degF678
4166707.02122-02-11 20:00:0037.900001525878906degC676
5166707.02122-02-13 00:00:0097.800003051757812degF678
6185910.02166-08-11 21:00:0099.099998474121094degF678
7185910.02166-08-24 04:00:0098.900001525878906degF678
8176176.02116-12-24 16:00:0099.099998474121094degF678
9185910.02166-08-13 03:00:0099.199996948242188degF678
10185910.02166-08-30 18:00:0098.800003051757812degF678
11181750.02192-11-24 08:00:0098.5degF678
12185910.02166-08-14 12:00:00100.40000152587891degF678
13185910.02166-08-14 13:00:00100.69999694824219degF678
14185910.02166-08-15 22:00:0099.800003051757812degF678
15185910.02166-08-16 21:00:00100.59999847412109degF678
16185910.02166-08-23 08:00:0098degF678
17190797.02191-01-10 08:00:0097.900001525878906degF678
18157907.02129-06-10 23:00:00100.09999847412109degF678
19157907.02129-06-11 04:30:0098.800003051757812degF678
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 185910.0 2166-08-25 00:00:00 98.400001525878906 degF 678\n", "1 188670.0 2183-08-23 20:00:00 99.300003051757812 degF 678\n", "2 185910.0 2166-08-12 08:00:00 98.699996948242188 degF 678\n", "3 185910.0 2166-08-13 17:00:00 99.900001525878906 degF 678\n", "4 166707.0 2122-02-11 20:00:00 37.900001525878906 degC 676\n", "5 166707.0 2122-02-13 00:00:00 97.800003051757812 degF 678\n", "6 185910.0 2166-08-11 21:00:00 99.099998474121094 degF 678\n", "7 185910.0 2166-08-24 04:00:00 98.900001525878906 degF 678\n", "8 176176.0 2116-12-24 16:00:00 99.099998474121094 degF 678\n", "9 185910.0 2166-08-13 03:00:00 99.199996948242188 degF 678\n", "10 185910.0 2166-08-30 18:00:00 98.800003051757812 degF 678\n", "11 181750.0 2192-11-24 08:00:00 98.5 degF 678\n", "12 185910.0 2166-08-14 12:00:00 100.40000152587891 degF 678\n", "13 185910.0 2166-08-14 13:00:00 100.69999694824219 degF 678\n", "14 185910.0 2166-08-15 22:00:00 99.800003051757812 degF 678\n", "15 185910.0 2166-08-16 21:00:00 100.59999847412109 degF 678\n", "16 185910.0 2166-08-23 08:00:00 98 degF 678\n", "17 190797.0 2191-01-10 08:00:00 97.900001525878906 degF 678\n", "18 157907.0 2129-06-10 23:00:00 100.09999847412109 degF 678\n", "19 157907.0 2129-06-11 04:30:00 98.800003051757812 degF 678" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp.head(20)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df_temp_tr = pipeline.transform(df_temp)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
unitsdegFdegC
description223761678223762676
iddatetime
1000012117-09-11 13:48:0098NoneNoneNone
2117-09-11 16:00:0097.9NoneNoneNone
2117-09-11 19:00:0099.5NoneNoneNone
2117-09-11 22:00:00100NoneNoneNone
2117-09-12 01:00:0099.9NoneNoneNone
2117-09-12 04:00:0097.7NoneNoneNone
2117-09-12 08:00:0097.8NoneNoneNone
2117-09-12 12:00:0097.5NoneNoneNone
2117-09-12 19:00:0099.8NoneNoneNone
2117-09-13 00:00:0099.7NoneNoneNone
2117-09-13 04:00:0099.3NoneNoneNone
2117-09-13 08:00:0099.4NoneNoneNone
2117-09-13 12:00:0098.8NoneNoneNone
2117-09-13 16:00:0099.3NoneNoneNone
2117-09-13 22:00:0099.9NoneNoneNone
2117-09-14 00:00:0099.4NoneNoneNone
2117-09-14 04:00:0099.6NoneNoneNone
2117-09-14 08:00:0099NoneNoneNone
2117-09-14 13:00:0098NoneNoneNone
2117-09-14 19:00:0099.5NoneNoneNone
\n", "
" ], "text/plain": [ "label temperature body \n", "units degF degC \n", "description 223761 678 223762 676 \n", "id datetime \n", "100001 2117-09-11 13:48:00 98 None None None\n", " 2117-09-11 16:00:00 97.9 None None None\n", " 2117-09-11 19:00:00 99.5 None None None\n", " 2117-09-11 22:00:00 100 None None None\n", " 2117-09-12 01:00:00 99.9 None None None\n", " 2117-09-12 04:00:00 97.7 None None None\n", " 2117-09-12 08:00:00 97.8 None None None\n", " 2117-09-12 12:00:00 97.5 None None None\n", " 2117-09-12 19:00:00 99.8 None None None\n", " 2117-09-13 00:00:00 99.7 None None None\n", " 2117-09-13 04:00:00 99.3 None None None\n", " 2117-09-13 08:00:00 99.4 None None None\n", " 2117-09-13 12:00:00 98.8 None None None\n", " 2117-09-13 16:00:00 99.3 None None None\n", " 2117-09-13 22:00:00 99.9 None None None\n", " 2117-09-14 00:00:00 99.4 None None None\n", " 2117-09-14 04:00:00 99.6 None None None\n", " 2117-09-14 08:00:00 99 None None None\n", " 2117-09-14 13:00:00 98 None None None\n", " 2117-09-14 19:00:00 99.5 None None None" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp_tr.head(20)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(transformers)\n", "reload(units)\n", "reload(icu_data_defs)\n", "data_dict = icu_data_defs.data_dictionary('config/data_definitions.xlsx')\n", "agg_func = lambda x:x.iloc[0]\n", "ureg = units.MedicalUreg()\n", "standard_pipeline = Pipeline([\n", " ('drop_small_columns',transformers.remove_small_columns(threshold=5)),\n", " ('aggregate_same_datetime',transformers.same_index_aggregator(agg_func)),\n", " ('split_dtype',transformers.split_dtype())\n", " ])\n", "\n", "stnd_cols = transformers.column_standardizer(data_dict,ureg)\n", "drop_oob = transformers.oob_value_remover(data_dict)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df_temp_cl = standard_pipeline.transform(df_temp_tr)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
unitsdegFdegC
description223761678223762676
iddatetime
1000012117-09-11 13:48:0098.0NaNNaNNaN
2117-09-11 16:00:0097.9NaNNaNNaN
2117-09-11 19:00:0099.5NaNNaNNaN
2117-09-11 22:00:00100.0NaNNaNNaN
2117-09-12 01:00:0099.9NaNNaNNaN
2117-09-12 04:00:0097.7NaNNaNNaN
2117-09-12 08:00:0097.8NaNNaNNaN
2117-09-12 12:00:0097.5NaNNaNNaN
2117-09-12 19:00:0099.8NaNNaNNaN
2117-09-13 00:00:0099.7NaNNaNNaN
2117-09-13 04:00:0099.3NaNNaNNaN
2117-09-13 08:00:0099.4NaNNaNNaN
2117-09-13 12:00:0098.8NaNNaNNaN
2117-09-13 16:00:0099.3NaNNaNNaN
2117-09-13 22:00:0099.9NaNNaNNaN
2117-09-14 00:00:0099.4NaNNaNNaN
2117-09-14 04:00:0099.6NaNNaNNaN
2117-09-14 08:00:0099.0NaNNaNNaN
2117-09-14 13:00:0098.0NaNNaNNaN
2117-09-14 19:00:0099.5NaNNaNNaN
\n", "
" ], "text/plain": [ "label temperature body \n", "units degF degC \n", "description 223761 678 223762 676 \n", "id datetime \n", "100001 2117-09-11 13:48:00 98.0 NaN NaN NaN\n", " 2117-09-11 16:00:00 97.9 NaN NaN NaN\n", " 2117-09-11 19:00:00 99.5 NaN NaN NaN\n", " 2117-09-11 22:00:00 100.0 NaN NaN NaN\n", " 2117-09-12 01:00:00 99.9 NaN NaN NaN\n", " 2117-09-12 04:00:00 97.7 NaN NaN NaN\n", " 2117-09-12 08:00:00 97.8 NaN NaN NaN\n", " 2117-09-12 12:00:00 97.5 NaN NaN NaN\n", " 2117-09-12 19:00:00 99.8 NaN NaN NaN\n", " 2117-09-13 00:00:00 99.7 NaN NaN NaN\n", " 2117-09-13 04:00:00 99.3 NaN NaN NaN\n", " 2117-09-13 08:00:00 99.4 NaN NaN NaN\n", " 2117-09-13 12:00:00 98.8 NaN NaN NaN\n", " 2117-09-13 16:00:00 99.3 NaN NaN NaN\n", " 2117-09-13 22:00:00 99.9 NaN NaN NaN\n", " 2117-09-14 00:00:00 99.4 NaN NaN NaN\n", " 2117-09-14 04:00:00 99.6 NaN NaN NaN\n", " 2117-09-14 08:00:00 99.0 NaN NaN NaN\n", " 2117-09-14 13:00:00 98.0 NaN NaN NaN\n", " 2117-09-14 19:00:00 99.5 NaN NaN NaN" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp_cl.head(20)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "df_temp_conv = stnd_cols.transform(df_temp_cl)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
statusknown
variable_typeqn
unitsdegF
description223761678223762(degC)676(degC)
count522143.000000768158.00000074144.000000370309.000000
mean98.40797098.57060699.19382198.765740
std14.6811272.6666149.7352242.608901
min-99.9000000.00000026.60000032.000000
25%97.50000097.59999897.52000097.879998
50%98.30000098.59999898.60000098.960002
75%99.30000099.59999899.68000099.860002
max9637.000000109.000000709.700000115.700000
\n", "
" ], "text/plain": [ "label temperature body \n", "status known \n", "variable_type qn \n", "units degF \n", "description 223761 678 223762(degC) 676(degC)\n", "count 522143.000000 768158.000000 74144.000000 370309.000000\n", "mean 98.407970 98.570606 99.193821 98.765740\n", "std 14.681127 2.666614 9.735224 2.608901\n", "min -99.900000 0.000000 26.600000 32.000000\n", "25% 97.500000 97.599998 97.520000 97.879998\n", "50% 98.300000 98.599998 98.600000 98.960002\n", "75% 99.300000 99.599998 99.680000 99.860002\n", "max 9637.000000 109.000000 709.700000 115.700000" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp_conv.describe()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df_temp_no_oob = drop_oob.transform(df_temp_conv)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
statusknown
variable_typeqn
unitsdegF
description223761678223762(degC)676(degC)
count522116.000000768158.00000073638.000000370309.000000
mean98.35264298.57060698.43064698.765740
std2.8259222.6666142.4671252.608901
min0.0000000.00000026.60000032.000000
25%97.50000097.59999897.34000097.879998
50%98.30000098.59999898.60000098.960002
75%99.30000099.59999899.68000099.860002
max129.000000109.000000113.000000115.700000
\n", "
" ], "text/plain": [ "label temperature body \n", "status known \n", "variable_type qn \n", "units degF \n", "description 223761 678 223762(degC) 676(degC)\n", "count 522116.000000 768158.000000 73638.000000 370309.000000\n", "mean 98.352642 98.570606 98.430646 98.765740\n", "std 2.825922 2.666614 2.467125 2.608901\n", "min 0.000000 0.000000 26.600000 32.000000\n", "25% 97.500000 97.599998 97.340000 97.879998\n", "50% 98.300000 98.599998 98.600000 98.960002\n", "75% 99.300000 99.599998 99.680000 99.860002\n", "max 129.000000 109.000000 113.000000 115.700000" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp_no_oob.describe()" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "collapsed": false }, "outputs": [], "source": [ "combine_like = transformers.combine_like_cols()\n", "df_temp_combined = combine_like.transform(df_temp_no_oob)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
statusknown
variable_typeqn
unitsdegF
descriptionall
iddatetime
1000012117-09-11 13:48:0098.000000
2117-09-11 16:00:0097.900000
2117-09-11 19:00:0099.500000
2117-09-11 22:00:00100.000000
2117-09-12 01:00:0099.900000
2117-09-12 04:00:0097.700000
2117-09-12 08:00:0097.800000
2117-09-12 12:00:0097.500000
2117-09-12 19:00:0099.800000
2117-09-13 00:00:0099.700000
2117-09-13 04:00:0099.300000
2117-09-13 08:00:0099.400000
2117-09-13 12:00:0098.800000
2117-09-13 16:00:0099.300000
2117-09-13 22:00:0099.900000
2117-09-14 00:00:0099.400000
2117-09-14 04:00:0099.600000
2117-09-14 08:00:0099.000000
2117-09-14 13:00:0098.000000
2117-09-14 19:00:0099.500000
2117-09-15 00:00:0099.600000
2117-09-15 08:00:0098.300000
2117-09-15 12:00:0097.800000
2117-09-15 16:00:0098.600000
1000032150-04-17 20:31:0095.900000
2150-04-17 22:00:0098.200000
2150-04-18 00:00:0097.400000
2150-04-18 04:00:0098.000000
2150-04-18 08:00:0097.400000
2150-04-18 11:18:0096.300000
.........
1999982119-02-21 02:00:00100.400000
2119-02-21 03:00:00100.220003
2119-02-21 04:00:0099.860002
2119-02-21 05:00:00100.220003
2119-02-21 06:00:00100.400000
2119-02-21 07:00:00100.400000
2119-02-21 08:00:00100.400000
1999992136-04-06 16:07:0097.800000
2136-04-06 20:00:0097.500000
2136-04-07 00:00:0097.400000
2136-04-07 02:00:00101.000000
2136-04-07 05:00:00100.500000
2136-04-07 08:00:0099.000000
2136-04-07 12:00:0099.800000
2136-04-07 15:00:00100.700000
2136-04-07 18:00:0097.600000
2136-04-07 22:00:0098.600000
2136-04-08 04:00:0098.500000
2136-04-08 09:00:0097.800000
2136-04-08 16:00:0097.700000
2136-04-08 20:00:0099.000000
2136-04-09 00:00:0099.600000
2136-04-09 05:00:0099.100000
2136-04-09 08:00:0097.800000
2136-04-09 12:00:0098.300000
2136-04-09 16:00:0098.000000
2136-04-09 20:00:0097.400000
2136-04-10 00:00:0097.400000
2136-04-10 03:00:0097.400000
2136-04-10 08:00:0098.100000
\n", "

1731503 rows × 1 columns

\n", "
" ], "text/plain": [ "label temperature body\n", "status known\n", "variable_type qn\n", "units degF\n", "description all\n", "id datetime \n", "100001 2117-09-11 13:48:00 98.000000\n", " 2117-09-11 16:00:00 97.900000\n", " 2117-09-11 19:00:00 99.500000\n", " 2117-09-11 22:00:00 100.000000\n", " 2117-09-12 01:00:00 99.900000\n", " 2117-09-12 04:00:00 97.700000\n", " 2117-09-12 08:00:00 97.800000\n", " 2117-09-12 12:00:00 97.500000\n", " 2117-09-12 19:00:00 99.800000\n", " 2117-09-13 00:00:00 99.700000\n", " 2117-09-13 04:00:00 99.300000\n", " 2117-09-13 08:00:00 99.400000\n", " 2117-09-13 12:00:00 98.800000\n", " 2117-09-13 16:00:00 99.300000\n", " 2117-09-13 22:00:00 99.900000\n", " 2117-09-14 00:00:00 99.400000\n", " 2117-09-14 04:00:00 99.600000\n", " 2117-09-14 08:00:00 99.000000\n", " 2117-09-14 13:00:00 98.000000\n", " 2117-09-14 19:00:00 99.500000\n", " 2117-09-15 00:00:00 99.600000\n", " 2117-09-15 08:00:00 98.300000\n", " 2117-09-15 12:00:00 97.800000\n", " 2117-09-15 16:00:00 98.600000\n", "100003 2150-04-17 20:31:00 95.900000\n", " 2150-04-17 22:00:00 98.200000\n", " 2150-04-18 00:00:00 97.400000\n", " 2150-04-18 04:00:00 98.000000\n", " 2150-04-18 08:00:00 97.400000\n", " 2150-04-18 11:18:00 96.300000\n", "... ...\n", "199998 2119-02-21 02:00:00 100.400000\n", " 2119-02-21 03:00:00 100.220003\n", " 2119-02-21 04:00:00 99.860002\n", " 2119-02-21 05:00:00 100.220003\n", " 2119-02-21 06:00:00 100.400000\n", " 2119-02-21 07:00:00 100.400000\n", " 2119-02-21 08:00:00 100.400000\n", "199999 2136-04-06 16:07:00 97.800000\n", " 2136-04-06 20:00:00 97.500000\n", " 2136-04-07 00:00:00 97.400000\n", " 2136-04-07 02:00:00 101.000000\n", " 2136-04-07 05:00:00 100.500000\n", " 2136-04-07 08:00:00 99.000000\n", " 2136-04-07 12:00:00 99.800000\n", " 2136-04-07 15:00:00 100.700000\n", " 2136-04-07 18:00:00 97.600000\n", " 2136-04-07 22:00:00 98.600000\n", " 2136-04-08 04:00:00 98.500000\n", " 2136-04-08 09:00:00 97.800000\n", " 2136-04-08 16:00:00 97.700000\n", " 2136-04-08 20:00:00 99.000000\n", " 2136-04-09 00:00:00 99.600000\n", " 2136-04-09 05:00:00 99.100000\n", " 2136-04-09 08:00:00 97.800000\n", " 2136-04-09 12:00:00 98.300000\n", " 2136-04-09 16:00:00 98.000000\n", " 2136-04-09 20:00:00 97.400000\n", " 2136-04-10 00:00:00 97.400000\n", " 2136-04-10 03:00:00 97.400000\n", " 2136-04-10 08:00:00 98.100000\n", "\n", "[1731503 rows x 1 columns]" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp_combined" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Segmenting" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import segmenting" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "collapsed": true }, "outputs": [], "source": [ "df_context = utils.open_df('data/mimic_data','context')" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pt_ididstart_dtend_dtlangreligionmarital_statusethnicitydx_infoadmission_location...dodicd_codesageicustay_iddbsourcefirst_iculast_icuintimeouttimelos
0871906592191-02-25 20:30:002191-04-25 15:18:00UNOBTAINABLEUNKNOWN/NOT SPECIFIEDNEWBORNPHYS REFERRAL/NORMAL DELI...None[V3101, 76515, 76524, 769, 7707, 7775, 7742, 7...0 days 20:30:00277633.0carevueNICUNICU2191-02-25 21:09:052191-04-25 15:38:5658.7707
1881230102111-08-29 03:03:002111-09-03 14:24:00BLACK/AFRICAN AMERICANS/P MOTOR VEHICLE ACCIDENT-STABBINGEMERGENCY ROOM ADMIT...None[86113, 5119, 4239, E966]10467 days 03:03:00297289.0carevueTSICUTSICU2111-08-29 03:04:422111-08-30 21:08:091.7524
2891886462185-06-17 05:22:002185-06-21 11:15:00UNOBTAINABLEUNKNOWN/NOT SPECIFIEDNEWBORNPHYS REFERRAL/NORMAL DELI...None[V3101, 7742, 76516, 76527, 22801, V290]0 days 05:22:00249786.0carevueNICUNICU2185-06-17 05:46:002185-06-21 12:00:414.2602
3911212052177-04-23 00:08:002177-05-10 15:16:00JEWISHMARRIEDWHITEFEVEREMERGENCY ROOM ADMIT...2177-05-10 00:00:00[20008, 2765, 5559, 99812, 51881, 4470, 135, 5...29794 days 00:08:00218528.0carevueCSRUCSRU2177-04-27 02:08:002177-04-27 14:03:000.4965
4911212052177-04-23 00:08:002177-05-10 15:16:00JEWISHMARRIEDWHITEFEVEREMERGENCY ROOM ADMIT...2177-05-10 00:00:00[20008, 2765, 5559, 99812, 51881, 4470, 135, 5...29794 days 00:08:00256972.0carevueMICUMICU2177-05-07 03:52:002177-05-10 15:16:003.4750
\n", "

5 rows × 22 columns

\n", "
" ], "text/plain": [ " pt_id id start_dt end_dt lang religion \\\n", "0 87 190659 2191-02-25 20:30:00 2191-04-25 15:18:00 UNOBTAINABLE \n", "1 88 123010 2111-08-29 03:03:00 2111-09-03 14:24:00 \n", "2 89 188646 2185-06-17 05:22:00 2185-06-21 11:15:00 UNOBTAINABLE \n", "3 91 121205 2177-04-23 00:08:00 2177-05-10 15:16:00 JEWISH \n", "4 91 121205 2177-04-23 00:08:00 2177-05-10 15:16:00 JEWISH \n", "\n", " marital_status ethnicity dx_info \\\n", "0 UNKNOWN/NOT SPECIFIED NEWBORN \n", "1 BLACK/AFRICAN AMERICAN S/P MOTOR VEHICLE ACCIDENT-STABBING \n", "2 UNKNOWN/NOT SPECIFIED NEWBORN \n", "3 MARRIED WHITE FEVER \n", "4 MARRIED WHITE FEVER \n", "\n", " admission_location ... dod \\\n", "0 PHYS REFERRAL/NORMAL DELI ... None \n", "1 EMERGENCY ROOM ADMIT ... None \n", "2 PHYS REFERRAL/NORMAL DELI ... None \n", "3 EMERGENCY ROOM ADMIT ... 2177-05-10 00:00:00 \n", "4 EMERGENCY ROOM ADMIT ... 2177-05-10 00:00:00 \n", "\n", " icd_codes age \\\n", "0 [V3101, 76515, 76524, 769, 7707, 7775, 7742, 7... 0 days 20:30:00 \n", "1 [86113, 5119, 4239, E966] 10467 days 03:03:00 \n", "2 [V3101, 7742, 76516, 76527, 22801, V290] 0 days 05:22:00 \n", "3 [20008, 2765, 5559, 99812, 51881, 4470, 135, 5... 29794 days 00:08:00 \n", "4 [20008, 2765, 5559, 99812, 51881, 4470, 135, 5... 29794 days 00:08:00 \n", "\n", " icustay_id dbsource first_icu last_icu intime \\\n", "0 277633.0 carevue NICU NICU 2191-02-25 21:09:05 \n", "1 297289.0 carevue TSICU TSICU 2111-08-29 03:04:42 \n", "2 249786.0 carevue NICU NICU 2185-06-17 05:46:00 \n", "3 218528.0 carevue CSRU CSRU 2177-04-27 02:08:00 \n", "4 256972.0 carevue MICU MICU 2177-05-07 03:52:00 \n", "\n", " outtime los \n", "0 2191-04-25 15:38:56 58.7707 \n", "1 2111-08-30 21:08:09 1.7524 \n", "2 2185-06-21 12:00:41 4.2602 \n", "3 2177-04-27 14:03:00 0.4965 \n", "4 2177-05-10 15:16:00 3.4750 \n", "\n", "[5 rows x 22 columns]" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_context.head()" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "collapsed": false }, "outputs": [], "source": [ "end_dt = df_temp_combined.iloc[:,0].groupby(level='id').apply(lambda x:x.sample(1))\n", "end_dt = end_dt.reset_index(level=0,drop=True).reset_index(level=1,drop=False).iloc[:,0]" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(segmenting)\n", "reload(utils)\n", "all_before = segmenting.all_before(end_dt,df_context)\n", "n_hrs_before = segmenting.n_hrs_before(end_dt,10)\n", "periodic = segmenting.periodic(10,df_context)" ] }, { "cell_type": "code", "execution_count": 138, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\tools\\merge.py:480: UserWarning: merging between different levels can give an unintended result (3 levels on the left, 1 on the right)\n", " warnings.warn(msg, UserWarning)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
unitsdegFdegC
description223761678223762676
iddatetimeseg_id
1000012117-09-11 13:48:00098.0NaNNaNNaN
2117-09-11 16:00:00097.9NaNNaNNaN
2117-09-11 19:00:00099.5NaNNaNNaN
2117-09-11 22:00:000100.0NaNNaNNaN
2117-09-12 01:00:00099.9NaNNaNNaN
2117-09-12 04:00:00097.7NaNNaNNaN
2117-09-12 08:00:00097.8NaNNaNNaN
2117-09-12 12:00:00097.5NaNNaNNaN
1000032150-04-17 20:31:00095.9NaNNaNNaN
2150-04-17 22:00:00098.2NaNNaNNaN
2150-04-18 00:00:00097.4NaNNaNNaN
1000062108-04-06 16:30:000NaN97.000000NaNNaN
2108-04-06 20:00:000NaN97.400002NaNNaN
2108-04-07 00:00:000NaN97.800003NaNNaN
2108-04-07 04:00:000NaN97.000000NaNNaN
2108-04-07 08:00:000NaN97.099998NaNNaN
2108-04-07 12:00:000NaN97.000000NaNNaN
2108-04-07 20:00:000NaN98.800003NaNNaN
2108-04-08 00:00:000NaN98.599998NaNNaN
2108-04-08 04:00:000NaN98.000000NaNNaN
2108-04-08 08:00:000NaN97.199997NaNNaN
2108-04-08 12:00:000NaN97.000000NaNNaN
2108-04-08 16:00:000NaN98.000000NaNNaN
2108-04-08 20:00:000NaN98.800003NaNNaN
2108-04-09 00:00:000NaN97.199997NaNNaN
2108-04-09 04:00:000NaN97.400002NaNNaN
2108-04-09 08:00:000NaN97.099998NaNNaN
2108-04-09 12:00:000NaN97.300003NaNNaN
2108-04-09 20:00:000NaN97.599998NaNNaN
2108-04-10 00:00:000NaN97.000000NaNNaN
.....................
1999982119-02-20 17:15:000NaNNaNNaN36.299999
2119-02-20 17:30:000NaNNaNNaN36.500000
2119-02-20 17:45:000NaNNaNNaN36.599998
2119-02-20 18:00:000NaNNaNNaN36.900002
2119-02-20 18:30:000NaNNaNNaN37.200001
2119-02-20 19:00:000NaNNaNNaN37.599998
2119-02-20 20:00:000NaNNaNNaN37.799999
2119-02-20 21:00:000NaNNaNNaN37.799999
2119-02-20 22:00:000NaNNaNNaN37.799999
2119-02-20 23:00:000NaNNaNNaN37.700001
1999992136-04-06 16:07:00097.8NaNNaNNaN
2136-04-06 20:00:00097.5NaNNaNNaN
2136-04-07 00:00:00097.4NaNNaNNaN
2136-04-07 02:00:000101.0NaNNaNNaN
2136-04-07 05:00:000100.5NaNNaNNaN
2136-04-07 08:00:00099.0NaNNaNNaN
2136-04-07 12:00:00099.8NaNNaNNaN
2136-04-07 15:00:000100.7NaNNaNNaN
2136-04-07 18:00:00097.6NaNNaNNaN
2136-04-07 22:00:00098.6NaNNaNNaN
2136-04-08 04:00:00098.5NaNNaNNaN
2136-04-08 09:00:00097.8NaNNaNNaN
2136-04-08 16:00:00097.7NaNNaNNaN
2136-04-08 20:00:00099.0NaNNaNNaN
2136-04-09 00:00:00099.6NaNNaNNaN
2136-04-09 05:00:00099.1NaNNaNNaN
2136-04-09 08:00:00097.8NaNNaNNaN
2136-04-09 12:00:00098.3NaNNaNNaN
2136-04-09 16:00:00098.0NaNNaNNaN
2136-04-09 20:00:00097.4NaNNaNNaN
\n", "

839877 rows × 4 columns

\n", "
" ], "text/plain": [ "label temperature body \\\n", "units degF degC \n", "description 223761 678 223762 \n", "id datetime seg_id \n", "100001 2117-09-11 13:48:00 0 98.0 NaN NaN \n", " 2117-09-11 16:00:00 0 97.9 NaN NaN \n", " 2117-09-11 19:00:00 0 99.5 NaN NaN \n", " 2117-09-11 22:00:00 0 100.0 NaN NaN \n", " 2117-09-12 01:00:00 0 99.9 NaN NaN \n", " 2117-09-12 04:00:00 0 97.7 NaN NaN \n", " 2117-09-12 08:00:00 0 97.8 NaN NaN \n", " 2117-09-12 12:00:00 0 97.5 NaN NaN \n", "100003 2150-04-17 20:31:00 0 95.9 NaN NaN \n", " 2150-04-17 22:00:00 0 98.2 NaN NaN \n", " 2150-04-18 00:00:00 0 97.4 NaN NaN \n", "100006 2108-04-06 16:30:00 0 NaN 97.000000 NaN \n", " 2108-04-06 20:00:00 0 NaN 97.400002 NaN \n", " 2108-04-07 00:00:00 0 NaN 97.800003 NaN \n", " 2108-04-07 04:00:00 0 NaN 97.000000 NaN \n", " 2108-04-07 08:00:00 0 NaN 97.099998 NaN \n", " 2108-04-07 12:00:00 0 NaN 97.000000 NaN \n", " 2108-04-07 20:00:00 0 NaN 98.800003 NaN \n", " 2108-04-08 00:00:00 0 NaN 98.599998 NaN \n", " 2108-04-08 04:00:00 0 NaN 98.000000 NaN \n", " 2108-04-08 08:00:00 0 NaN 97.199997 NaN \n", " 2108-04-08 12:00:00 0 NaN 97.000000 NaN \n", " 2108-04-08 16:00:00 0 NaN 98.000000 NaN \n", " 2108-04-08 20:00:00 0 NaN 98.800003 NaN \n", " 2108-04-09 00:00:00 0 NaN 97.199997 NaN \n", " 2108-04-09 04:00:00 0 NaN 97.400002 NaN \n", " 2108-04-09 08:00:00 0 NaN 97.099998 NaN \n", " 2108-04-09 12:00:00 0 NaN 97.300003 NaN \n", " 2108-04-09 20:00:00 0 NaN 97.599998 NaN \n", " 2108-04-10 00:00:00 0 NaN 97.000000 NaN \n", "... ... ... ... \n", "199998 2119-02-20 17:15:00 0 NaN NaN NaN \n", " 2119-02-20 17:30:00 0 NaN NaN NaN \n", " 2119-02-20 17:45:00 0 NaN NaN NaN \n", " 2119-02-20 18:00:00 0 NaN NaN NaN \n", " 2119-02-20 18:30:00 0 NaN NaN NaN \n", " 2119-02-20 19:00:00 0 NaN NaN NaN \n", " 2119-02-20 20:00:00 0 NaN NaN NaN \n", " 2119-02-20 21:00:00 0 NaN NaN NaN \n", " 2119-02-20 22:00:00 0 NaN NaN NaN \n", " 2119-02-20 23:00:00 0 NaN NaN NaN \n", "199999 2136-04-06 16:07:00 0 97.8 NaN NaN \n", " 2136-04-06 20:00:00 0 97.5 NaN NaN \n", " 2136-04-07 00:00:00 0 97.4 NaN NaN \n", " 2136-04-07 02:00:00 0 101.0 NaN NaN \n", " 2136-04-07 05:00:00 0 100.5 NaN NaN \n", " 2136-04-07 08:00:00 0 99.0 NaN NaN \n", " 2136-04-07 12:00:00 0 99.8 NaN NaN \n", " 2136-04-07 15:00:00 0 100.7 NaN NaN \n", " 2136-04-07 18:00:00 0 97.6 NaN NaN \n", " 2136-04-07 22:00:00 0 98.6 NaN NaN \n", " 2136-04-08 04:00:00 0 98.5 NaN NaN \n", " 2136-04-08 09:00:00 0 97.8 NaN NaN \n", " 2136-04-08 16:00:00 0 97.7 NaN NaN \n", " 2136-04-08 20:00:00 0 99.0 NaN NaN \n", " 2136-04-09 00:00:00 0 99.6 NaN NaN \n", " 2136-04-09 05:00:00 0 99.1 NaN NaN \n", " 2136-04-09 08:00:00 0 97.8 NaN NaN \n", " 2136-04-09 12:00:00 0 98.3 NaN NaN \n", " 2136-04-09 16:00:00 0 98.0 NaN NaN \n", " 2136-04-09 20:00:00 0 97.4 NaN NaN \n", "\n", "label \n", "units \n", "description 676 \n", "id datetime seg_id \n", "100001 2117-09-11 13:48:00 0 NaN \n", " 2117-09-11 16:00:00 0 NaN \n", " 2117-09-11 19:00:00 0 NaN \n", " 2117-09-11 22:00:00 0 NaN \n", " 2117-09-12 01:00:00 0 NaN \n", " 2117-09-12 04:00:00 0 NaN \n", " 2117-09-12 08:00:00 0 NaN \n", " 2117-09-12 12:00:00 0 NaN \n", "100003 2150-04-17 20:31:00 0 NaN \n", " 2150-04-17 22:00:00 0 NaN \n", " 2150-04-18 00:00:00 0 NaN \n", "100006 2108-04-06 16:30:00 0 NaN \n", " 2108-04-06 20:00:00 0 NaN \n", " 2108-04-07 00:00:00 0 NaN \n", " 2108-04-07 04:00:00 0 NaN \n", " 2108-04-07 08:00:00 0 NaN \n", " 2108-04-07 12:00:00 0 NaN \n", " 2108-04-07 20:00:00 0 NaN \n", " 2108-04-08 00:00:00 0 NaN \n", " 2108-04-08 04:00:00 0 NaN \n", " 2108-04-08 08:00:00 0 NaN \n", " 2108-04-08 12:00:00 0 NaN \n", " 2108-04-08 16:00:00 0 NaN \n", " 2108-04-08 20:00:00 0 NaN \n", " 2108-04-09 00:00:00 0 NaN \n", " 2108-04-09 04:00:00 0 NaN \n", " 2108-04-09 08:00:00 0 NaN \n", " 2108-04-09 12:00:00 0 NaN \n", " 2108-04-09 20:00:00 0 NaN \n", " 2108-04-10 00:00:00 0 NaN \n", "... ... \n", "199998 2119-02-20 17:15:00 0 36.299999 \n", " 2119-02-20 17:30:00 0 36.500000 \n", " 2119-02-20 17:45:00 0 36.599998 \n", " 2119-02-20 18:00:00 0 36.900002 \n", " 2119-02-20 18:30:00 0 37.200001 \n", " 2119-02-20 19:00:00 0 37.599998 \n", " 2119-02-20 20:00:00 0 37.799999 \n", " 2119-02-20 21:00:00 0 37.799999 \n", " 2119-02-20 22:00:00 0 37.799999 \n", " 2119-02-20 23:00:00 0 37.700001 \n", "199999 2136-04-06 16:07:00 0 NaN \n", " 2136-04-06 20:00:00 0 NaN \n", " 2136-04-07 00:00:00 0 NaN \n", " 2136-04-07 02:00:00 0 NaN \n", " 2136-04-07 05:00:00 0 NaN \n", " 2136-04-07 08:00:00 0 NaN \n", " 2136-04-07 12:00:00 0 NaN \n", " 2136-04-07 15:00:00 0 NaN \n", " 2136-04-07 18:00:00 0 NaN \n", " 2136-04-07 22:00:00 0 NaN \n", " 2136-04-08 04:00:00 0 NaN \n", " 2136-04-08 09:00:00 0 NaN \n", " 2136-04-08 16:00:00 0 NaN \n", " 2136-04-08 20:00:00 0 NaN \n", " 2136-04-09 00:00:00 0 NaN \n", " 2136-04-09 05:00:00 0 NaN \n", " 2136-04-09 08:00:00 0 NaN \n", " 2136-04-09 12:00:00 0 NaN \n", " 2136-04-09 16:00:00 0 NaN \n", " 2136-04-09 20:00:00 0 NaN \n", "\n", "[839877 rows x 4 columns]" ] }, "execution_count": 138, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp_all_before = all_before.transform(df_temp_cl)\n", "df_temp_all_before[df_temp_all_before.index.get_level_values('seg_id') > -1]" ] }, { "cell_type": "code", "execution_count": 262, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " seg_id\n", "id datetime \n", "100001 2117-09-11 22:00:00 0\n", " 2117-09-12 01:00:00 0\n", " 2117-09-12 04:00:00 0\n", "100006 2108-04-07 20:00:00 0\n", "100007 2145-04-04 00:00:00 0\n", " 2145-04-04 04:00:00 0\n", "100009 2162-05-19 03:00:00 0\n", "100010 2109-12-10 22:47:00 0\n", " 2109-12-11 00:00:00 0\n", "100011 2177-09-07 09:00:00 0\n", " 2177-09-07 13:00:00 0\n", " 2177-09-07 17:00:00 0\n", "label temperature body \n", "status known \n", "variable_type qn \n", "units degF \n", "description 223761 678 223762(degC) 676(degC)\n", "id datetime seg_id \n", "100001 2117-09-11 13:48:00 -1 98.0 NaN NaN NaN\n", " 2117-09-11 16:00:00 -1 97.9 NaN NaN NaN\n", " 2117-09-11 19:00:00 -1 99.5 NaN NaN NaN\n", " 2117-09-11 22:00:00 0 100.0 NaN NaN NaN\n", " 2117-09-12 01:00:00 0 99.9 NaN NaN NaN\n", " 2117-09-12 04:00:00 0 97.7 NaN NaN NaN\n", " 2117-09-12 08:00:00 -1 97.8 NaN NaN NaN\n", " 2117-09-12 12:00:00 -1 97.5 NaN NaN NaN\n", " 2117-09-12 19:00:00 -1 99.8 NaN NaN NaN\n", " 2117-09-13 00:00:00 -1 99.7 NaN NaN NaN\n", " 2117-09-13 04:00:00 -1 99.3 NaN NaN NaN\n", " 2117-09-13 08:00:00 -1 99.4 NaN NaN NaN\n", " 2117-09-13 12:00:00 -1 98.8 NaN NaN NaN\n", " 2117-09-13 16:00:00 -1 99.3 NaN NaN NaN\n", " 2117-09-13 22:00:00 -1 99.9 NaN NaN NaN\n", " 2117-09-14 00:00:00 -1 99.4 NaN NaN NaN\n", " 2117-09-14 04:00:00 -1 99.6 NaN NaN NaN\n", " 2117-09-14 08:00:00 -1 99.0 NaN NaN NaN\n", " 2117-09-14 13:00:00 -1 98.0 NaN NaN NaN\n", " 2117-09-14 19:00:00 -1 99.5 NaN NaN NaN\n", " 2117-09-15 00:00:00 -1 99.6 NaN NaN NaN\n", " 2117-09-15 08:00:00 -1 98.3 NaN NaN NaN\n", " 2117-09-15 12:00:00 -1 97.8 NaN NaN NaN\n", " 2117-09-15 16:00:00 -1 98.6 NaN NaN NaN\n", "100003 2150-04-17 20:31:00 -1 95.9 NaN NaN NaN\n", " 2150-04-17 22:00:00 -1 98.2 NaN NaN NaN\n", " 2150-04-18 00:00:00 -1 97.4 NaN NaN NaN\n", " 2150-04-18 04:00:00 -1 98.0 NaN NaN NaN\n", " 2150-04-18 08:00:00 -1 97.4 NaN NaN NaN\n", " 2150-04-18 11:18:00 -1 96.3 NaN NaN NaN\n", "... ... .. ... ...\n", "100011 2177-09-04 19:00:00 -1 101.0 NaN NaN NaN\n", " 2177-09-04 20:00:00 -1 101.2 NaN NaN NaN\n", " 2177-09-04 22:00:00 -1 102.1 NaN NaN NaN\n", " 2177-09-05 00:00:00 -1 101.8 NaN NaN NaN\n", " 2177-09-05 02:00:00 -1 100.8 NaN NaN NaN\n", " 2177-09-05 04:00:00 -1 100.1 NaN NaN NaN\n", " 2177-09-05 06:00:00 -1 100.1 NaN NaN NaN\n", " 2177-09-05 08:00:00 -1 99.5 NaN NaN NaN\n", " 2177-09-05 10:00:00 -1 101.4 NaN NaN NaN\n", " 2177-09-05 12:00:00 -1 101.1 NaN NaN NaN\n", " 2177-09-05 14:00:00 -1 99.8 NaN NaN NaN\n", " 2177-09-05 16:00:00 -1 97.5 NaN NaN NaN\n", " 2177-09-05 20:00:00 -1 102.0 NaN NaN NaN\n", " 2177-09-05 23:00:00 -1 100.8 NaN NaN NaN\n", " 2177-09-06 02:00:00 -1 101.0 NaN NaN NaN\n", " 2177-09-06 04:00:00 -1 101.2 NaN NaN NaN\n", " 2177-09-06 08:00:00 -1 100.1 NaN NaN NaN\n", " 2177-09-06 12:00:00 -1 100.1 NaN NaN NaN\n", " 2177-09-06 16:00:00 -1 102.7 NaN NaN NaN\n", " 2177-09-06 18:00:00 -1 102.0 NaN NaN NaN\n", " 2177-09-06 20:00:00 -1 101.3 NaN NaN NaN\n", " 2177-09-06 23:00:00 -1 101.9 NaN NaN NaN\n", " 2177-09-07 00:00:00 -1 99.9 NaN NaN NaN\n", " 2177-09-07 04:00:00 -1 100.5 NaN NaN NaN\n", " 2177-09-07 09:00:00 0 99.5 NaN NaN NaN\n", " 2177-09-07 13:00:00 0 99.1 NaN NaN NaN\n", " 2177-09-07 17:00:00 0 99.2 NaN NaN NaN\n", " 2177-09-07 19:00:00 -1 99.9 NaN NaN NaN\n", " 2177-09-08 00:00:00 -1 99.8 NaN NaN NaN\n", " 2177-09-08 04:00:00 -1 99.5 NaN NaN NaN\n", "\n", "[200 rows x 4 columns]\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
statusknown
variable_typeqn
unitsdegF
description223761678223762(degC)676(degC)
iddatetimeseg_id
1000012117-09-11 22:00:000100.0NaNNaNNaN
2117-09-12 01:00:00099.9NaNNaNNaN
2117-09-12 04:00:00097.7NaNNaNNaN
1000062108-04-07 20:00:000NaN98.800003NaNNaN
1000072145-04-04 00:00:000NaN98.699997NaNNaN
2145-04-04 04:00:000NaN97.400002NaNNaN
1000092162-05-19 03:00:00098.6NaNNaNNaN
1000102109-12-10 22:47:000100.2NaNNaNNaN
2109-12-11 00:00:00098.8NaNNaNNaN
1000112177-09-07 09:00:00099.5NaNNaNNaN
2177-09-07 13:00:00099.1NaNNaNNaN
2177-09-07 17:00:00099.2NaNNaNNaN
\n", "
" ], "text/plain": [ "label temperature body \\\n", "status known \n", "variable_type qn \n", "units degF \n", "description 223761 678 223762(degC) \n", "id datetime seg_id \n", "100001 2117-09-11 22:00:00 0 100.0 NaN NaN \n", " 2117-09-12 01:00:00 0 99.9 NaN NaN \n", " 2117-09-12 04:00:00 0 97.7 NaN NaN \n", "100006 2108-04-07 20:00:00 0 NaN 98.800003 NaN \n", "100007 2145-04-04 00:00:00 0 NaN 98.699997 NaN \n", " 2145-04-04 04:00:00 0 NaN 97.400002 NaN \n", "100009 2162-05-19 03:00:00 0 98.6 NaN NaN \n", "100010 2109-12-10 22:47:00 0 100.2 NaN NaN \n", " 2109-12-11 00:00:00 0 98.8 NaN NaN \n", "100011 2177-09-07 09:00:00 0 99.5 NaN NaN \n", " 2177-09-07 13:00:00 0 99.1 NaN NaN \n", " 2177-09-07 17:00:00 0 99.2 NaN NaN \n", "\n", "label \n", "status \n", "variable_type \n", "units \n", "description 676(degC) \n", "id datetime seg_id \n", "100001 2117-09-11 22:00:00 0 NaN \n", " 2117-09-12 01:00:00 0 NaN \n", " 2117-09-12 04:00:00 0 NaN \n", "100006 2108-04-07 20:00:00 0 NaN \n", "100007 2145-04-04 00:00:00 0 NaN \n", " 2145-04-04 04:00:00 0 NaN \n", "100009 2162-05-19 03:00:00 0 NaN \n", "100010 2109-12-10 22:47:00 0 NaN \n", " 2109-12-11 00:00:00 0 NaN \n", "100011 2177-09-07 09:00:00 0 NaN \n", " 2177-09-07 13:00:00 0 NaN \n", " 2177-09-07 17:00:00 0 NaN " ] }, "execution_count": 262, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp_Nhr_before = n_hrs_before.transform(df_temp_combined.head(200))\n", "df_temp_Nhr_before[df_temp_Nhr_before.index.get_level_values('seg_id') > -1]" ] }, { "cell_type": "code", "execution_count": 266, "metadata": { "collapsed": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labeltemperature body
statusknown
variable_typeqn
unitsdegF
description223761678223762(degC)676(degC)
iddatetimeseg_id
1000012117-09-11 13:48:00098.0NaNNaNNaN
2117-09-11 16:00:00097.9NaNNaNNaN
2117-09-11 19:00:00099.5NaNNaNNaN
2117-09-11 22:00:001100.0NaNNaNNaN
2117-09-12 01:00:00199.9NaNNaNNaN
2117-09-12 04:00:00197.7NaNNaNNaN
2117-09-12 08:00:00297.8NaNNaNNaN
2117-09-12 12:00:00297.5NaNNaNNaN
2117-09-12 19:00:00399.8NaNNaNNaN
2117-09-13 00:00:00399.7NaNNaNNaN
2117-09-13 04:00:00499.3NaNNaNNaN
2117-09-13 08:00:00499.4NaNNaNNaN
2117-09-13 12:00:00498.8NaNNaNNaN
2117-09-13 16:00:00599.3NaNNaNNaN
2117-09-13 22:00:00599.9NaNNaNNaN
2117-09-14 00:00:00699.4NaNNaNNaN
2117-09-14 04:00:00699.6NaNNaNNaN
2117-09-14 08:00:00699.0NaNNaNNaN
2117-09-14 13:00:00798.0NaNNaNNaN
2117-09-14 19:00:00799.5NaNNaNNaN
2117-09-15 00:00:00899.6NaNNaNNaN
2117-09-15 08:00:00998.3NaNNaNNaN
2117-09-15 12:00:00997.8NaNNaNNaN
2117-09-15 16:00:001098.6NaNNaNNaN
1000032150-04-17 20:31:00095.9NaNNaNNaN
2150-04-17 22:00:00098.2NaNNaNNaN
2150-04-18 00:00:00097.4NaNNaNNaN
2150-04-18 04:00:00198.0NaNNaNNaN
2150-04-18 08:00:00197.4NaNNaNNaN
2150-04-18 11:18:00196.3NaNNaNNaN
.....................
1001142157-09-19 18:15:003NaN98.400002NaNNaN
2157-09-19 18:30:003NaN98.300003NaNNaN
2157-09-19 20:00:003NaN98.400002NaNNaN
2157-09-19 23:00:003NaN98.400002NaNNaN
2157-09-20 00:00:003NaN98.599998NaNNaN
2157-09-20 01:00:003NaN98.000000NaNNaN
2157-09-20 01:15:004NaN98.199997NaNNaN
2157-09-20 02:00:004NaN97.099998NaNNaN
2157-09-20 06:00:004NaN98.000000NaNNaN
2157-09-20 08:00:004NaN96.699997NaNNaN
2157-09-20 09:15:004NaN96.900002NaNNaN
2157-09-20 09:30:004NaN97.000000NaNNaN
2157-09-20 10:00:004NaN96.900002NaNNaN
2157-09-20 12:00:005NaN97.199997NaNNaN
2157-09-20 15:00:005NaN98.400002NaNNaN
2157-09-20 18:00:005NaN97.800003NaNNaN
1001162173-07-27 05:13:00098.9NaNNaNNaN
2173-07-27 08:00:000100.0NaNNaNNaN
2173-07-27 09:30:00099.7NaNNaNNaN
2173-07-27 11:00:00098.7NaNNaNNaN
2173-07-27 14:25:000100.4NaNNaNNaN
2173-07-27 16:30:001100.3NaNNaNNaN
2173-07-27 18:00:00199.9NaNNaNNaN
2173-07-27 20:00:001100.3NaNNaNNaN
2173-07-28 00:00:00199.7NaNNaNNaN
2173-07-28 04:00:00298.5NaNNaNNaN
2173-07-28 08:00:00299.4NaNNaNNaN
2173-07-28 12:00:00398.5NaNNaNNaN
2173-07-28 16:00:00399.4NaNNaNNaN
1001172166-04-30 21:39:00099.7NaNNaNNaN
\n", "

2000 rows × 4 columns

\n", "
" ], "text/plain": [ "label temperature body \\\n", "status known \n", "variable_type qn \n", "units degF \n", "description 223761 678 223762(degC) \n", "id datetime seg_id \n", "100001 2117-09-11 13:48:00 0 98.0 NaN NaN \n", " 2117-09-11 16:00:00 0 97.9 NaN NaN \n", " 2117-09-11 19:00:00 0 99.5 NaN NaN \n", " 2117-09-11 22:00:00 1 100.0 NaN NaN \n", " 2117-09-12 01:00:00 1 99.9 NaN NaN \n", " 2117-09-12 04:00:00 1 97.7 NaN NaN \n", " 2117-09-12 08:00:00 2 97.8 NaN NaN \n", " 2117-09-12 12:00:00 2 97.5 NaN NaN \n", " 2117-09-12 19:00:00 3 99.8 NaN NaN \n", " 2117-09-13 00:00:00 3 99.7 NaN NaN \n", " 2117-09-13 04:00:00 4 99.3 NaN NaN \n", " 2117-09-13 08:00:00 4 99.4 NaN NaN \n", " 2117-09-13 12:00:00 4 98.8 NaN NaN \n", " 2117-09-13 16:00:00 5 99.3 NaN NaN \n", " 2117-09-13 22:00:00 5 99.9 NaN NaN \n", " 2117-09-14 00:00:00 6 99.4 NaN NaN \n", " 2117-09-14 04:00:00 6 99.6 NaN NaN \n", " 2117-09-14 08:00:00 6 99.0 NaN NaN \n", " 2117-09-14 13:00:00 7 98.0 NaN NaN \n", " 2117-09-14 19:00:00 7 99.5 NaN NaN \n", " 2117-09-15 00:00:00 8 99.6 NaN NaN \n", " 2117-09-15 08:00:00 9 98.3 NaN NaN \n", " 2117-09-15 12:00:00 9 97.8 NaN NaN \n", " 2117-09-15 16:00:00 10 98.6 NaN NaN \n", "100003 2150-04-17 20:31:00 0 95.9 NaN NaN \n", " 2150-04-17 22:00:00 0 98.2 NaN NaN \n", " 2150-04-18 00:00:00 0 97.4 NaN NaN \n", " 2150-04-18 04:00:00 1 98.0 NaN NaN \n", " 2150-04-18 08:00:00 1 97.4 NaN NaN \n", " 2150-04-18 11:18:00 1 96.3 NaN NaN \n", "... ... ... ... \n", "100114 2157-09-19 18:15:00 3 NaN 98.400002 NaN \n", " 2157-09-19 18:30:00 3 NaN 98.300003 NaN \n", " 2157-09-19 20:00:00 3 NaN 98.400002 NaN \n", " 2157-09-19 23:00:00 3 NaN 98.400002 NaN \n", " 2157-09-20 00:00:00 3 NaN 98.599998 NaN \n", " 2157-09-20 01:00:00 3 NaN 98.000000 NaN \n", " 2157-09-20 01:15:00 4 NaN 98.199997 NaN \n", " 2157-09-20 02:00:00 4 NaN 97.099998 NaN \n", " 2157-09-20 06:00:00 4 NaN 98.000000 NaN \n", " 2157-09-20 08:00:00 4 NaN 96.699997 NaN \n", " 2157-09-20 09:15:00 4 NaN 96.900002 NaN \n", " 2157-09-20 09:30:00 4 NaN 97.000000 NaN \n", " 2157-09-20 10:00:00 4 NaN 96.900002 NaN \n", " 2157-09-20 12:00:00 5 NaN 97.199997 NaN \n", " 2157-09-20 15:00:00 5 NaN 98.400002 NaN \n", " 2157-09-20 18:00:00 5 NaN 97.800003 NaN \n", "100116 2173-07-27 05:13:00 0 98.9 NaN NaN \n", " 2173-07-27 08:00:00 0 100.0 NaN NaN \n", " 2173-07-27 09:30:00 0 99.7 NaN NaN \n", " 2173-07-27 11:00:00 0 98.7 NaN NaN \n", " 2173-07-27 14:25:00 0 100.4 NaN NaN \n", " 2173-07-27 16:30:00 1 100.3 NaN NaN \n", " 2173-07-27 18:00:00 1 99.9 NaN NaN \n", " 2173-07-27 20:00:00 1 100.3 NaN NaN \n", " 2173-07-28 00:00:00 1 99.7 NaN NaN \n", " 2173-07-28 04:00:00 2 98.5 NaN NaN \n", " 2173-07-28 08:00:00 2 99.4 NaN NaN \n", " 2173-07-28 12:00:00 3 98.5 NaN NaN \n", " 2173-07-28 16:00:00 3 99.4 NaN NaN \n", "100117 2166-04-30 21:39:00 0 99.7 NaN NaN \n", "\n", "label \n", "status \n", "variable_type \n", "units \n", "description 676(degC) \n", "id datetime seg_id \n", "100001 2117-09-11 13:48:00 0 NaN \n", " 2117-09-11 16:00:00 0 NaN \n", " 2117-09-11 19:00:00 0 NaN \n", " 2117-09-11 22:00:00 1 NaN \n", " 2117-09-12 01:00:00 1 NaN \n", " 2117-09-12 04:00:00 1 NaN \n", " 2117-09-12 08:00:00 2 NaN \n", " 2117-09-12 12:00:00 2 NaN \n", " 2117-09-12 19:00:00 3 NaN \n", " 2117-09-13 00:00:00 3 NaN \n", " 2117-09-13 04:00:00 4 NaN \n", " 2117-09-13 08:00:00 4 NaN \n", " 2117-09-13 12:00:00 4 NaN \n", " 2117-09-13 16:00:00 5 NaN \n", " 2117-09-13 22:00:00 5 NaN \n", " 2117-09-14 00:00:00 6 NaN \n", " 2117-09-14 04:00:00 6 NaN \n", " 2117-09-14 08:00:00 6 NaN \n", " 2117-09-14 13:00:00 7 NaN \n", " 2117-09-14 19:00:00 7 NaN \n", " 2117-09-15 00:00:00 8 NaN \n", " 2117-09-15 08:00:00 9 NaN \n", " 2117-09-15 12:00:00 9 NaN \n", " 2117-09-15 16:00:00 10 NaN \n", "100003 2150-04-17 20:31:00 0 NaN \n", " 2150-04-17 22:00:00 0 NaN \n", " 2150-04-18 00:00:00 0 NaN \n", " 2150-04-18 04:00:00 1 NaN \n", " 2150-04-18 08:00:00 1 NaN \n", " 2150-04-18 11:18:00 1 NaN \n", "... ... \n", "100114 2157-09-19 18:15:00 3 NaN \n", " 2157-09-19 18:30:00 3 NaN \n", " 2157-09-19 20:00:00 3 NaN \n", " 2157-09-19 23:00:00 3 NaN \n", " 2157-09-20 00:00:00 3 NaN \n", " 2157-09-20 01:00:00 3 NaN \n", " 2157-09-20 01:15:00 4 NaN \n", " 2157-09-20 02:00:00 4 NaN \n", " 2157-09-20 06:00:00 4 NaN \n", " 2157-09-20 08:00:00 4 NaN \n", " 2157-09-20 09:15:00 4 NaN \n", " 2157-09-20 09:30:00 4 NaN \n", " 2157-09-20 10:00:00 4 NaN \n", " 2157-09-20 12:00:00 5 NaN \n", " 2157-09-20 15:00:00 5 NaN \n", " 2157-09-20 18:00:00 5 NaN \n", "100116 2173-07-27 05:13:00 0 NaN \n", " 2173-07-27 08:00:00 0 NaN \n", " 2173-07-27 09:30:00 0 NaN \n", " 2173-07-27 11:00:00 0 NaN \n", " 2173-07-27 14:25:00 0 NaN \n", " 2173-07-27 16:30:00 1 NaN \n", " 2173-07-27 18:00:00 1 NaN \n", " 2173-07-27 20:00:00 1 NaN \n", " 2173-07-28 00:00:00 1 NaN \n", " 2173-07-28 04:00:00 2 NaN \n", " 2173-07-28 08:00:00 2 NaN \n", " 2173-07-28 12:00:00 3 NaN \n", " 2173-07-28 16:00:00 3 NaN \n", "100117 2166-04-30 21:39:00 0 NaN \n", "\n", "[2000 rows x 4 columns]" ] }, "execution_count": 266, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp_periodic = periodic.transform(df_temp_combined.head(2000))\n", "df_temp_periodic[df_temp_periodic.index.get_level_values('seg_id') > -1]" ] }, { "cell_type": "code", "execution_count": 252, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import utils\n", "import mimic\n", "import transformers\n", "from sklearn.pipeline import Pipeline\n", "import icu_data_defs\n", "import units\n", "import segmenting\n", "from sklearn_pandas import DataFrameMapper\n", "import constants\n", "import features" ] }, { "cell_type": "code", "execution_count": 251, "metadata": { "collapsed": true }, "outputs": [], "source": [ "hdf5_fname = 'data/mimic_data'\n", "data_dict = icu_data_defs.data_dictionary('config/data_definitions.xlsx')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "df_cleaned" ] }, { "cell_type": "code", "execution_count": 249, "metadata": { "collapsed": true }, "outputs": [], "source": [ "end_dt = df_temp_combined.iloc[:,0].groupby(level='id').apply(lambda x:x.sample(1))\n", "end_dt = end_dt.reset_index(level=0,drop=True).reset_index(level=1,drop=False).iloc[:,0]\n", "all_before = segmenting.all_before(end_dt,df_context)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Feature creation" ] }, { "cell_type": "code", "execution_count": 247, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import utils\n", "import transformers\n", "from sklearn.pipeline import Pipeline\n", "import units\n", "from sklearn_pandas import DataFrameMapper\n", "import constants\n", "import features" ] }, { "cell_type": "code", "execution_count": 248, "metadata": { "collapsed": false }, "outputs": [ { "ename": "ImportError", "evalue": "cannot import name CUSTOM_FILTER", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mImportError\u001b[0m Traceback (most recent call last)", "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mreload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mreload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mutils\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[0mreload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0munits\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[0mreload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconstants\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\icu_ml_project\\v5\\features.pyc\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbase\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mTransformerMixin\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mBaseEstimator\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpipeline\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mFeatureUnion\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mconstants\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcolumn_names\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mSEG_ID\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mNO_SEGMENT\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mALL\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mCUSTOM_FILTER\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn_pandas\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDataFrameMapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;31mImportError\u001b[0m: cannot import name CUSTOM_FILTER" ] } ], "source": [ "reload(features)\n", "reload(utils)\n", "reload(units)\n", "reload(constants)\n", "\n", "df = df_temp_all_before\n", "ureg = units.MedicalUreg()\n", "\n", "\n", "\n", "def summable_filter(df):\n", " filter_func= lambda x: (ureg.is_volume(str(x[-2])) or ureg.is_mass(str(x[-2]))) and (x[0] != 'weight body')\n", " return df.loc[:,df.columns.map(filter_func)]\n", "\n", "feature_tuples = [\n", " ('MEAN',features.segment_mean(),ALL),\n", " ('STD',features.segment_std(),ALL),\n", " ('COUNT',features.segment_count(),ALL),\n", " ('LAST',features.segment_last(),ALL),\n", " ('SUM',features.segment_sum(),{constants.CUSTOM_FILTER:summable_filter})\n", "]\n", "\n", "mapped_ft = features.make_mapper(feature_tuples,df)" ] }, { "cell_type": "code", "execution_count": 232, "metadata": { "collapsed": false }, "outputs": [], "source": [ "feature_df = mapped_ft.transform(df)" ] }, { "cell_type": "code", "execution_count": 233, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
temperature body_degF_223761_MEANtemperature body_degF_678_MEANtemperature body_degC_223762_MEANtemperature body_degC_676_MEANtemperature body_degF_223761_STDtemperature body_degF_678_STDtemperature body_degC_223762_STDtemperature body_degC_676_STDtemperature body_degF_223761_COUNTtemperature body_degF_678_COUNTtemperature body_degC_223762_COUNTtemperature body_degC_676_COUNTtemperature body_degF_223761_LASTtemperature body_degF_678_LASTtemperature body_degC_223762_LASTtemperature body_degC_676_LAST
098.537500NaNNaNNaN1.064945NaNNaNNaN8.00.00.00.098.0NaNNaNNaN
197.166667NaNNaNNaN1.167619NaNNaNNaN3.00.00.00.095.9NaNNaNNaN
2NaN97.533334NaNNaNNaN0.607729NaNNaN0.021.00.00.0NaN97.000000NaNNaN
3NaN98.591667NaNNaNNaN1.466263NaNNaN0.012.00.00.0NaN95.699997NaNNaN
498.100000NaNNaNNaN0.458258NaNNaNNaN3.00.00.00.098.6NaNNaNNaN
598.550000NaNNaNNaN1.036822NaNNaNNaN6.00.00.00.0100.2NaNNaNNaN
698.380000NaN36.511765NaN0.641872NaN0.790767NaN5.00.034.00.0NaNNaN34.5NaN
7NaN100.649998NaNNaNNaN0.777816NaNNaN0.02.00.00.0NaN101.199997NaNNaN
897.975000NaNNaNNaN0.875032NaNNaNNaN12.00.00.00.097.4NaNNaNNaN
996.050000NaNNaNNaN0.777817NaNNaNNaN2.00.00.00.096.6NaNNaNNaN
10NaN98.536364NaNNaNNaN0.492489NaNNaN0.011.00.00.0NaN98.500000NaNNaN
11NaNNaN36.378571NaNNaNNaN0.260705NaN0.00.014.00.0NaNNaN36.6NaN
1299.466667NaNNaNNaN1.495650NaNNaNNaN12.00.00.00.098.8NaNNaNNaN
13NaN99.099998NaN36.766037NaNNaNNaN0.3746460.01.00.0106.0NaNNaNNaN36.599998
14NaN95.300003NaNNaNNaNNaNNaNNaN0.01.00.00.0NaN95.300003NaNNaN
1598.500000NaNNaNNaN0.141421NaNNaNNaN2.00.00.00.098.6NaNNaNNaN
16100.772222NaN37.719512NaN1.075287NaN0.281265NaN18.00.041.00.0NaNNaN37.5NaN
17NaNNaNNaN37.193750NaNNaNNaN0.4015600.00.00.032.0NaNNaNNaN36.799999
1899.345455NaNNaNNaN0.864975NaNNaNNaN33.00.00.00.098.6NaNNaNNaN
19NaN98.800003NaNNaNNaNNaNNaNNaN0.01.00.00.0NaN98.800003NaNNaN
2098.312500NaNNaNNaN1.022986NaNNaNNaN16.00.00.00.097.6NaNNaNNaN
2197.760000NaNNaNNaN0.585662NaNNaNNaN5.00.00.00.097.8NaNNaNNaN
22NaN98.445455NaNNaNNaN1.849254NaNNaN0.022.00.00.0NaN97.800003NaNNaN
23NaNNaN34.990909NaNNaNNaN4.190574NaN0.00.011.00.0NaNNaN28.4NaN
24NaNNaNNaN36.799999NaNNaNNaNNaN0.00.00.01.0NaNNaNNaN36.799999
25NaN97.320000NaNNaNNaN0.277491NaNNaN0.05.00.00.0NaN97.199997NaNNaN
26NaN96.700000NaNNaNNaN0.754119NaNNaN0.024.00.00.0NaN97.000000NaNNaN
27NaN96.920000NaNNaNNaN1.787487NaNNaN0.010.00.00.0NaN94.000000NaNNaN
28NaNNaNNaN36.847368NaNNaNNaN0.3006810.00.00.019.0NaNNaNNaN36.200001
2996.525000NaNNaNNaN0.450000NaNNaNNaN4.00.00.00.096.9NaNNaNNaN
...................................................
44216NaN98.302703NaN37.021739NaN1.047082NaN0.9973110.037.00.046.0NaN96.000000NaNNaN
4421798.475000NaNNaNNaN1.081280NaNNaNNaN4.00.00.00.097.1NaNNaNNaN
4421896.275000NaNNaNNaN0.420034NaNNaNNaN8.00.00.00.096.3NaNNaNNaN
44219NaN97.800003NaNNaNNaNNaNNaNNaN0.01.00.00.0NaN97.800003NaNNaN
44220NaN95.740001NaNNaNNaN1.021275NaNNaN0.05.00.00.0NaN94.099998NaNNaN
44221NaN97.920000NaNNaNNaN0.672309NaNNaN0.05.00.00.0NaN97.099998NaNNaN
4422299.240000NaNNaNNaN0.512835NaNNaNNaN5.00.00.00.099.8NaNNaNNaN
4422398.450000NaNNaNNaN0.070711NaNNaNNaN2.00.00.00.098.4NaNNaNNaN
44224NaN98.130000NaNNaNNaN0.531351NaNNaN0.010.00.00.0NaN97.800003NaNNaN
44225NaN97.900002NaNNaNNaN0.199997NaNNaN0.04.00.00.0NaN97.800003NaNNaN
4422699.753333NaNNaNNaN0.765195NaNNaNNaN15.00.00.00.098.7NaNNaNNaN
44227NaN98.071875NaN37.449181NaN0.944674NaN0.9209820.032.00.061.0NaNNaNNaN34.500000
44228NaN100.526316NaNNaNNaN1.175605NaNNaN0.019.00.00.0NaN98.000000NaNNaN
4422997.580000NaNNaNNaN0.740570NaNNaNNaN10.00.00.00.096.9NaNNaNNaN
44230NaNNaNNaN37.443518NaNNaNNaN0.3324430.00.00.0108.0NaNNaNNaN37.299999
44231NaNNaNNaN36.071429NaNNaNNaN0.2399640.00.00.014.0NaNNaNNaN35.599998
4423298.484375NaNNaNNaN1.070456NaNNaNNaN32.00.00.00.0100.0NaNNaNNaN
44233NaN100.278462NaNNaNNaN1.300803NaNNaN0.0130.00.00.0NaN97.900002NaNNaN
44234NaNNaN36.173333NaNNaNNaN0.688131NaN0.00.015.00.0NaNNaN35.0NaN
44235NaN97.074999NaNNaNNaN1.056330NaNNaN0.04.00.00.0NaN95.599998NaNNaN
4423697.977778NaNNaNNaN1.032930NaNNaNNaN9.00.00.00.098.4NaNNaNNaN
44237NaN97.485713NaNNaNNaN0.977118NaNNaN0.07.00.00.0NaN95.900002NaNNaN
44238NaN97.264285NaNNaNNaN0.759157NaNNaN0.014.00.00.0NaN96.599998NaNNaN
44239NaN95.400002NaNNaNNaN0.565688NaNNaN0.02.00.00.0NaN95.000000NaNNaN
4424095.800000NaN34.800000NaNNaNNaNNaNNaN1.00.01.00.0NaNNaN34.8NaN
44241NaN97.670000NaN37.420725NaN0.668415NaN0.3780230.010.00.0193.0NaN98.699997NaNNaN
44242NaN98.980953NaNNaNNaN0.933605NaNNaN0.021.00.00.0NaN99.400002NaNNaN
44243NaNNaNNaN35.750000NaNNaNNaN0.9192410.00.00.02.0NaNNaNNaN35.099998
44244NaNNaNNaN36.176191NaNNaNNaN1.1597000.00.00.021.0NaNNaNNaN34.700001
4424598.655000NaNNaNNaN1.147297NaNNaNNaN20.00.00.00.097.8NaNNaNNaN
\n", "

44246 rows × 16 columns

\n", "
" ], "text/plain": [ " temperature body_degF_223761_MEAN temperature body_degF_678_MEAN \\\n", "0 98.537500 NaN \n", "1 97.166667 NaN \n", "2 NaN 97.533334 \n", "3 NaN 98.591667 \n", "4 98.100000 NaN \n", "5 98.550000 NaN \n", "6 98.380000 NaN \n", "7 NaN 100.649998 \n", "8 97.975000 NaN \n", "9 96.050000 NaN \n", "10 NaN 98.536364 \n", "11 NaN NaN \n", "12 99.466667 NaN \n", "13 NaN 99.099998 \n", "14 NaN 95.300003 \n", "15 98.500000 NaN \n", "16 100.772222 NaN \n", "17 NaN NaN \n", "18 99.345455 NaN \n", "19 NaN 98.800003 \n", "20 98.312500 NaN \n", "21 97.760000 NaN \n", "22 NaN 98.445455 \n", "23 NaN NaN \n", "24 NaN NaN \n", "25 NaN 97.320000 \n", "26 NaN 96.700000 \n", "27 NaN 96.920000 \n", "28 NaN NaN \n", "29 96.525000 NaN \n", "... ... ... \n", "44216 NaN 98.302703 \n", "44217 98.475000 NaN \n", "44218 96.275000 NaN \n", "44219 NaN 97.800003 \n", "44220 NaN 95.740001 \n", "44221 NaN 97.920000 \n", "44222 99.240000 NaN \n", "44223 98.450000 NaN \n", "44224 NaN 98.130000 \n", "44225 NaN 97.900002 \n", "44226 99.753333 NaN \n", "44227 NaN 98.071875 \n", "44228 NaN 100.526316 \n", "44229 97.580000 NaN \n", "44230 NaN NaN \n", "44231 NaN NaN \n", "44232 98.484375 NaN \n", "44233 NaN 100.278462 \n", "44234 NaN NaN \n", "44235 NaN 97.074999 \n", "44236 97.977778 NaN \n", "44237 NaN 97.485713 \n", "44238 NaN 97.264285 \n", "44239 NaN 95.400002 \n", "44240 95.800000 NaN \n", "44241 NaN 97.670000 \n", "44242 NaN 98.980953 \n", "44243 NaN NaN \n", "44244 NaN NaN \n", "44245 98.655000 NaN \n", "\n", " temperature body_degC_223762_MEAN temperature body_degC_676_MEAN \\\n", "0 NaN NaN \n", "1 NaN NaN \n", "2 NaN NaN \n", "3 NaN NaN \n", "4 NaN NaN \n", "5 NaN NaN \n", "6 36.511765 NaN \n", "7 NaN NaN \n", "8 NaN NaN \n", "9 NaN NaN \n", "10 NaN NaN \n", "11 36.378571 NaN \n", "12 NaN NaN \n", "13 NaN 36.766037 \n", "14 NaN NaN \n", "15 NaN NaN \n", "16 37.719512 NaN \n", "17 NaN 37.193750 \n", "18 NaN NaN \n", "19 NaN NaN \n", "20 NaN NaN \n", "21 NaN NaN \n", "22 NaN NaN \n", "23 34.990909 NaN \n", "24 NaN 36.799999 \n", "25 NaN NaN \n", "26 NaN NaN \n", "27 NaN NaN \n", "28 NaN 36.847368 \n", "29 NaN NaN \n", "... ... ... \n", "44216 NaN 37.021739 \n", "44217 NaN NaN \n", "44218 NaN NaN \n", "44219 NaN NaN \n", "44220 NaN NaN \n", "44221 NaN NaN \n", "44222 NaN NaN \n", "44223 NaN NaN \n", "44224 NaN NaN \n", "44225 NaN NaN \n", "44226 NaN NaN \n", "44227 NaN 37.449181 \n", "44228 NaN NaN \n", "44229 NaN NaN \n", "44230 NaN 37.443518 \n", "44231 NaN 36.071429 \n", "44232 NaN NaN \n", "44233 NaN NaN \n", "44234 36.173333 NaN \n", "44235 NaN NaN \n", "44236 NaN NaN \n", "44237 NaN NaN \n", "44238 NaN NaN \n", "44239 NaN NaN \n", "44240 34.800000 NaN \n", "44241 NaN 37.420725 \n", "44242 NaN NaN \n", "44243 NaN 35.750000 \n", "44244 NaN 36.176191 \n", "44245 NaN NaN \n", "\n", " temperature body_degF_223761_STD temperature body_degF_678_STD \\\n", "0 1.064945 NaN \n", "1 1.167619 NaN \n", "2 NaN 0.607729 \n", "3 NaN 1.466263 \n", "4 0.458258 NaN \n", "5 1.036822 NaN \n", "6 0.641872 NaN \n", "7 NaN 0.777816 \n", "8 0.875032 NaN \n", "9 0.777817 NaN \n", "10 NaN 0.492489 \n", "11 NaN NaN \n", "12 1.495650 NaN \n", "13 NaN NaN \n", "14 NaN NaN \n", "15 0.141421 NaN \n", "16 1.075287 NaN \n", "17 NaN NaN \n", "18 0.864975 NaN \n", "19 NaN NaN \n", "20 1.022986 NaN \n", "21 0.585662 NaN \n", "22 NaN 1.849254 \n", "23 NaN NaN \n", "24 NaN NaN \n", "25 NaN 0.277491 \n", "26 NaN 0.754119 \n", "27 NaN 1.787487 \n", "28 NaN NaN \n", "29 0.450000 NaN \n", "... ... ... \n", "44216 NaN 1.047082 \n", "44217 1.081280 NaN \n", "44218 0.420034 NaN \n", "44219 NaN NaN \n", "44220 NaN 1.021275 \n", "44221 NaN 0.672309 \n", "44222 0.512835 NaN \n", "44223 0.070711 NaN \n", "44224 NaN 0.531351 \n", "44225 NaN 0.199997 \n", "44226 0.765195 NaN \n", "44227 NaN 0.944674 \n", "44228 NaN 1.175605 \n", "44229 0.740570 NaN \n", "44230 NaN NaN \n", "44231 NaN NaN \n", "44232 1.070456 NaN \n", "44233 NaN 1.300803 \n", "44234 NaN NaN \n", "44235 NaN 1.056330 \n", "44236 1.032930 NaN \n", "44237 NaN 0.977118 \n", "44238 NaN 0.759157 \n", "44239 NaN 0.565688 \n", "44240 NaN NaN \n", "44241 NaN 0.668415 \n", "44242 NaN 0.933605 \n", "44243 NaN NaN \n", "44244 NaN NaN \n", "44245 1.147297 NaN \n", "\n", " temperature body_degC_223762_STD temperature body_degC_676_STD \\\n", "0 NaN NaN \n", "1 NaN NaN \n", "2 NaN NaN \n", "3 NaN NaN \n", "4 NaN NaN \n", "5 NaN NaN \n", "6 0.790767 NaN \n", "7 NaN NaN \n", "8 NaN NaN \n", "9 NaN NaN \n", "10 NaN NaN \n", "11 0.260705 NaN \n", "12 NaN NaN \n", "13 NaN 0.374646 \n", "14 NaN NaN \n", "15 NaN NaN \n", "16 0.281265 NaN \n", "17 NaN 0.401560 \n", "18 NaN NaN \n", "19 NaN NaN \n", "20 NaN NaN \n", "21 NaN NaN \n", "22 NaN NaN \n", "23 4.190574 NaN \n", "24 NaN NaN \n", "25 NaN NaN \n", "26 NaN NaN \n", "27 NaN NaN \n", "28 NaN 0.300681 \n", "29 NaN NaN \n", "... ... ... \n", "44216 NaN 0.997311 \n", "44217 NaN NaN \n", "44218 NaN NaN \n", "44219 NaN NaN \n", "44220 NaN NaN \n", "44221 NaN NaN \n", "44222 NaN NaN \n", "44223 NaN NaN \n", "44224 NaN NaN \n", "44225 NaN NaN \n", "44226 NaN NaN \n", "44227 NaN 0.920982 \n", "44228 NaN NaN \n", "44229 NaN NaN \n", "44230 NaN 0.332443 \n", "44231 NaN 0.239964 \n", "44232 NaN NaN \n", "44233 NaN NaN \n", "44234 0.688131 NaN \n", "44235 NaN NaN \n", "44236 NaN NaN \n", "44237 NaN NaN \n", "44238 NaN NaN \n", "44239 NaN NaN \n", "44240 NaN NaN \n", "44241 NaN 0.378023 \n", "44242 NaN NaN \n", "44243 NaN 0.919241 \n", "44244 NaN 1.159700 \n", "44245 NaN NaN \n", "\n", " temperature body_degF_223761_COUNT temperature body_degF_678_COUNT \\\n", "0 8.0 0.0 \n", "1 3.0 0.0 \n", "2 0.0 21.0 \n", "3 0.0 12.0 \n", "4 3.0 0.0 \n", "5 6.0 0.0 \n", "6 5.0 0.0 \n", "7 0.0 2.0 \n", "8 12.0 0.0 \n", "9 2.0 0.0 \n", "10 0.0 11.0 \n", "11 0.0 0.0 \n", "12 12.0 0.0 \n", "13 0.0 1.0 \n", "14 0.0 1.0 \n", "15 2.0 0.0 \n", "16 18.0 0.0 \n", "17 0.0 0.0 \n", "18 33.0 0.0 \n", "19 0.0 1.0 \n", "20 16.0 0.0 \n", "21 5.0 0.0 \n", "22 0.0 22.0 \n", "23 0.0 0.0 \n", "24 0.0 0.0 \n", "25 0.0 5.0 \n", "26 0.0 24.0 \n", "27 0.0 10.0 \n", "28 0.0 0.0 \n", "29 4.0 0.0 \n", "... ... ... \n", "44216 0.0 37.0 \n", "44217 4.0 0.0 \n", "44218 8.0 0.0 \n", "44219 0.0 1.0 \n", "44220 0.0 5.0 \n", "44221 0.0 5.0 \n", "44222 5.0 0.0 \n", "44223 2.0 0.0 \n", "44224 0.0 10.0 \n", "44225 0.0 4.0 \n", "44226 15.0 0.0 \n", "44227 0.0 32.0 \n", "44228 0.0 19.0 \n", "44229 10.0 0.0 \n", "44230 0.0 0.0 \n", "44231 0.0 0.0 \n", "44232 32.0 0.0 \n", "44233 0.0 130.0 \n", "44234 0.0 0.0 \n", "44235 0.0 4.0 \n", "44236 9.0 0.0 \n", "44237 0.0 7.0 \n", "44238 0.0 14.0 \n", "44239 0.0 2.0 \n", "44240 1.0 0.0 \n", "44241 0.0 10.0 \n", "44242 0.0 21.0 \n", "44243 0.0 0.0 \n", "44244 0.0 0.0 \n", "44245 20.0 0.0 \n", "\n", " temperature body_degC_223762_COUNT temperature body_degC_676_COUNT \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 0.0 0.0 \n", "4 0.0 0.0 \n", "5 0.0 0.0 \n", "6 34.0 0.0 \n", "7 0.0 0.0 \n", "8 0.0 0.0 \n", "9 0.0 0.0 \n", "10 0.0 0.0 \n", "11 14.0 0.0 \n", "12 0.0 0.0 \n", "13 0.0 106.0 \n", "14 0.0 0.0 \n", "15 0.0 0.0 \n", "16 41.0 0.0 \n", "17 0.0 32.0 \n", "18 0.0 0.0 \n", "19 0.0 0.0 \n", "20 0.0 0.0 \n", "21 0.0 0.0 \n", "22 0.0 0.0 \n", "23 11.0 0.0 \n", "24 0.0 1.0 \n", "25 0.0 0.0 \n", "26 0.0 0.0 \n", "27 0.0 0.0 \n", "28 0.0 19.0 \n", "29 0.0 0.0 \n", "... ... ... \n", "44216 0.0 46.0 \n", "44217 0.0 0.0 \n", "44218 0.0 0.0 \n", "44219 0.0 0.0 \n", "44220 0.0 0.0 \n", "44221 0.0 0.0 \n", "44222 0.0 0.0 \n", "44223 0.0 0.0 \n", "44224 0.0 0.0 \n", "44225 0.0 0.0 \n", "44226 0.0 0.0 \n", "44227 0.0 61.0 \n", "44228 0.0 0.0 \n", "44229 0.0 0.0 \n", "44230 0.0 108.0 \n", "44231 0.0 14.0 \n", "44232 0.0 0.0 \n", "44233 0.0 0.0 \n", "44234 15.0 0.0 \n", "44235 0.0 0.0 \n", "44236 0.0 0.0 \n", "44237 0.0 0.0 \n", "44238 0.0 0.0 \n", "44239 0.0 0.0 \n", "44240 1.0 0.0 \n", "44241 0.0 193.0 \n", "44242 0.0 0.0 \n", "44243 0.0 2.0 \n", "44244 0.0 21.0 \n", "44245 0.0 0.0 \n", "\n", " temperature body_degF_223761_LAST temperature body_degF_678_LAST \\\n", "0 98.0 NaN \n", "1 95.9 NaN \n", "2 NaN 97.000000 \n", "3 NaN 95.699997 \n", "4 98.6 NaN \n", "5 100.2 NaN \n", "6 NaN NaN \n", "7 NaN 101.199997 \n", "8 97.4 NaN \n", "9 96.6 NaN \n", "10 NaN 98.500000 \n", "11 NaN NaN \n", "12 98.8 NaN \n", "13 NaN NaN \n", "14 NaN 95.300003 \n", "15 98.6 NaN \n", "16 NaN NaN \n", "17 NaN NaN \n", "18 98.6 NaN \n", "19 NaN 98.800003 \n", "20 97.6 NaN \n", "21 97.8 NaN \n", "22 NaN 97.800003 \n", "23 NaN NaN \n", "24 NaN NaN \n", "25 NaN 97.199997 \n", "26 NaN 97.000000 \n", "27 NaN 94.000000 \n", "28 NaN NaN \n", "29 96.9 NaN \n", "... ... ... \n", "44216 NaN 96.000000 \n", "44217 97.1 NaN \n", "44218 96.3 NaN \n", "44219 NaN 97.800003 \n", "44220 NaN 94.099998 \n", "44221 NaN 97.099998 \n", "44222 99.8 NaN \n", "44223 98.4 NaN \n", "44224 NaN 97.800003 \n", "44225 NaN 97.800003 \n", "44226 98.7 NaN \n", "44227 NaN NaN \n", "44228 NaN 98.000000 \n", "44229 96.9 NaN \n", "44230 NaN NaN \n", "44231 NaN NaN \n", "44232 100.0 NaN \n", "44233 NaN 97.900002 \n", "44234 NaN NaN \n", "44235 NaN 95.599998 \n", "44236 98.4 NaN \n", "44237 NaN 95.900002 \n", "44238 NaN 96.599998 \n", "44239 NaN 95.000000 \n", "44240 NaN NaN \n", "44241 NaN 98.699997 \n", "44242 NaN 99.400002 \n", "44243 NaN NaN \n", "44244 NaN NaN \n", "44245 97.8 NaN \n", "\n", " temperature body_degC_223762_LAST temperature body_degC_676_LAST \n", "0 NaN NaN \n", "1 NaN NaN \n", "2 NaN NaN \n", "3 NaN NaN \n", "4 NaN NaN \n", "5 NaN NaN \n", "6 34.5 NaN \n", "7 NaN NaN \n", "8 NaN NaN \n", "9 NaN NaN \n", "10 NaN NaN \n", "11 36.6 NaN \n", "12 NaN NaN \n", "13 NaN 36.599998 \n", "14 NaN NaN \n", "15 NaN NaN \n", "16 37.5 NaN \n", "17 NaN 36.799999 \n", "18 NaN NaN \n", "19 NaN NaN \n", "20 NaN NaN \n", "21 NaN NaN \n", "22 NaN NaN \n", "23 28.4 NaN \n", "24 NaN 36.799999 \n", "25 NaN NaN \n", "26 NaN NaN \n", "27 NaN NaN \n", "28 NaN 36.200001 \n", "29 NaN NaN \n", "... ... ... \n", "44216 NaN NaN \n", "44217 NaN NaN \n", "44218 NaN NaN \n", "44219 NaN NaN \n", "44220 NaN NaN \n", "44221 NaN NaN \n", "44222 NaN NaN \n", "44223 NaN NaN \n", "44224 NaN NaN \n", "44225 NaN NaN \n", "44226 NaN NaN \n", "44227 NaN 34.500000 \n", "44228 NaN NaN \n", "44229 NaN NaN \n", "44230 NaN 37.299999 \n", "44231 NaN 35.599998 \n", "44232 NaN NaN \n", "44233 NaN NaN \n", "44234 35.0 NaN \n", "44235 NaN NaN \n", "44236 NaN NaN \n", "44237 NaN NaN \n", "44238 NaN NaN \n", "44239 NaN NaN \n", "44240 34.8 NaN \n", "44241 NaN NaN \n", "44242 NaN NaN \n", "44243 NaN 35.099998 \n", "44244 NaN 34.700001 \n", "44245 NaN NaN \n", "\n", "[44246 rows x 16 columns]" ] }, "execution_count": 233, "metadata": {}, "output_type": "execute_result" } ], "source": [ "feature_df" ] }, { "cell_type": "code", "execution_count": 120, "metadata": { "collapsed": true }, "outputs": [], "source": [ "test_mapper_pipeline = Pipeline([\n", " ('ft_mapper',mapped_ft)\n", " ])" ] }, { "cell_type": "code", "execution_count": 121, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
temperature body_known_qn_degF_223761_MEANtemperature body_known_qn_degF_678_MEANtemperature body_known_qn_degF_223762(degC)_MEANtemperature body_known_qn_degF_676(degC)_MEANtemperature body_known_qn_degF_223761_STDtemperature body_known_qn_degF_678_STDtemperature body_known_qn_degF_223762(degC)_STDtemperature body_known_qn_degF_676(degC)_STDtemperature body_known_qn_degF_223761_COUNTtemperature body_known_qn_degF_678_COUNTtemperature body_known_qn_degF_223762(degC)_COUNTtemperature body_known_qn_degF_676(degC)_COUNTtemperature body_known_qn_degF_223761_LASTtemperature body_known_qn_degF_678_LASTtemperature body_known_qn_degF_223762(degC)_LASTtemperature body_known_qn_degF_676(degC)_LAST
098.537500NaNNaNNaN98.537500NaNNaNNaN98.537500NaNNaNNaN98.537500NaNNaNNaN
197.166667NaNNaNNaN97.166667NaNNaNNaN97.166667NaNNaNNaN97.166667NaNNaNNaN
2NaN97.533334NaNNaNNaN97.533334NaNNaNNaN97.533334NaNNaNNaN97.533334NaNNaN
3NaN98.591667NaNNaNNaN98.591667NaNNaNNaN98.591667NaNNaNNaN98.591667NaNNaN
498.100000NaNNaNNaN98.100000NaNNaNNaN98.100000NaNNaNNaN98.100000NaNNaNNaN
598.550000NaNNaNNaN98.550000NaNNaNNaN98.550000NaNNaNNaN98.550000NaNNaNNaN
698.380000NaN97.721177NaN98.380000NaN97.721177NaN98.380000NaN97.721177NaN98.380000NaN97.721177NaN
7NaN100.649998NaNNaNNaN100.649998NaNNaNNaN100.649998NaNNaNNaN100.649998NaNNaN
897.975000NaNNaNNaN97.975000NaNNaNNaN97.975000NaNNaNNaN97.975000NaNNaNNaN
996.050000NaNNaNNaN96.050000NaNNaNNaN96.050000NaNNaNNaN96.050000NaNNaNNaN
10NaN98.536364NaNNaNNaN98.536364NaNNaNNaN98.536364NaNNaNNaN98.536364NaNNaN
11NaNNaN97.481429NaNNaNNaN97.481429NaNNaNNaN97.481429NaNNaNNaN97.481429NaN
1299.466667NaNNaNNaN99.466667NaNNaNNaN99.466667NaNNaNNaN99.466667NaNNaNNaN
13NaN99.099998NaN98.178868NaN99.099998NaN98.178868NaN99.099998NaN98.178868NaN99.099998NaN98.178868
14NaN95.300003NaNNaNNaN95.300003NaNNaNNaN95.300003NaNNaNNaN95.300003NaNNaN
1598.500000NaNNaNNaN98.500000NaNNaNNaN98.500000NaNNaNNaN98.500000NaNNaNNaN
16100.772222NaN99.895122NaN100.772222NaN99.895122NaN100.772222NaN99.895122NaN100.772222NaN99.895122NaN
17NaNNaNNaN98.948750NaNNaNNaN98.948750NaNNaNNaN98.948750NaNNaNNaN98.948750
1899.345455NaNNaNNaN99.345455NaNNaNNaN99.345455NaNNaNNaN99.345455NaNNaNNaN
19NaN98.800003NaNNaNNaN98.800003NaNNaNNaN98.800003NaNNaNNaN98.800003NaNNaN
2098.312500NaNNaNNaN98.312500NaNNaNNaN98.312500NaNNaNNaN98.312500NaNNaNNaN
2197.760000NaNNaNNaN97.760000NaNNaNNaN97.760000NaNNaNNaN97.760000NaNNaNNaN
22NaN98.445455NaNNaNNaN98.445455NaNNaNNaN98.445455NaNNaNNaN98.445455NaNNaN
23NaNNaN94.983637NaNNaNNaN94.983637NaNNaNNaN94.983637NaNNaNNaN94.983637NaN
24NaNNaNNaN98.239999NaNNaNNaN98.239999NaNNaNNaN98.239999NaNNaNNaN98.239999
25NaN97.320000NaNNaNNaN97.320000NaNNaNNaN97.320000NaNNaNNaN97.320000NaNNaN
26NaN96.700000NaNNaNNaN96.700000NaNNaNNaN96.700000NaNNaNNaN96.700000NaNNaN
27NaN96.920000NaNNaNNaN96.920000NaNNaNNaN96.920000NaNNaNNaN96.920000NaNNaN
28NaNNaNNaN98.325263NaNNaNNaN98.325263NaNNaNNaN98.325263NaNNaNNaN98.325263
2996.525000NaNNaNNaN96.525000NaNNaNNaN96.525000NaNNaNNaN96.525000NaNNaNNaN
...................................................
44216NaN98.302703NaN98.639131NaN98.302703NaN98.639131NaN98.302703NaN98.639131NaN98.302703NaN98.639131
4421798.475000NaNNaNNaN98.475000NaNNaNNaN98.475000NaNNaNNaN98.475000NaNNaNNaN
4421896.275000NaNNaNNaN96.275000NaNNaNNaN96.275000NaNNaNNaN96.275000NaNNaNNaN
44219NaN97.800003NaNNaNNaN97.800003NaNNaNNaN97.800003NaNNaNNaN97.800003NaNNaN
44220NaN95.740001NaNNaNNaN95.740001NaNNaNNaN95.740001NaNNaNNaN95.740001NaNNaN
44221NaN97.920000NaNNaNNaN97.920000NaNNaNNaN97.920000NaNNaNNaN97.920000NaNNaN
4422299.240000NaNNaNNaN99.240000NaNNaNNaN99.240000NaNNaNNaN99.240000NaNNaNNaN
4422398.450000NaNNaNNaN98.450000NaNNaNNaN98.450000NaNNaNNaN98.450000NaNNaNNaN
44224NaN98.130000NaNNaNNaN98.130000NaNNaNNaN98.130000NaNNaNNaN98.130000NaNNaN
44225NaN97.900002NaNNaNNaN97.900002NaNNaNNaN97.900002NaNNaNNaN97.900002NaNNaN
4422699.753333NaNNaNNaN99.753333NaNNaNNaN99.753333NaNNaNNaN99.753333NaNNaNNaN
44227NaN98.071875NaN99.408525NaN98.071875NaN99.408525NaN98.071875NaN99.408525NaN98.071875NaN99.408525
44228NaN100.526316NaNNaNNaN100.526316NaNNaNNaN100.526316NaNNaNNaN100.526316NaNNaN
4422997.580000NaNNaNNaN97.580000NaNNaNNaN97.580000NaNNaNNaN97.580000NaNNaNNaN
44230NaNNaNNaN99.398333NaNNaNNaN99.398333NaNNaNNaN99.398333NaNNaNNaN99.398333
44231NaNNaNNaN96.928572NaNNaNNaN96.928572NaNNaNNaN96.928572NaNNaNNaN96.928572
4423298.484375NaNNaNNaN98.484375NaNNaNNaN98.484375NaNNaNNaN98.484375NaNNaNNaN
44233NaN100.278462NaNNaNNaN100.278462NaNNaNNaN100.278462NaNNaNNaN100.278462NaNNaN
44234NaNNaN97.112000NaNNaNNaN97.112000NaNNaNNaN97.112000NaNNaNNaN97.112000NaN
44235NaN97.074999NaNNaNNaN97.074999NaNNaNNaN97.074999NaNNaNNaN97.074999NaNNaN
4423697.977778NaNNaNNaN97.977778NaNNaNNaN97.977778NaNNaNNaN97.977778NaNNaNNaN
44237NaN97.485713NaNNaNNaN97.485713NaNNaNNaN97.485713NaNNaNNaN97.485713NaNNaN
44238NaN97.264285NaNNaNNaN97.264285NaNNaNNaN97.264285NaNNaNNaN97.264285NaNNaN
44239NaN95.400002NaNNaNNaN95.400002NaNNaNNaN95.400002NaNNaNNaN95.400002NaNNaN
4424095.800000NaN94.640000NaN95.800000NaN94.640000NaN95.800000NaN94.640000NaN95.800000NaN94.640000NaN
44241NaN97.670000NaN99.357306NaN97.670000NaN99.357306NaN97.670000NaN99.357306NaN97.670000NaN99.357306
44242NaN98.980953NaNNaNNaN98.980953NaNNaNNaN98.980953NaNNaNNaN98.980953NaNNaN
44243NaNNaNNaN96.350000NaNNaNNaN96.350000NaNNaNNaN96.350000NaNNaNNaN96.350000
44244NaNNaNNaN97.117143NaNNaNNaN97.117143NaNNaNNaN97.117143NaNNaNNaN97.117143
4424598.655000NaNNaNNaN98.655000NaNNaNNaN98.655000NaNNaNNaN98.655000NaNNaNNaN
\n", "

44246 rows × 16 columns

\n", "
" ], "text/plain": [ " temperature body_known_qn_degF_223761_MEAN \\\n", "0 98.537500 \n", "1 97.166667 \n", "2 NaN \n", "3 NaN \n", "4 98.100000 \n", "5 98.550000 \n", "6 98.380000 \n", "7 NaN \n", "8 97.975000 \n", "9 96.050000 \n", "10 NaN \n", "11 NaN \n", "12 99.466667 \n", "13 NaN \n", "14 NaN \n", "15 98.500000 \n", "16 100.772222 \n", "17 NaN \n", "18 99.345455 \n", "19 NaN \n", "20 98.312500 \n", "21 97.760000 \n", "22 NaN \n", "23 NaN \n", "24 NaN \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 NaN \n", "29 96.525000 \n", "... ... \n", "44216 NaN \n", "44217 98.475000 \n", "44218 96.275000 \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 99.240000 \n", "44223 98.450000 \n", "44224 NaN \n", "44225 NaN \n", "44226 99.753333 \n", "44227 NaN \n", "44228 NaN \n", "44229 97.580000 \n", "44230 NaN \n", "44231 NaN \n", "44232 98.484375 \n", "44233 NaN \n", "44234 NaN \n", "44235 NaN \n", "44236 97.977778 \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 95.800000 \n", "44241 NaN \n", "44242 NaN \n", "44243 NaN \n", "44244 NaN \n", "44245 98.655000 \n", "\n", " temperature body_known_qn_degF_678_MEAN \\\n", "0 NaN \n", "1 NaN \n", "2 97.533334 \n", "3 98.591667 \n", "4 NaN \n", "5 NaN \n", "6 NaN \n", "7 100.649998 \n", "8 NaN \n", "9 NaN \n", "10 98.536364 \n", "11 NaN \n", "12 NaN \n", "13 99.099998 \n", "14 95.300003 \n", "15 NaN \n", "16 NaN \n", "17 NaN \n", "18 NaN \n", "19 98.800003 \n", "20 NaN \n", "21 NaN \n", "22 98.445455 \n", "23 NaN \n", "24 NaN \n", "25 97.320000 \n", "26 96.700000 \n", "27 96.920000 \n", "28 NaN \n", "29 NaN \n", "... ... \n", "44216 98.302703 \n", "44217 NaN \n", "44218 NaN \n", "44219 97.800003 \n", "44220 95.740001 \n", "44221 97.920000 \n", "44222 NaN \n", "44223 NaN \n", "44224 98.130000 \n", "44225 97.900002 \n", "44226 NaN \n", "44227 98.071875 \n", "44228 100.526316 \n", "44229 NaN \n", "44230 NaN \n", "44231 NaN \n", "44232 NaN \n", "44233 100.278462 \n", "44234 NaN \n", "44235 97.074999 \n", "44236 NaN \n", "44237 97.485713 \n", "44238 97.264285 \n", "44239 95.400002 \n", "44240 NaN \n", "44241 97.670000 \n", "44242 98.980953 \n", "44243 NaN \n", "44244 NaN \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_223762(degC)_MEAN \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "5 NaN \n", "6 97.721177 \n", "7 NaN \n", "8 NaN \n", "9 NaN \n", "10 NaN \n", "11 97.481429 \n", "12 NaN \n", "13 NaN \n", "14 NaN \n", "15 NaN \n", "16 99.895122 \n", "17 NaN \n", "18 NaN \n", "19 NaN \n", "20 NaN \n", "21 NaN \n", "22 NaN \n", "23 94.983637 \n", "24 NaN \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 NaN \n", "29 NaN \n", "... ... \n", "44216 NaN \n", "44217 NaN \n", "44218 NaN \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 NaN \n", "44223 NaN \n", "44224 NaN \n", "44225 NaN \n", "44226 NaN \n", "44227 NaN \n", "44228 NaN \n", "44229 NaN \n", "44230 NaN \n", "44231 NaN \n", "44232 NaN \n", "44233 NaN \n", "44234 97.112000 \n", "44235 NaN \n", "44236 NaN \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 94.640000 \n", "44241 NaN \n", "44242 NaN \n", "44243 NaN \n", "44244 NaN \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_676(degC)_MEAN \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "5 NaN \n", "6 NaN \n", "7 NaN \n", "8 NaN \n", "9 NaN \n", "10 NaN \n", "11 NaN \n", "12 NaN \n", "13 98.178868 \n", "14 NaN \n", "15 NaN \n", "16 NaN \n", "17 98.948750 \n", "18 NaN \n", "19 NaN \n", "20 NaN \n", "21 NaN \n", "22 NaN \n", "23 NaN \n", "24 98.239999 \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 98.325263 \n", "29 NaN \n", "... ... \n", "44216 98.639131 \n", "44217 NaN \n", "44218 NaN \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 NaN \n", "44223 NaN \n", "44224 NaN \n", "44225 NaN \n", "44226 NaN \n", "44227 99.408525 \n", "44228 NaN \n", "44229 NaN \n", "44230 99.398333 \n", "44231 96.928572 \n", "44232 NaN \n", "44233 NaN \n", "44234 NaN \n", "44235 NaN \n", "44236 NaN \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 NaN \n", "44241 99.357306 \n", "44242 NaN \n", "44243 96.350000 \n", "44244 97.117143 \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_223761_STD \\\n", "0 98.537500 \n", "1 97.166667 \n", "2 NaN \n", "3 NaN \n", "4 98.100000 \n", "5 98.550000 \n", "6 98.380000 \n", "7 NaN \n", "8 97.975000 \n", "9 96.050000 \n", "10 NaN \n", "11 NaN \n", "12 99.466667 \n", "13 NaN \n", "14 NaN \n", "15 98.500000 \n", "16 100.772222 \n", "17 NaN \n", "18 99.345455 \n", "19 NaN \n", "20 98.312500 \n", "21 97.760000 \n", "22 NaN \n", "23 NaN \n", "24 NaN \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 NaN \n", "29 96.525000 \n", "... ... \n", "44216 NaN \n", "44217 98.475000 \n", "44218 96.275000 \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 99.240000 \n", "44223 98.450000 \n", "44224 NaN \n", "44225 NaN \n", "44226 99.753333 \n", "44227 NaN \n", "44228 NaN \n", "44229 97.580000 \n", "44230 NaN \n", "44231 NaN \n", "44232 98.484375 \n", "44233 NaN \n", "44234 NaN \n", "44235 NaN \n", "44236 97.977778 \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 95.800000 \n", "44241 NaN \n", "44242 NaN \n", "44243 NaN \n", "44244 NaN \n", "44245 98.655000 \n", "\n", " temperature body_known_qn_degF_678_STD \\\n", "0 NaN \n", "1 NaN \n", "2 97.533334 \n", "3 98.591667 \n", "4 NaN \n", "5 NaN \n", "6 NaN \n", "7 100.649998 \n", "8 NaN \n", "9 NaN \n", "10 98.536364 \n", "11 NaN \n", "12 NaN \n", "13 99.099998 \n", "14 95.300003 \n", "15 NaN \n", "16 NaN \n", "17 NaN \n", "18 NaN \n", "19 98.800003 \n", "20 NaN \n", "21 NaN \n", "22 98.445455 \n", "23 NaN \n", "24 NaN \n", "25 97.320000 \n", "26 96.700000 \n", "27 96.920000 \n", "28 NaN \n", "29 NaN \n", "... ... \n", "44216 98.302703 \n", "44217 NaN \n", "44218 NaN \n", "44219 97.800003 \n", "44220 95.740001 \n", "44221 97.920000 \n", "44222 NaN \n", "44223 NaN \n", "44224 98.130000 \n", "44225 97.900002 \n", "44226 NaN \n", "44227 98.071875 \n", "44228 100.526316 \n", "44229 NaN \n", "44230 NaN \n", "44231 NaN \n", "44232 NaN \n", "44233 100.278462 \n", "44234 NaN \n", "44235 97.074999 \n", "44236 NaN \n", "44237 97.485713 \n", "44238 97.264285 \n", "44239 95.400002 \n", "44240 NaN \n", "44241 97.670000 \n", "44242 98.980953 \n", "44243 NaN \n", "44244 NaN \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_223762(degC)_STD \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "5 NaN \n", "6 97.721177 \n", "7 NaN \n", "8 NaN \n", "9 NaN \n", "10 NaN \n", "11 97.481429 \n", "12 NaN \n", "13 NaN \n", "14 NaN \n", "15 NaN \n", "16 99.895122 \n", "17 NaN \n", "18 NaN \n", "19 NaN \n", "20 NaN \n", "21 NaN \n", "22 NaN \n", "23 94.983637 \n", "24 NaN \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 NaN \n", "29 NaN \n", "... ... \n", "44216 NaN \n", "44217 NaN \n", "44218 NaN \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 NaN \n", "44223 NaN \n", "44224 NaN \n", "44225 NaN \n", "44226 NaN \n", "44227 NaN \n", "44228 NaN \n", "44229 NaN \n", "44230 NaN \n", "44231 NaN \n", "44232 NaN \n", "44233 NaN \n", "44234 97.112000 \n", "44235 NaN \n", "44236 NaN \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 94.640000 \n", "44241 NaN \n", "44242 NaN \n", "44243 NaN \n", "44244 NaN \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_676(degC)_STD \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "5 NaN \n", "6 NaN \n", "7 NaN \n", "8 NaN \n", "9 NaN \n", "10 NaN \n", "11 NaN \n", "12 NaN \n", "13 98.178868 \n", "14 NaN \n", "15 NaN \n", "16 NaN \n", "17 98.948750 \n", "18 NaN \n", "19 NaN \n", "20 NaN \n", "21 NaN \n", "22 NaN \n", "23 NaN \n", "24 98.239999 \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 98.325263 \n", "29 NaN \n", "... ... \n", "44216 98.639131 \n", "44217 NaN \n", "44218 NaN \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 NaN \n", "44223 NaN \n", "44224 NaN \n", "44225 NaN \n", "44226 NaN \n", "44227 99.408525 \n", "44228 NaN \n", "44229 NaN \n", "44230 99.398333 \n", "44231 96.928572 \n", "44232 NaN \n", "44233 NaN \n", "44234 NaN \n", "44235 NaN \n", "44236 NaN \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 NaN \n", "44241 99.357306 \n", "44242 NaN \n", "44243 96.350000 \n", "44244 97.117143 \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_223761_COUNT \\\n", "0 98.537500 \n", "1 97.166667 \n", "2 NaN \n", "3 NaN \n", "4 98.100000 \n", "5 98.550000 \n", "6 98.380000 \n", "7 NaN \n", "8 97.975000 \n", "9 96.050000 \n", "10 NaN \n", "11 NaN \n", "12 99.466667 \n", "13 NaN \n", "14 NaN \n", "15 98.500000 \n", "16 100.772222 \n", "17 NaN \n", "18 99.345455 \n", "19 NaN \n", "20 98.312500 \n", "21 97.760000 \n", "22 NaN \n", "23 NaN \n", "24 NaN \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 NaN \n", "29 96.525000 \n", "... ... \n", "44216 NaN \n", "44217 98.475000 \n", "44218 96.275000 \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 99.240000 \n", "44223 98.450000 \n", "44224 NaN \n", "44225 NaN \n", "44226 99.753333 \n", "44227 NaN \n", "44228 NaN \n", "44229 97.580000 \n", "44230 NaN \n", "44231 NaN \n", "44232 98.484375 \n", "44233 NaN \n", "44234 NaN \n", "44235 NaN \n", "44236 97.977778 \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 95.800000 \n", "44241 NaN \n", "44242 NaN \n", "44243 NaN \n", "44244 NaN \n", "44245 98.655000 \n", "\n", " temperature body_known_qn_degF_678_COUNT \\\n", "0 NaN \n", "1 NaN \n", "2 97.533334 \n", "3 98.591667 \n", "4 NaN \n", "5 NaN \n", "6 NaN \n", "7 100.649998 \n", "8 NaN \n", "9 NaN \n", "10 98.536364 \n", "11 NaN \n", "12 NaN \n", "13 99.099998 \n", "14 95.300003 \n", "15 NaN \n", "16 NaN \n", "17 NaN \n", "18 NaN \n", "19 98.800003 \n", "20 NaN \n", "21 NaN \n", "22 98.445455 \n", "23 NaN \n", "24 NaN \n", "25 97.320000 \n", "26 96.700000 \n", "27 96.920000 \n", "28 NaN \n", "29 NaN \n", "... ... \n", "44216 98.302703 \n", "44217 NaN \n", "44218 NaN \n", "44219 97.800003 \n", "44220 95.740001 \n", "44221 97.920000 \n", "44222 NaN \n", "44223 NaN \n", "44224 98.130000 \n", "44225 97.900002 \n", "44226 NaN \n", "44227 98.071875 \n", "44228 100.526316 \n", "44229 NaN \n", "44230 NaN \n", "44231 NaN \n", "44232 NaN \n", "44233 100.278462 \n", "44234 NaN \n", "44235 97.074999 \n", "44236 NaN \n", "44237 97.485713 \n", "44238 97.264285 \n", "44239 95.400002 \n", "44240 NaN \n", "44241 97.670000 \n", "44242 98.980953 \n", "44243 NaN \n", "44244 NaN \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_223762(degC)_COUNT \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "5 NaN \n", "6 97.721177 \n", "7 NaN \n", "8 NaN \n", "9 NaN \n", "10 NaN \n", "11 97.481429 \n", "12 NaN \n", "13 NaN \n", "14 NaN \n", "15 NaN \n", "16 99.895122 \n", "17 NaN \n", "18 NaN \n", "19 NaN \n", "20 NaN \n", "21 NaN \n", "22 NaN \n", "23 94.983637 \n", "24 NaN \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 NaN \n", "29 NaN \n", "... ... \n", "44216 NaN \n", "44217 NaN \n", "44218 NaN \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 NaN \n", "44223 NaN \n", "44224 NaN \n", "44225 NaN \n", "44226 NaN \n", "44227 NaN \n", "44228 NaN \n", "44229 NaN \n", "44230 NaN \n", "44231 NaN \n", "44232 NaN \n", "44233 NaN \n", "44234 97.112000 \n", "44235 NaN \n", "44236 NaN \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 94.640000 \n", "44241 NaN \n", "44242 NaN \n", "44243 NaN \n", "44244 NaN \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_676(degC)_COUNT \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "5 NaN \n", "6 NaN \n", "7 NaN \n", "8 NaN \n", "9 NaN \n", "10 NaN \n", "11 NaN \n", "12 NaN \n", "13 98.178868 \n", "14 NaN \n", "15 NaN \n", "16 NaN \n", "17 98.948750 \n", "18 NaN \n", "19 NaN \n", "20 NaN \n", "21 NaN \n", "22 NaN \n", "23 NaN \n", "24 98.239999 \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 98.325263 \n", "29 NaN \n", "... ... \n", "44216 98.639131 \n", "44217 NaN \n", "44218 NaN \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 NaN \n", "44223 NaN \n", "44224 NaN \n", "44225 NaN \n", "44226 NaN \n", "44227 99.408525 \n", "44228 NaN \n", "44229 NaN \n", "44230 99.398333 \n", "44231 96.928572 \n", "44232 NaN \n", "44233 NaN \n", "44234 NaN \n", "44235 NaN \n", "44236 NaN \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 NaN \n", "44241 99.357306 \n", "44242 NaN \n", "44243 96.350000 \n", "44244 97.117143 \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_223761_LAST \\\n", "0 98.537500 \n", "1 97.166667 \n", "2 NaN \n", "3 NaN \n", "4 98.100000 \n", "5 98.550000 \n", "6 98.380000 \n", "7 NaN \n", "8 97.975000 \n", "9 96.050000 \n", "10 NaN \n", "11 NaN \n", "12 99.466667 \n", "13 NaN \n", "14 NaN \n", "15 98.500000 \n", "16 100.772222 \n", "17 NaN \n", "18 99.345455 \n", "19 NaN \n", "20 98.312500 \n", "21 97.760000 \n", "22 NaN \n", "23 NaN \n", "24 NaN \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 NaN \n", "29 96.525000 \n", "... ... \n", "44216 NaN \n", "44217 98.475000 \n", "44218 96.275000 \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 99.240000 \n", "44223 98.450000 \n", "44224 NaN \n", "44225 NaN \n", "44226 99.753333 \n", "44227 NaN \n", "44228 NaN \n", "44229 97.580000 \n", "44230 NaN \n", "44231 NaN \n", "44232 98.484375 \n", "44233 NaN \n", "44234 NaN \n", "44235 NaN \n", "44236 97.977778 \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 95.800000 \n", "44241 NaN \n", "44242 NaN \n", "44243 NaN \n", "44244 NaN \n", "44245 98.655000 \n", "\n", " temperature body_known_qn_degF_678_LAST \\\n", "0 NaN \n", "1 NaN \n", "2 97.533334 \n", "3 98.591667 \n", "4 NaN \n", "5 NaN \n", "6 NaN \n", "7 100.649998 \n", "8 NaN \n", "9 NaN \n", "10 98.536364 \n", "11 NaN \n", "12 NaN \n", "13 99.099998 \n", "14 95.300003 \n", "15 NaN \n", "16 NaN \n", "17 NaN \n", "18 NaN \n", "19 98.800003 \n", "20 NaN \n", "21 NaN \n", "22 98.445455 \n", "23 NaN \n", "24 NaN \n", "25 97.320000 \n", "26 96.700000 \n", "27 96.920000 \n", "28 NaN \n", "29 NaN \n", "... ... \n", "44216 98.302703 \n", "44217 NaN \n", "44218 NaN \n", "44219 97.800003 \n", "44220 95.740001 \n", "44221 97.920000 \n", "44222 NaN \n", "44223 NaN \n", "44224 98.130000 \n", "44225 97.900002 \n", "44226 NaN \n", "44227 98.071875 \n", "44228 100.526316 \n", "44229 NaN \n", "44230 NaN \n", "44231 NaN \n", "44232 NaN \n", "44233 100.278462 \n", "44234 NaN \n", "44235 97.074999 \n", "44236 NaN \n", "44237 97.485713 \n", "44238 97.264285 \n", "44239 95.400002 \n", "44240 NaN \n", "44241 97.670000 \n", "44242 98.980953 \n", "44243 NaN \n", "44244 NaN \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_223762(degC)_LAST \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "5 NaN \n", "6 97.721177 \n", "7 NaN \n", "8 NaN \n", "9 NaN \n", "10 NaN \n", "11 97.481429 \n", "12 NaN \n", "13 NaN \n", "14 NaN \n", "15 NaN \n", "16 99.895122 \n", "17 NaN \n", "18 NaN \n", "19 NaN \n", "20 NaN \n", "21 NaN \n", "22 NaN \n", "23 94.983637 \n", "24 NaN \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 NaN \n", "29 NaN \n", "... ... \n", "44216 NaN \n", "44217 NaN \n", "44218 NaN \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 NaN \n", "44223 NaN \n", "44224 NaN \n", "44225 NaN \n", "44226 NaN \n", "44227 NaN \n", "44228 NaN \n", "44229 NaN \n", "44230 NaN \n", "44231 NaN \n", "44232 NaN \n", "44233 NaN \n", "44234 97.112000 \n", "44235 NaN \n", "44236 NaN \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 94.640000 \n", "44241 NaN \n", "44242 NaN \n", "44243 NaN \n", "44244 NaN \n", "44245 NaN \n", "\n", " temperature body_known_qn_degF_676(degC)_LAST \n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "5 NaN \n", "6 NaN \n", "7 NaN \n", "8 NaN \n", "9 NaN \n", "10 NaN \n", "11 NaN \n", "12 NaN \n", "13 98.178868 \n", "14 NaN \n", "15 NaN \n", "16 NaN \n", "17 98.948750 \n", "18 NaN \n", "19 NaN \n", "20 NaN \n", "21 NaN \n", "22 NaN \n", "23 NaN \n", "24 98.239999 \n", "25 NaN \n", "26 NaN \n", "27 NaN \n", "28 98.325263 \n", "29 NaN \n", "... ... \n", "44216 98.639131 \n", "44217 NaN \n", "44218 NaN \n", "44219 NaN \n", "44220 NaN \n", "44221 NaN \n", "44222 NaN \n", "44223 NaN \n", "44224 NaN \n", "44225 NaN \n", "44226 NaN \n", "44227 99.408525 \n", "44228 NaN \n", "44229 NaN \n", "44230 99.398333 \n", "44231 96.928572 \n", "44232 NaN \n", "44233 NaN \n", "44234 NaN \n", "44235 NaN \n", "44236 NaN \n", "44237 NaN \n", "44238 NaN \n", "44239 NaN \n", "44240 NaN \n", "44241 99.357306 \n", "44242 NaN \n", "44243 96.350000 \n", "44244 97.117143 \n", "44245 NaN \n", "\n", "[44246 rows x 16 columns]" ] }, "execution_count": 121, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_mapper_pipeline.transform(df)" ] }, { "cell_type": "code", "execution_count": 152, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 152, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ureg" ] }, { "cell_type": "code", "execution_count": 197, "metadata": { "collapsed": false }, "outputs": [], "source": [ "mass = ureg.parse_units('degF')" ] }, { "cell_type": "code", "execution_count": 198, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 198, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mass.dimensionality" ] }, { "cell_type": "code", "execution_count": 157, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from pint.unit import UnitsContainer" ] }, { "cell_type": "code", "execution_count": 160, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 160, "metadata": {}, "output_type": "execute_result" } ], "source": [ "units.dimensionality == UnitsContainer({'[length]':3.0})" ] }, { "cell_type": "code", "execution_count": 166, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "1e-06 kilogram" ], "text/latex": [ "$1e-06\\ \\mathrm{kilogram}$" ], "text/plain": [ "" ] }, "execution_count": 166, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(1*units).to_base_units()" ] }, { "cell_type": "code", "execution_count": 161, "metadata": { "collapsed": true }, "outputs": [], "source": [ "uc = UnitsContainer({'[length]':3.0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "uc.dim" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import constants\n", "import mimic\n", "import utils\n", "import pandas as pd\n", "import icu_data_defs\n", "import units\n", "from sklearn.pipeline import Pipeline\n", "import transformers\n", "import logger\n", "import features\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(mimic)\n", "reload(units)\n", "reload(utils)\n", "reload(transformers)\n", "def mimic_ETL(components,data_dict,hdf5_fname,\n", " hadm_ids=constants.ALL,\n", " agg_func=lambda x:x.iloc[0]):\n", " \n", " logger.log('BEGIN ETL for {} admissions'.format(hadm_ids if hadm_ids == constants.ALL else len(hadm_ids)),new_level=True)\n", " category_map = mimic.mimic_category_map(data_dict)\n", " ureg = units.MedicalUreg()\n", " extractor = mimic.mimic_extractor('config/mimic_item_map.csv',data_dict)\n", " \n", "\n", " transform_pipeline = mimic.transform_pipeline()\n", "\n", " standard_clean_pipeline = Pipeline([\n", " ('aggregate_same_datetime',transformers.same_index_aggregator(agg_func)),\n", " ('split_dtype',transformers.split_dtype()),\n", " ('standardize_columns',transformers.column_standardizer(data_dict,ureg)),\n", " ('standardize_categories',transformers.standardize_categories(data_dict,category_map)),\n", " ('split_bad_categories',transformers.split_bad_categories(data_dict)),\n", " ('one_hotter',transformers.nominal_to_onehot()),\n", " ('drop_oob_values',transformers.oob_value_remover(data_dict))\n", " ])\n", " \n", "\n", " logger.log('Extract CONTEXT...')\n", " df_context = mimic.get_context_data(hadm_ids)\n", " utils.save_df(df_context,hdf5_fname,'context')\n", "\n", " \n", " for component in components:\n", " logger.log(component.upper(),new_level=True)\n", " \n", " logger.log(\"Extracting...\",new_level=True)\n", " df_extracted = extractor.extract_component(component,hadm_ids)\n", " utils.save_df(df_extracted,hdf5_fname,'extracted/{}'.format(component))\n", " logger.end_log_level()\n", " \n", " display(df_extracted.head())\n", " \n", " logger.log(\"Transforming... {}\".format(df_extracted.shape))\n", " transform_pipeline.set_params(add_level__level_val=component)\n", " df_transformed = transform_pipeline.transform(df_extracted)\n", " utils.save_df(df_transformed,hdf5_fname,'transformed/{}'.format(component))\n", "\n", " display(df_transformed.head())\n", "\n", " display(df_transformed.describe())\n", "\n", " print utils.data_loss(df_extracted.set_index('id').value.to_frame(),df_transformed)\n", " \n", " logger.log(\"Cleaning... {}\".format(df_transformed.shape)) \n", " df_cleaned = standard_clean_pipeline.transform(df_transformed)\n", " utils.save_df(df_cleaned,hdf5_fname,'cleaned/{}'.format(component))\n", " \n", " display(df_cleaned.head())\n", "\n", " display(df_cleaned.describe())\n", "\n", " print utils.data_loss(df_extracted.set_index('id').value.to_frame(),df_cleaned)\n", " \n", " del df_cleaned,df_transformed,df_extracted\n", " logger.end_log_level()\n", " \n", " \n", " \n", " logger.end_log()\n", " \n", " return\n", " \n", "\n", "def mimic_features(hdf5_fname,specific_path,labels,\n", " custom_cleaners,segmenter,feature_tuples):\n", " \n", " \n", " df_all = None\n", " \n", " for label in labels:\n", " df_base = utils.open_df(hdf5_fname,'cleaned/{}'.format(label))\n", " \n", " df_cleaned = custom_cleaners.transform(df_base)\n", " utils.save_df(df_cleaned,hdf5_fname,'{}/cleaned/{}'.format(specific_path,label))\n", " \n", " if df_all is None:\n", " df_all = df_cleaned\n", " else:\n", " df_all = df_all.join(df_cleaned,how='outer')\n", " del df_cleaned\n", " \n", " utils.save_df(df_all,hdf5_fname,'{}/cleaned/all')\n", " \n", " df_segmented = segmenter.transform(df_all)\n", " utils.save_df(df_segmented,hdf5_fname_target,'{}/segmented'.format(specific_path))\n", " del df_all\n", " \n", " mapped_ft = features.make_mapper(feature_tuples,df_segmented)\n", " df_features = mapped_ft.transform(df_segmented)\n", " utils.save_df(df_features,hdf5_fname_target,'{}/features'.format(specific_path))\n", " del df_segmented\n", " \n", " return df_features" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "collapsed": false }, "outputs": [], "source": [ "def do_ETL(data_dict,components,tag,n,seed=42):\n", " hdf5_fname = 'data/mimic_{}_{}'.format(tag,n)\n", " hadm_ids = n if n == constants.ALL else mimic.sample_hadm_ids(n,seed) \n", " mimic_ETL(components,data_dict,hdf5_fname,hadm_ids=hadm_ids)\n", " return hdf5_fname" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "collapsed": false }, "outputs": [], "source": [ "reload(logger)\n", "\n", "data_dict = icu_data_defs.data_dictionary('config/data_definitions.xlsx')\n", "components = data_dict.get_panel_defintions(12).component.unique().tolist() #12 is \"simple data\"\n", "tag = 'simple'" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-20 10:03:09) BEGIN ETL for 1000 admissions\n", "(2017-06-20 10:03:09)>> Extract CONTEXT...\n", "(2017-06-20 10:03:17)<< DONE (8.0s)\n", "(2017-06-20 10:03:17)>> BLOOD PRESSURE SYSTOLIC\n", "(2017-06-20 10:03:17)>>>> Extracting...\n", "(2017-06-20 10:03:17)>>>>>> Extracting 14 items from chartevents\n", "51 35823\n", "455 26680\n", "220179 21957\n", "220050 15186\n", "3313 2250\n", "225309 1251\n", "3315 50\n", "3317 24\n", "3323 21\n", "442 19\n", "3321 19\n", "224167 12\n", "227243 8\n", "Name: itemid, dtype: int64\n", "Empty DataFrame\n", "Columns: [id, datetime, value, units, itemid]\n", "Index: []\n", "(2017-06-20 10:04:09)<<<<<< DONE (52.0s)\n", "(2017-06-20 10:04:09)>>>>>> Combine DF\n", "(2017-06-20 10:04:09)<<<<<< DONE (0.0s)\n", "(2017-06-20 10:04:09)>>>>>> Clean UOM\n", "(2017-06-20 10:04:10)<<<<<< DONE (1.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
iddatetimevalueunitsitemid
01078802106-06-19 22:00:0099mmHg51
11078802106-06-19 23:00:0098mmHg51
21078802106-06-20 00:00:00122mmHg51
31078802106-06-20 01:00:00150mmHg51
41078802106-06-20 02:00:00133mmHg51
\n", "
" ], "text/plain": [ " id datetime value units itemid\n", "0 107880 2106-06-19 22:00:00 99 mmHg 51\n", "1 107880 2106-06-19 23:00:00 98 mmHg 51\n", "2 107880 2106-06-20 00:00:00 122 mmHg 51\n", "3 107880 2106-06-20 01:00:00 150 mmHg 51\n", "4 107880 2106-06-20 02:00:00 133 mmHg 51" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-20 10:04:10)<<<< DONE (53.0s)\n", "(2017-06-20 10:04:10)>>>> Transforming... (103300, 5)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
componentblood pressure systolic
unitscc/minmmHgcc/minmmHg
description3313455512201794422200503315331733213323224167227243225309
iddatetime
1003042161-10-01 22:00:0067NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
2161-10-01 23:00:0068NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
1007882120-04-10 14:00:00None127NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
2120-04-10 15:00:00None127NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
2120-04-10 16:00:00None123NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
\n", "
" ], "text/plain": [ "component blood pressure systolic \\\n", "units cc/min mmHg \n", "description 3313 455 51 220179 \n", "id datetime \n", "100304 2161-10-01 22:00:00 67 None None None \n", " 2161-10-01 23:00:00 68 None None None \n", "100788 2120-04-10 14:00:00 None 127 None None \n", " 2120-04-10 15:00:00 None 127 None None \n", " 2120-04-10 16:00:00 None 123 None None \n", "\n", "component \\\n", "units cc/min mmHg \n", "description 442 220050 3315 3317 3321 3323 224167 \n", "id datetime \n", "100304 2161-10-01 22:00:00 None None None None None None None \n", " 2161-10-01 23:00:00 None None None None None None None \n", "100788 2120-04-10 14:00:00 None None None None None None None \n", " 2120-04-10 15:00:00 None None None None None None None \n", " 2120-04-10 16:00:00 None None None None None None None \n", "\n", "component \n", "units \n", "description 227243 225309 \n", "id datetime \n", "100304 2161-10-01 22:00:00 None None \n", " 2161-10-01 23:00:00 None None \n", "100788 2120-04-10 14:00:00 None None \n", " 2120-04-10 15:00:00 None None \n", " 2120-04-10 16:00:00 None None " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
componentblood pressure systolic
unitscc/minmmHgcc/minmmHg
description3313455512201794422200503315331733213323224167227243225309
count22412648035622219571715186221613141281251
unique81181208176141722113121198126
top7210610811212212089746680102110107
freq91494641457229622224141
\n", "
" ], "text/plain": [ "component blood pressure systolic \\\n", "units cc/min mmHg cc/min \n", "description 3313 455 51 220179 442 220050 3315 \n", "count 2241 26480 35622 21957 17 15186 22 \n", "unique 81 181 208 176 14 172 21 \n", "top 72 106 108 112 122 120 89 \n", "freq 91 494 641 457 2 296 2 \n", "\n", "component \n", "units mmHg \n", "description 3317 3321 3323 224167 227243 225309 \n", "count 16 13 14 12 8 1251 \n", "unique 13 12 11 9 8 126 \n", "top 74 66 80 102 110 107 \n", "freq 2 2 2 4 1 41 " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "((103300, 1), (97535, 13), 461L, 2, '0.2094% records')\n", "(2017-06-20 10:04:12)<<<< DONE (2.0s)\n", "(2017-06-20 10:04:12)>>>> Cleaning... (97535, 13)\n", "(2017-06-20 10:04:14)<<<< DONE (2.0s)\n", "(2017-06-20 10:04:14)>>>> Nominal to OneHot\n", "(2017-06-20 10:04:14)<<<< DONE (0.0s)\n", "(2017-06-20 10:04:14)>>>> Drop OOB data | (97527, 13)\n", "(2017-06-20 10:04:14)>>>>>> blood pressure systolic, mmHg, 100525\n", "(2017-06-20 10:04:19)<<<<<< DONE (5.0s)\n", "(2017-06-20 10:04:19)>>>>>> blood pressure systolic, cc/min, 2306\n", "(2017-06-20 10:04:19)<<<<<< DONE (0.0s)\n" ] }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
componentblood pressure systolic
statusknownunknown
variable_typeqnqn
unitsmmHgcc/min
description2200502201792241672253092272434424555133133315331733213323
iddatetime
1003042161-10-01 22:00:00NaNNaNNaNNaNNaNNaNNaNNaN67.0NaNNaNNaNNaN
2161-10-01 23:00:00NaNNaNNaNNaNNaNNaNNaNNaN68.0NaNNaNNaNNaN
1007882120-04-10 14:00:00NaNNaNNaNNaNNaNNaN127.0NaNNaNNaNNaNNaNNaN
2120-04-10 15:00:00NaNNaNNaNNaNNaNNaN127.0NaNNaNNaNNaNNaNNaN
2120-04-10 16:00:00NaNNaNNaNNaNNaNNaN123.0NaNNaNNaNNaNNaNNaN
\n", "
" ], "text/plain": [ "component blood pressure systolic \\\n", "status known \n", "variable_type qn \n", "units mmHg \n", "description 220050 220179 224167 225309 \n", "id datetime \n", "100304 2161-10-01 22:00:00 NaN NaN NaN NaN \n", " 2161-10-01 23:00:00 NaN NaN NaN NaN \n", "100788 2120-04-10 14:00:00 NaN NaN NaN NaN \n", " 2120-04-10 15:00:00 NaN NaN NaN NaN \n", " 2120-04-10 16:00:00 NaN NaN NaN NaN \n", "\n", "component \n", "status unknown \n", "variable_type qn \n", "units cc/min \n", "description 227243 442 455 51 3313 3315 3317 3321 3323 \n", "id datetime \n", "100304 2161-10-01 22:00:00 NaN NaN NaN NaN 67.0 NaN NaN NaN NaN \n", " 2161-10-01 23:00:00 NaN NaN NaN NaN 68.0 NaN NaN NaN NaN \n", "100788 2120-04-10 14:00:00 NaN NaN 127.0 NaN NaN NaN NaN NaN NaN \n", " 2120-04-10 15:00:00 NaN NaN 127.0 NaN NaN NaN NaN NaN NaN \n", " 2120-04-10 16:00:00 NaN NaN 123.0 NaN NaN NaN NaN NaN NaN " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
componentblood pressure systolic
statusknownunknown
variable_typeqnqn
unitsmmHgcc/min
description2200502201792241672253092272434424555133133315331733213323
count15186.00000021957.00000012.0000001251.0000008.0000017.00000026478.00000035616.0000002241.00000022.00000016.0000013.00000014.000000
mean118.988213119.200483104.416667110.457234111.87500122.882353119.150623121.26462870.37170977.72727376.6875077.84615475.714286
std23.36600721.75774815.55902720.46570719.0520931.26276223.20657325.92791612.96997515.49556617.7265114.93232614.514865
min0.00000033.00000086.00000023.00000084.0000070.0000000.0000000.00000038.00000054.00000050.0000058.00000055.000000
25%103.000000104.00000097.50000099.00000097.2500096.000000102.000000104.00000061.00000066.25000062.7500067.00000066.000000
50%117.000000117.000000102.000000108.000000116.00000125.000000117.000000119.00000070.00000076.00000076.0000075.00000075.500000
75%133.000000133.000000104.000000120.000000128.50000138.000000135.000000137.00000077.00000086.75000086.0000084.00000080.000000
max244.000000247.000000144.000000273.000000132.00000174.000000234.000000284.000000123.000000112.000000120.00000108.00000099.000000
\n", "
" ], "text/plain": [ "component blood pressure systolic \\\n", "status known \n", "variable_type qn \n", "units mmHg \n", "description 220050 220179 224167 225309 \n", "count 15186.000000 21957.000000 12.000000 1251.000000 \n", "mean 118.988213 119.200483 104.416667 110.457234 \n", "std 23.366007 21.757748 15.559027 20.465707 \n", "min 0.000000 33.000000 86.000000 23.000000 \n", "25% 103.000000 104.000000 97.500000 99.000000 \n", "50% 117.000000 117.000000 102.000000 108.000000 \n", "75% 133.000000 133.000000 104.000000 120.000000 \n", "max 244.000000 247.000000 144.000000 273.000000 \n", "\n", "component \\\n", "status unknown \n", "variable_type qn \n", "units cc/min \n", "description 227243 442 455 51 3313 \n", "count 8.00000 17.000000 26478.000000 35616.000000 2241.000000 \n", "mean 111.87500 122.882353 119.150623 121.264628 70.371709 \n", "std 19.05209 31.262762 23.206573 25.927916 12.969975 \n", "min 84.00000 70.000000 0.000000 0.000000 38.000000 \n", "25% 97.25000 96.000000 102.000000 104.000000 61.000000 \n", "50% 116.00000 125.000000 117.000000 119.000000 70.000000 \n", "75% 128.50000 138.000000 135.000000 137.000000 77.000000 \n", "max 132.00000 174.000000 234.000000 284.000000 123.000000 \n", "\n", "component \n", "status \n", "variable_type \n", "units \n", "description 3315 3317 3321 3323 \n", "count 22.000000 16.00000 13.000000 14.000000 \n", "mean 77.727273 76.68750 77.846154 75.714286 \n", "std 15.495566 17.72651 14.932326 14.514865 \n", "min 54.000000 50.00000 58.000000 55.000000 \n", "25% 66.250000 62.75000 67.000000 66.000000 \n", "50% 76.000000 76.00000 75.000000 75.500000 \n", "75% 86.750000 86.00000 84.000000 80.000000 \n", "max 112.000000 120.00000 108.000000 99.000000 " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "((103300, 1), (97527, 13), 469L, 2, '0.2094% records')\n", "(2017-06-20 10:04:19)<<<< DONE (5.0s)\n", "(2017-06-20 10:04:19)<< DONE (62.0s)\n", "(2017-06-20 10:04:19) DONE (70.0s)\n" ] } ], "source": [ "n = 100\n", "hdf5_fname = do_ETL(data_dict,[data_dict.components.BLOOD_PRESSURE_SYSTOLIC],tag,n)" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2017-06-20 10:09:14) BEGIN ETL for all admissions\n", "(2017-06-20 10:09:15)>> Extract CONTEXT...\n", "(2017-06-20 10:09:27)<< DONE (12.0s)\n", "(2017-06-20 10:09:27)>> BLOOD PRESSURE SYSTOLIC\n", "(2017-06-20 10:09:27)>>>> Extracting...\n", "(2017-06-20 10:09:28)>>>>>> Extracting 14 items from chartevents\n", "51 2099353\n", "455 1586769\n", "220179 1290488\n", "220050 1149788\n", "3313 148105\n", "225309 86557\n", "3315 3762\n", "442 2565\n", "3317 2052\n", "3323 2039\n", "3321 2032\n", "224167 740\n", "227243 541\n", "6 33\n", "Name: itemid, dtype: int64\n", " id datetime value units itemid\n", "963709 136796.0 2139-10-06 21:00:00 113/55 6\n", "963711 136796.0 2139-10-06 22:00:00 126/58 6\n", "963713 136796.0 2139-10-06 23:00:00 139/64 6\n", "964245 136796.0 2139-10-07 07:00:00 132/59 6\n", "964247 136796.0 2139-10-07 08:00:00 143/63 6\n", "964301 136796.0 2139-10-07 16:00:00 152/68 6\n", "964304 136796.0 2139-10-07 18:00:00 141/59 6\n", "964710 136796.0 2139-10-07 09:00:00 133/59 6\n", "964743 136796.0 2139-10-07 10:00:00 131/60 6\n", "965088 136796.0 2139-10-07 11:00:00 123/54 6\n", "965090 136796.0 2139-10-07 12:00:00 185/79 6\n", "965092 136796.0 2139-10-07 13:00:00 116/54 6\n", "965094 136796.0 2139-10-07 14:00:00 122/57 6\n", "965096 136796.0 2139-10-07 15:00:00 120/56 6\n", "965438 136796.0 2139-10-06 13:00:00 122/55 6\n", "965440 136796.0 2139-10-06 14:00:00 110/56 6\n", "965854 136796.0 2139-10-06 18:00:00 108/56 6\n", "965856 136796.0 2139-10-06 20:00:00 115/56 6\n", "966407 136796.0 2139-10-07 03:00:00 129/57 6\n", "966409 136796.0 2139-10-07 04:00:00 121/55 6\n", "966411 136796.0 2139-10-07 05:00:00 125/56 6\n", "968122 136796.0 2139-10-06 09:00:00 119/55 6\n", "968125 136796.0 2139-10-06 12:00:00 113/60 6\n", "968615 136796.0 2139-10-07 00:00:00 120/54 6\n", "968617 136796.0 2139-10-07 01:00:00 130/58 6\n", "968619 136796.0 2139-10-07 02:00:00 124/56 6\n" ] }, { "ename": "KeyError", "evalue": "0L", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)", "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[0mn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconstants\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mALL\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mhdf5_fname\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdo_ETL\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata_dict\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdata_dict\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcomponents\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBLOOD_PRESSURE_SYSTOLIC\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'bp'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[1;32m\u001b[0m in \u001b[0;36mdo_ETL\u001b[1;34m(data_dict, components, tag, n, seed)\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[0mhdf5_fname\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'data/mimic_{}_{}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtag\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[0mhadm_ids\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mn\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mn\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mconstants\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mALL\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mmimic\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msample_hadm_ids\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mmimic_ETL\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcomponents\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdata_dict\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhdf5_fname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhadm_ids\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mhadm_ids\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 5\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mhdf5_fname\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32m\u001b[0m in \u001b[0;36mmimic_ETL\u001b[1;34m(components, data_dict, hdf5_fname, hadm_ids, agg_func)\u001b[0m\n\u001b[0;32m 35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 36\u001b[0m \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Extracting...\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnew_level\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m \u001b[0mdf_extracted\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextractor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextract_component\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhadm_ids\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 38\u001b[0m \u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_df\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_extracted\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhdf5_fname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'extracted/{}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 39\u001b[0m \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend_log_level\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\icu_ml_project\\v5\\mimic.py\u001b[0m in \u001b[0;36mextract_component\u001b[1;34m(self, component, hadm_ids)\u001b[0m\n\u001b[0;32m 95\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 96\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mextract_component\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhadm_ids\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mALL\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 97\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mextract_component\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem_map\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata_dict\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhadm_ids\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 98\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 99\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\icu_ml_project\\v5\\mimic.py\u001b[0m in \u001b[0;36mextract_component\u001b[1;34m(mimic_conn, component, item_map, data_dict, hadm_ids)\u001b[0m\n\u001b[0;32m 132\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mbp_slice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 133\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mcomponent\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mdata_dict\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcomponents\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBLOOD_PRESSURE_SYSTOLIC\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 134\u001b[1;33m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mis_item_6\u001b[0m \u001b[1;33m&\u001b[0m \u001b[0mhas_slash\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mis_item_6\u001b[0m \u001b[1;33m&\u001b[0m \u001b[0mhas_slash\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'/'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 135\u001b[0m \u001b[1;32melif\u001b[0m \u001b[0mcomponent\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mdata_dict\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcomponents\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBLOOD_PRESSURE_DIASTOLIC\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 136\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mhas_slash\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mhas_slash\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'/'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\core\\series.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m 599\u001b[0m \u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_apply_if_callable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 600\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 601\u001b[1;33m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 602\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 603\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mC:\\Users\\genkinjz\\AppData\\Local\\Continuum\\Anaconda2\\lib\\site-packages\\pandas\\indexes\\base.pyc\u001b[0m in \u001b[0;36mget_value\u001b[1;34m(self, series, key)\u001b[0m\n\u001b[0;32m 2137\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2138\u001b[0m return self._engine.get_value(s, k,\n\u001b[1;32m-> 2139\u001b[1;33m tz=getattr(series.dtype, 'tz', None))\n\u001b[0m\u001b[0;32m 2140\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2141\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minferred_type\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'integer'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'boolean'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;32mpandas\\index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_value (pandas\\index.c:3338)\u001b[1;34m()\u001b[0m\n", "\u001b[1;32mpandas\\index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_value (pandas\\index.c:3041)\u001b[1;34m()\u001b[0m\n", "\u001b[1;32mpandas\\index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas\\index.c:4024)\u001b[1;34m()\u001b[0m\n", "\u001b[1;32mpandas\\src\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas.hashtable.Int64HashTable.get_item (pandas\\hashtable.c:8141)\u001b[1;34m()\u001b[0m\n", "\u001b[1;32mpandas\\src\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas.hashtable.Int64HashTable.get_item (pandas\\hashtable.c:8085)\u001b[1;34m()\u001b[0m\n", "\u001b[1;31mKeyError\u001b[0m: 0L" ] } ], "source": [ "n = constants.ALL\n", "hdf5_fname = do_ETL(data_dict,[data_dict.components.BLOOD_PRESSURE_SYSTOLIC],'bp',n)" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "\n", "File path: data/mimic_simple_all\n", "/cleaned/blood pressure diastolic frame \n", "/cleaned/blood pressure mean frame \n", "/cleaned/blood pressure systolic frame \n", "/cleaned/glasgow coma scale eye opening frame \n", "/cleaned/glasgow coma scale motor frame \n", "/cleaned/glasgow coma scale verbal frame \n", "/cleaned/heart rate frame \n", "/cleaned/hemoglobin frame \n", "/cleaned/lactate frame \n", "/cleaned/lactated ringers frame \n", "/cleaned/norepinephrine frame \n", "/cleaned/normal saline frame \n", "/cleaned/output urine frame \n", "/cleaned/oxygen saturation pulse oximetry frame \n", "/cleaned/respiratory rate frame \n", "/cleaned/temperature body frame \n", "/cleaned/vasopressin frame \n", "/cleaned/weight body frame \n", "/context frame (shape->[62722,22]) \n", "/extracted/blood pressure diastolic frame (shape->[6371249,5])\n", "/extracted/blood pressure mean frame (shape->[2536271,5])\n", "/extracted/blood pressure systolic frame (shape->[6374824,5])\n", "/extracted/glasgow coma scale eye opening frame (shape->[956672,5]) \n", "/extracted/glasgow coma scale motor frame (shape->[952565,5]) \n", "/extracted/glasgow coma scale verbal frame (shape->[954700,5]) \n", "/extracted/heart rate frame (shape->[7952939,5])\n", "/extracted/hemoglobin frame (shape->[1167921,5])\n", "/extracted/lactate frame (shape->[393608,5]) \n", "/extracted/lactated ringers frame (shape->[504306,5]) \n", "/extracted/norepinephrine frame (shape->[1136938,5])\n", "/extracted/normal saline frame (shape->[817373,5]) \n", "/extracted/output urine frame (shape->[3644639,5])\n", "/extracted/oxygen saturation pulse oximetry frame (shape->[6099827,5])\n", "/extracted/respiratory rate frame (shape->[7810019,5])\n", "/extracted/temperature body frame (shape->[1751447,5])\n", "/extracted/vasopressin frame (shape->[339184,5]) \n", "/extracted/weight body frame (shape->[95425,5]) \n", "/transformed/blood pressure diastolic frame \n", "/transformed/blood pressure mean frame \n", "/transformed/blood pressure systolic frame \n", "/transformed/glasgow coma scale eye opening frame \n", "/transformed/glasgow coma scale motor frame \n", "/transformed/glasgow coma scale verbal frame \n", "/transformed/heart rate frame \n", "/transformed/hemoglobin frame \n", "/transformed/lactate frame \n", "/transformed/lactated ringers frame \n", "/transformed/norepinephrine frame \n", "/transformed/normal saline frame \n", "/transformed/output urine frame \n", "/transformed/oxygen saturation pulse oximetry frame \n", "/transformed/respiratory rate frame \n", "/transformed/temperature body frame \n", "/transformed/vasopressin frame \n", "/transformed/weight body frame " ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "store = pd.HDFStore(hdf5_fname)\n", "store" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "collapsed": true }, "outputs": [], "source": [ "store.close()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import utils\n", "import mimic\n", "import icu_data_defs\n", "import transformers" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false }, "outputs": [], "source": [ "data_dict = icu_data_defs.data_dictionary('config/data_definitions.xlsx')\n", "components = data_dict.get_panel_defintions(1).component.unique().tolist() #1 is vitals" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "[u'heart rate',\n", " u'blood pressure systolic',\n", " u'blood pressure diastolic',\n", " u'blood pressure mean',\n", " u'respiratory rate',\n", " u'temperature body',\n", " u'oxygen saturation pulse oximetry',\n", " u'weight body']" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "components" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(2017-07-04 22:28:15)<<<<<<<< --- (25.0s)\n", "(2017-07-04 22:28:15)>>>>>>>> ***ETL***\n", "(2017-07-04 22:28:15)>>>>>>>>>> SETUP\n", "(2017-07-04 22:28:15)<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:28:15)>>>>>>>>>> BEGIN ETL for all admissions and 8 components: [u'heart rate', u'blood pressure systolic', u'blood pressure diastolic', u'blood pressure mean', u'respiratory rate', u'temperature body', u'oxygen saturation pulse oximetry', u'weight body']\n", "(2017-07-04 22:28:15)>>>>>>>>>>>> HEART RATE: 1/8\n", "(2017-07-04 22:28:15)>>>>>>>>>>>>>> Extracting...\n", "(2017-07-04 22:28:16)>>>>>>>>>>>>>>>> Extracting 5 items from chartevents\n", "(2017-07-04 22:29:12)<<<<<<<<<<<<<<<< --- (56.0s)\n", "(2017-07-04 22:29:12)>>>>>>>>>>>>>>>> Combine DF\n", "(2017-07-04 22:29:13)<<<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 22:29:13)>>>>>>>>>>>>>>>> Clean UOM\n", "(2017-07-04 22:29:33)<<<<<<<<<<<<<<<< --- (20.0s)\n", "(2017-07-04 22:29:33)<<<<<<<<<<<<<< --- (78.0s)\n", "(2017-07-04 22:29:33)>>>>>>>>>>>>>> Transforming... (7952939, 5)\n", "Data Loss (Extract > Transformed): ((7952939, 1), (7923711, 6), 29066L, 171, '0.3015% records')\n", "(2017-07-04 22:31:25)<<<<<<<<<<<<<< --- (112.0s)\n", "(2017-07-04 22:31:25)>>>>>>>>>>>>>> Cleaning... (7923711, 6)\n", "(2017-07-04 22:32:14)>>>>>>>>>>>>>>>> Nominal to OneHot\n", "(2017-07-04 22:32:14)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:32:14)>>>>>>>>>>>>>>>> Drop OOB data | (7922986, 6)\n", "(2017-07-04 22:32:18)>>>>>>>>>>>>>>>>>> heart rate, beats/min, 7923117\n", "(2017-07-04 22:33:29)<<<<<<<<<<<<<<<<<< --- (71.0s)\n", "(2017-07-04 22:33:29)>>>>>>>>>>>>>>>>>> heart rate, no_units, 31\n", "(2017-07-04 22:33:29)<<<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:33:29)<<<<<<<<<<<<<<<< --- (75.0s)\n", "Data Loss (Extract > Cleaned): ((7952939, 1), (7922986, 6), 29804L, 171, '0.3015% records')\n", "(2017-07-04 22:33:31)<<<<<<<<<<<<<< --- (126.0s)\n", "(2017-07-04 22:33:31)>>>>>>>>>>>>>> Filter & sort - (7922986, 6)\n", "(2017-07-04 22:33:35)<<<<<<<<<<<<<< --- (4.0s)\n", "(2017-07-04 22:33:35)>>>>>>>>>>>>>> Convert to dask - (7922986, 6)\n", "(2017-07-04 22:33:36)<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 22:33:36)>>>>>>>>>>>>>> Join to big DF\n", "(2017-07-04 22:33:36)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:33:36)<<<<<<<<<<<< --- (321.0s)\n", "(2017-07-04 22:33:36)>>>>>>>>>>>> BLOOD PRESSURE SYSTOLIC: 2/8\n", "(2017-07-04 22:33:36)>>>>>>>>>>>>>> Extracting...\n", "(2017-07-04 22:33:36)>>>>>>>>>>>>>>>> Extracting 14 items from chartevents\n", "(2017-07-04 22:35:06)<<<<<<<<<<<<<<<< --- (90.0s)\n", "(2017-07-04 22:35:06)>>>>>>>>>>>>>>>> Combine DF\n", "(2017-07-04 22:35:06)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:35:06)>>>>>>>>>>>>>>>> Clean UOM\n", "(2017-07-04 22:35:10)<<<<<<<<<<<<<<<< --- (4.0s)\n", "(2017-07-04 22:35:10)<<<<<<<<<<<<<< --- (94.0s)\n", "(2017-07-04 22:35:10)>>>>>>>>>>>>>> Transforming... (6374824, 5)\n", "Data Loss (Extract > Transformed): ((6374824, 1), (5974719, 15), 43236L, 174, '0.307% records')\n", "(2017-07-04 22:36:52)<<<<<<<<<<<<<< --- (102.0s)\n", "(2017-07-04 22:36:52)>>>>>>>>>>>>>> Cleaning... (5974719, 15)\n", "(2017-07-04 22:42:30)>>>>>>>>>>>>>>>> Nominal to OneHot\n", "(2017-07-04 22:42:31)<<<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 22:42:31)>>>>>>>>>>>>>>>> Drop OOB data | (5974186, 14)\n", "(2017-07-04 22:42:34)>>>>>>>>>>>>>>>>>> blood pressure systolic, mmHg, 6177439\n", "(2017-07-04 22:45:18)<<<<<<<<<<<<<<<<<< --- (164.0s)\n", "(2017-07-04 22:45:18)>>>>>>>>>>>>>>>>>> blood pressure systolic, cc/min, 153573\n", "(2017-07-04 22:45:19)<<<<<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 22:45:19)<<<<<<<<<<<<<<<< --- (168.0s)\n", "Data Loss (Extract > Cleaned): ((6374824, 1), (5974186, 14), 43848L, 174, '0.307% records')\n", "(2017-07-04 22:45:20)<<<<<<<<<<<<<< --- (508.0s)\n", "(2017-07-04 22:45:20)>>>>>>>>>>>>>> Filter & sort - (5974186, 14)\n", "(2017-07-04 22:45:23)<<<<<<<<<<<<<< --- (3.0s)\n", "(2017-07-04 22:45:23)>>>>>>>>>>>>>> Convert to dask - (5974186, 14)\n", "(2017-07-04 22:45:25)<<<<<<<<<<<<<< --- (2.0s)\n", "(2017-07-04 22:45:25)>>>>>>>>>>>>>> Join to big DF\n", "(2017-07-04 22:45:25)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:45:25)<<<<<<<<<<<< --- (709.0s)\n", "(2017-07-04 22:45:25)>>>>>>>>>>>> BLOOD PRESSURE DIASTOLIC: 3/8\n", "(2017-07-04 22:45:25)>>>>>>>>>>>>>> Extracting...\n", "(2017-07-04 22:45:25)>>>>>>>>>>>>>>>> Extracting 16 items from chartevents\n", "(2017-07-04 22:48:00)<<<<<<<<<<<<<<<< --- (155.0s)\n", "(2017-07-04 22:48:00)>>>>>>>>>>>>>>>> Combine DF\n", "(2017-07-04 22:48:00)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:48:00)>>>>>>>>>>>>>>>> Clean UOM\n", "(2017-07-04 22:48:06)<<<<<<<<<<<<<<<< --- (6.0s)\n", "(2017-07-04 22:48:06)<<<<<<<<<<<<<< --- (161.0s)\n", "(2017-07-04 22:48:06)>>>>>>>>>>>>>> Transforming... (6371282, 5)\n", "Data Loss (Extract > Transformed): ((6371282, 1), (5976845, 17), 24410L, 170, '0.2999% records')\n", "(2017-07-04 22:49:52)<<<<<<<<<<<<<< --- (106.0s)\n", "(2017-07-04 22:49:52)>>>>>>>>>>>>>> Cleaning... (5976845, 17)\n", "(2017-07-04 22:56:05)>>>>>>>>>>>>>>>> Nominal to OneHot\n", "(2017-07-04 22:56:06)<<<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 22:56:06)>>>>>>>>>>>>>>>> Drop OOB data | (5976313, 16)\n", "(2017-07-04 22:56:10)>>>>>>>>>>>>>>>>>> blood pressure diastolic, mmHg, 6194656\n", "(2017-07-04 22:59:39)<<<<<<<<<<<<<<<<<< --- (209.0s)\n", "(2017-07-04 22:59:39)>>>>>>>>>>>>>>>>>> blood pressure diastolic, cc/min, 151640\n", "(2017-07-04 22:59:39)<<<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:59:39)<<<<<<<<<<<<<<<< --- (213.0s)\n", "Data Loss (Extract > Cleaned): ((6371282, 1), (5976313, 16), 25238L, 170, '0.2999% records')\n", "(2017-07-04 22:59:41)<<<<<<<<<<<<<< --- (589.0s)\n", "(2017-07-04 22:59:41)>>>>>>>>>>>>>> Filter & sort - (5976313, 16)\n", "(2017-07-04 22:59:45)<<<<<<<<<<<<<< --- (4.0s)\n", "(2017-07-04 22:59:45)>>>>>>>>>>>>>> Convert to dask - (5976313, 16)\n", "(2017-07-04 22:59:46)<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 22:59:46)>>>>>>>>>>>>>> Join to big DF\n", "(2017-07-04 22:59:46)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 22:59:46)<<<<<<<<<<<< --- (861.0s)\n", "(2017-07-04 22:59:46)>>>>>>>>>>>> BLOOD PRESSURE MEAN: 4/8\n", "(2017-07-04 22:59:46)>>>>>>>>>>>>>> Extracting...\n", "(2017-07-04 22:59:46)>>>>>>>>>>>>>>>> Extracting 3 items from chartevents\n", "(2017-07-04 23:00:18)<<<<<<<<<<<<<<<< --- (32.0s)\n", "(2017-07-04 23:00:18)>>>>>>>>>>>>>>>> Combine DF\n", "(2017-07-04 23:00:19)<<<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 23:00:19)>>>>>>>>>>>>>>>> Clean UOM\n", "(2017-07-04 23:00:20)<<<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 23:00:20)<<<<<<<<<<<<<< --- (34.0s)\n", "(2017-07-04 23:00:20)>>>>>>>>>>>>>> Transforming... (2536271, 5)\n", "Data Loss (Extract > Transformed): ((2536271, 1), (2416029, 3), 0L, 0, '0.0% records')\n", "(2017-07-04 23:01:04)<<<<<<<<<<<<<< --- (44.0s)\n", "(2017-07-04 23:01:04)>>>>>>>>>>>>>> Cleaning... (2416029, 3)\n", "(2017-07-04 23:01:19)>>>>>>>>>>>>>>>> Nominal to OneHot\n", "(2017-07-04 23:01:19)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:01:19)>>>>>>>>>>>>>>>> Drop OOB data | (2415995, 3)\n", "(2017-07-04 23:01:20)>>>>>>>>>>>>>>>>>> blood pressure mean, mmHg, 2536236\n", "(2017-07-04 23:01:55)<<<<<<<<<<<<<<<<<< --- (35.0s)\n", "(2017-07-04 23:01:55)<<<<<<<<<<<<<<<< --- (36.0s)\n", "Data Loss (Extract > Cleaned): ((2536271, 1), (2415995, 3), 1873L, 0, '0.0% records')\n", "(2017-07-04 23:01:55)<<<<<<<<<<<<<< --- (51.0s)\n", "(2017-07-04 23:01:55)>>>>>>>>>>>>>> Filter & sort - (2415995, 3)\n", "(2017-07-04 23:01:56)<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 23:01:56)>>>>>>>>>>>>>> Convert to dask - (2415995, 3)\n", "(2017-07-04 23:01:57)<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 23:01:57)>>>>>>>>>>>>>> Join to big DF\n", "(2017-07-04 23:01:57)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:01:57)<<<<<<<<<<<< --- (131.0s)\n", "(2017-07-04 23:01:57)>>>>>>>>>>>> RESPIRATORY RATE: 5/8\n", "(2017-07-04 23:01:57)>>>>>>>>>>>>>> Extracting...\n", "(2017-07-04 23:01:57)>>>>>>>>>>>>>>>> Extracting 4 items from chartevents\n", "(2017-07-04 23:06:09)<<<<<<<<<<<<<<<< --- (252.0s)\n", "(2017-07-04 23:06:09)>>>>>>>>>>>>>>>> Combine DF\n", "(2017-07-04 23:06:09)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:06:09)>>>>>>>>>>>>>>>> Clean UOM\n", "(2017-07-04 23:06:24)<<<<<<<<<<<<<<<< --- (15.0s)\n", "(2017-07-04 23:06:24)<<<<<<<<<<<<<< --- (267.0s)\n", "(2017-07-04 23:06:24)>>>>>>>>>>>>>> Transforming... (7810019, 5)\n", "Data Loss (Extract > Transformed): ((7810019, 1), (7780717, 5), 28707L, 172, '0.3035% records')\n", "(2017-07-04 23:08:13)<<<<<<<<<<<<<< --- (109.0s)\n", "(2017-07-04 23:08:13)>>>>>>>>>>>>>> Cleaning... (7780717, 5)\n", "(2017-07-04 23:11:07)>>>>>>>>>>>>>>>> Nominal to OneHot\n", "(2017-07-04 23:11:07)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:11:07)>>>>>>>>>>>>>>>> Drop OOB data | (7780015, 4)\n", "(2017-07-04 23:11:11)>>>>>>>>>>>>>>>>>> respiratory rate, insp/min, 6108262\n", "(2017-07-04 23:12:08)<<<<<<<<<<<<<<<<<< --- (57.0s)\n", "(2017-07-04 23:12:08)>>>>>>>>>>>>>>>>>> respiratory rate, Breath, 1671901\n", "(2017-07-04 23:12:08)<<<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:12:08)>>>>>>>>>>>>>>>>>> respiratory rate, no_units, 2\n", "(2017-07-04 23:12:08)<<<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:12:08)<<<<<<<<<<<<<<<< --- (61.0s)\n", "Data Loss (Extract > Cleaned): ((7810019, 1), (7780015, 4), 29907L, 172, '0.3035% records')\n", "(2017-07-04 23:12:09)<<<<<<<<<<<<<< --- (236.0s)\n", "(2017-07-04 23:12:10)>>>>>>>>>>>>>> Filter & sort - (7780015, 4)\n", "(2017-07-04 23:12:14)<<<<<<<<<<<<<< --- (4.0s)\n", "(2017-07-04 23:12:14)>>>>>>>>>>>>>> Convert to dask - (7780015, 4)\n", "(2017-07-04 23:12:14)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:12:14)>>>>>>>>>>>>>> Join to big DF\n", "(2017-07-04 23:12:14)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:12:14)<<<<<<<<<<<< --- (617.0s)\n", "(2017-07-04 23:12:14)>>>>>>>>>>>> TEMPERATURE BODY: 6/8\n", "(2017-07-04 23:12:15)>>>>>>>>>>>>>> Extracting...\n", "(2017-07-04 23:12:15)>>>>>>>>>>>>>>>> Extracting 4 items from chartevents\n", "(2017-07-04 23:13:06)<<<<<<<<<<<<<<<< --- (51.0s)\n", "(2017-07-04 23:13:06)>>>>>>>>>>>>>>>> Combine DF\n", "(2017-07-04 23:13:06)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:13:06)>>>>>>>>>>>>>>>> Clean UOM\n", "(2017-07-04 23:13:11)<<<<<<<<<<<<<<<< --- (5.0s)\n", "(2017-07-04 23:13:11)<<<<<<<<<<<<<< --- (56.0s)\n", "(2017-07-04 23:13:11)>>>>>>>>>>>>>> Transforming... (1751447, 5)\n", "Data Loss (Extract > Transformed): ((1751447, 1), (1731875, 4), 16612L, 156, '0.3189% records')\n", "(2017-07-04 23:13:42)<<<<<<<<<<<<<< --- (31.0s)\n", "(2017-07-04 23:13:42)>>>>>>>>>>>>>> Cleaning... (1731875, 4)\n", "(2017-07-04 23:13:54)>>>>>>>>>>>>>>>> Nominal to OneHot\n", "(2017-07-04 23:13:54)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:13:54)>>>>>>>>>>>>>>>> Drop OOB data | (1731794, 4)\n", "(2017-07-04 23:13:55)>>>>>>>>>>>>>>>>>> temperature body, degF, 1734754\n", "(2017-07-04 23:14:24)<<<<<<<<<<<<<<<<<< --- (29.0s)\n", "(2017-07-04 23:14:24)<<<<<<<<<<<<<<<< --- (30.0s)\n", "Data Loss (Extract > Cleaned): ((1751447, 1), (1731794, 4), 17226L, 156, '0.3189% records')\n", "(2017-07-04 23:14:24)<<<<<<<<<<<<<< --- (42.0s)\n", "(2017-07-04 23:14:24)>>>>>>>>>>>>>> Filter & sort - (1731794, 4)\n", "(2017-07-04 23:14:25)<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 23:14:25)>>>>>>>>>>>>>> Convert to dask - (1731794, 4)\n", "(2017-07-04 23:14:25)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:14:25)>>>>>>>>>>>>>> Join to big DF\n", "(2017-07-04 23:14:25)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:14:25)<<<<<<<<<<<< --- (131.0s)\n", "(2017-07-04 23:14:25)>>>>>>>>>>>> OXYGEN SATURATION PULSE OXIMETRY: 7/8\n", "(2017-07-04 23:14:25)>>>>>>>>>>>>>> Extracting...\n", "(2017-07-04 23:14:25)>>>>>>>>>>>>>>>> Extracting 2 items from chartevents\n", "(2017-07-04 23:15:41)<<<<<<<<<<<<<<<< --- (76.0s)\n", "(2017-07-04 23:15:41)>>>>>>>>>>>>>>>> Combine DF\n", "(2017-07-04 23:15:41)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:15:41)>>>>>>>>>>>>>>>> Clean UOM\n", "(2017-07-04 23:15:58)<<<<<<<<<<<<<<<< --- (17.0s)\n", "(2017-07-04 23:15:58)<<<<<<<<<<<<<< --- (93.0s)\n", "(2017-07-04 23:15:58)>>>>>>>>>>>>>> Transforming... (6099827, 5)\n", "Data Loss (Extract > Transformed): ((6099827, 1), (6073540, 2), 26134L, 163, '0.3326% records')\n", "(2017-07-04 23:17:28)<<<<<<<<<<<<<< --- (90.0s)\n", "(2017-07-04 23:17:28)>>>>>>>>>>>>>> Cleaning... (6073540, 2)\n", "(2017-07-04 23:18:01)>>>>>>>>>>>>>>>> Nominal to OneHot\n", "(2017-07-04 23:18:01)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:18:01)>>>>>>>>>>>>>>>> Drop OOB data | (6073019, 2)\n", "(2017-07-04 23:18:03)>>>>>>>>>>>>>>>>>> oxygen saturation pulse oximetry, percent, 6073172\n", "(2017-07-04 23:18:54)<<<<<<<<<<<<<<<<<< --- (51.0s)\n", "(2017-07-04 23:18:54)<<<<<<<<<<<<<<<< --- (53.0s)\n", "Data Loss (Extract > Cleaned): ((6099827, 1), (6073019, 2), 26707L, 163, '0.3326% records')\n", "(2017-07-04 23:18:55)<<<<<<<<<<<<<< --- (87.0s)\n", "(2017-07-04 23:18:56)>>>>>>>>>>>>>> Filter & sort - (6073019, 2)\n", "(2017-07-04 23:18:58)<<<<<<<<<<<<<< --- (2.0s)\n", "(2017-07-04 23:18:58)>>>>>>>>>>>>>> Convert to dask - (6073019, 2)\n", "(2017-07-04 23:18:59)<<<<<<<<<<<<<< --- (1.0s)\n", "(2017-07-04 23:18:59)>>>>>>>>>>>>>> Join to big DF\n", "(2017-07-04 23:18:59)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:18:59)<<<<<<<<<<<< --- (274.0s)\n", "(2017-07-04 23:18:59)>>>>>>>>>>>> WEIGHT BODY: 8/8\n", "(2017-07-04 23:18:59)>>>>>>>>>>>>>> Extracting...\n", "(2017-07-04 23:18:59)>>>>>>>>>>>>>>>> Extracting 3 items from chartevents\n", "(2017-07-04 23:19:34)<<<<<<<<<<<<<<<< --- (35.0s)\n", "(2017-07-04 23:19:34)>>>>>>>>>>>>>>>> Combine DF\n", "(2017-07-04 23:19:34)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:19:34)>>>>>>>>>>>>>>>> Clean UOM\n", "(2017-07-04 23:19:34)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:19:34)<<<<<<<<<<<<<< --- (35.0s)\n", "(2017-07-04 23:19:34)>>>>>>>>>>>>>> Transforming... (95425, 5)\n", "Data Loss (Extract > Transformed): ((95425, 1), (94484, 3), 941L, 158, '0.4958% records')\n", "(2017-07-04 23:19:36)<<<<<<<<<<<<<< --- (2.0s)\n", "(2017-07-04 23:19:36)>>>>>>>>>>>>>> Cleaning... (94484, 3)\n", "(2017-07-04 23:19:37)>>>>>>>>>>>>>>>> Nominal to OneHot\n", "(2017-07-04 23:19:37)<<<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:19:37)>>>>>>>>>>>>>>>> Drop OOB data | (94457, 3)\n", "(2017-07-04 23:19:37)>>>>>>>>>>>>>>>>>> weight body, kg, 94457\n", "(2017-07-04 23:19:42)<<<<<<<<<<<<<<<<<< --- (5.0s)\n", "(2017-07-04 23:19:42)<<<<<<<<<<<<<<<< --- (5.0s)\n", "Data Loss (Extract > Cleaned): ((95425, 1), (94457, 3), 979L, 158, '0.4958% records')\n", "(2017-07-04 23:19:42)<<<<<<<<<<<<<< --- (6.0s)\n", "(2017-07-04 23:19:42)>>>>>>>>>>>>>> Filter & sort - (94457, 3)\n", "(2017-07-04 23:19:42)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:19:42)>>>>>>>>>>>>>> Convert to dask - (94457, 3)\n", "(2017-07-04 23:19:42)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:19:42)>>>>>>>>>>>>>> Join to big DF\n", "(2017-07-04 23:19:42)<<<<<<<<<<<<<< --- (0.0s)\n", "(2017-07-04 23:19:42)<<<<<<<<<<<< --- (43.0s)\n", "(2017-07-04 23:19:42)<<<<<<<<<< --- (3087.0s)\n", "(2017-07-04 23:19:42)<<<<<<<< --- (3087.0s)\n" ] } ], "source": [ "df_all = mimic.ETL(mimic.mimic_extractor('config/mimic_item_map.csv',data_dict),\n", " components,\n", " data_dict,\n", " transformers.same_index_aggregator(agg_func=lambda x:x.iloc[0]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Get all of the data we want and join into a single DF" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import utils\n", "import logger\n", "import transformers\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "def get_big_df(hdf5_fname,components):\n", " \n", " df_all = None\n", " logger.log('Make DF for {} components...\\n{}'.format(len(components),'\\n'.join(components)),new_level=True)\n", " for component in components:\n", " logger.log('{}: {}/{}'.format(component.upper(),components.index(component)+1,len(components)),new_level=True)\n", "\n", " logger.log('Opening...')\n", " df = utils.open_df(hdf5_fname,'cleaned/{}'.format(component)).sort_index(axis=1).sort_index()\n", " display(df.describe(include='all'))\n", "\n", " df_cleaned = transformers.remove_small_columns(threshold=5).fit_transform(df)\n", " \n", " display(df_cleaned.describe(include='all'))\n", "\n", " print utils.data_loss(df,df_cleaned)\n", " \n", " logger.log('Join {} to {}'.format(df_cleaned.shape, None if df_all is None else df_all.shape))\n", " if df_all is None: df_all = df_cleaned\n", " else : \n", " df_all = df_all.join(df_cleaned,how='outer')\n", " del df,df_cleaned\n", " \n", " logger.end_log_level()\n", " logger.end_log()\n", "\n", " return df_all" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import icu_data_defs\n", "data_dict = icu_data_defs.data_dictionary('config/data_definitions.xlsx')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "components = data_dict.get_panel_defintions(12).component.unique().tolist()\n", "components" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "#everything up to urine output\n", "df1 = get_big_df('data/mimic_data',components[:8])\n", "utils.save_df(df1,'data/mimic_data','cleaned/part1')\n", "\n", "#urine output and forward\n", "df2 = get_big_df('data/mimic_data',components[8:])\n", "utils.save_df(df2,'data/mimic_data','cleaned/part2')\n", "\n", "df_combined = df1.join(df2,how='outer')\n", "\n", "del df1,df2\n", "df_all.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "utils.save_df(df_combined,'data/mimic_data','cleaned/all')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start here" ] }, { "cell_type": "code", "execution_count": 70, "metadata": { "collapsed": false }, "outputs": [], "source": [ "\n", "\n", "def summable_filter(df):\n", " ureg = units.MedicalUreg()\n", " filter_func= lambda x: (ureg.is_volume(str(x[-2])) or ureg.is_mass(str(x[-2]))) and (x[0] != data_dict.labels.WEIGHT_BODY)\n", " return df.loc[:,df.columns.map(filter_func)]\n", "\n", "def lactate_filter_admissions(hdf5_fname)\n", "\n", "def make_lactate_labels(hdf5_fname,custom_cleaners,data_dict):\n", " df = utils.open_df(hdf5_fname,'cleaned/{}'.format(data_dict.labels.LACTATE))\n", " df_cleaned = custom_cleaners.transform(df)\n", " max_col_cleaner = transformers.max_col_only()\n", " df_cleaned = max_col_cleaner.transform(df_cleaned)\n", " df_cleaned.groupby(level=constants.column_names.ID).agg(lambda x: x.iloc[])\n", " \n", " \n", "\n", "basic_feature_tuples = [\n", " ('MEAN',features.segment_mean(),constants.ALL),\n", " ('STD',features.segment_std(),constants.ALL),\n", " ('COUNT',features.segment_count(),constants.ALL),\n", " ('LAST',features.segment_last(),constants.ALL),\n", " ('SUM',features.segment_sum(),{constants.CUSTOM_FILTER:summable_filter})\n", "]\n", "\n", "custom_cleaners = Pipeline([\n", " ('drop_oob_values',transformers.oob_value_remover(data_dict)),\n", " ('drop_small_columns',transformers.remove_small_columns(threshold=50)),\n", " ('combine_like_columns',transformers.combine_like_cols()),\n", "# ('quantitative_only',transformers.filter_var_type(var_types_to_keep)),\n", "# ('known_col_only',transformers.known_col_only()),\n", " ])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "\n", "\n", "lactate_labels = make_lactate_labels(hdf5_fname,custom_cleaners)\n", "\n", "df_features = mimic_features(hdf5_fname,'basic_all_before',labels,custom_cleaners)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python [Root]", "language": "python", "name": "Python [Root]" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.12" } }, "nbformat": 4, "nbformat_minor": 0 }