diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 3eb1e129..1069f790 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -85,9 +85,11 @@ make_table_name () { # load data into database find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL echo "done!" done && echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iii/buildmimic/postgres/Makefile b/mimic-iii/buildmimic/postgres/Makefile index 15b4c521..83a61ee5 100644 --- a/mimic-iii/buildmimic/postgres/Makefile +++ b/mimic-iii/buildmimic/postgres/Makefile @@ -104,7 +104,7 @@ mimic-build-gz: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -151,7 +151,7 @@ mimic-build: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -184,7 +184,7 @@ ifeq ("$(physionetuser)","") @echo 'Call the makefile again with physionetuser=' @echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org' else - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" endif mimic-demo-download: @@ -192,7 +192,7 @@ mimic-demo-download: @echo '-- Downloading MIMIC-III from PhysioNet --' @echo '------------------------------------------' @echo '' - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip, #but need to maintain compatibility with Windows, which baffling lacks these things diff --git a/mimic-iii/buildmimic/postgres/create_mimic_user.sh b/mimic-iii/buildmimic/postgres/create_mimic_user.sh index 9b333dcf..838ddc9e 100755 --- a/mimic-iii/buildmimic/postgres/create_mimic_user.sh +++ b/mimic-iii/buildmimic/postgres/create_mimic_user.sh @@ -20,6 +20,13 @@ else echo "User is set to '$MIMIC_USER'"; fi +# escape ' in password for SQL string literal +MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g") +# quote SQL identifiers (double any embedded ") +sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; } +MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER") +MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB") + PSQL='psql' # add in the host/port, if they were specified (not null, -n) @@ -58,11 +65,11 @@ fi if [ "$MIMIC_USER" != "postgres" ]; then # we need to create this user via postgres # use SUDO to login as postgres - $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';" + $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';" fi if [ "$MIMIC_DB" != "postgres" ]; then # drop and recreate the database - $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;" - $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;" -fi \ No newline at end of file + $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;" + $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;" +fi diff --git a/mimic-iii/concepts/make-concepts.sh b/mimic-iii/concepts/make-concepts.sh index 8b0707a1..ae75fad5 100644 --- a/mimic-iii/concepts/make-concepts.sh +++ b/mimic-iii/concepts/make-concepts.sh @@ -15,108 +15,108 @@ echo '' set -x echo 'Top level files..' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".code_status < code_status.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".echo_data < echo_data.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".code_status < "code_status.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".echo_data < "echo_data.sql" echo 'Running queries in 10 directories.' echo 'Directory 1: demographics' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".heightweight < demographics/heightweight.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".icustay_detail < demographics/icustay_detail.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".heightweight < "demographics/heightweight.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".icustay_detail < "demographics/icustay_detail.sql" # Durations (usually of treatments) echo 'Directory 2: durations' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_classification < durations/ventilation_classification.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_durations < durations/ventilation_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".crrt_durations < durations/crrt_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".adenosine_durations < durations/adenosine_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dobutamine_durations < durations/dobutamine_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dopamine_durations < durations/dopamine_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".epinephrine_durations < durations/epinephrine_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".isuprel_durations < durations/isuprel_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".milrinone_durations < durations/milrinone_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".norepinephrine_durations < durations/norepinephrine_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".phenylephrine_durations < durations/phenylephrine_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressin_durations < durations/vasopressin_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressor_durations < durations/vasopressor_durations.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".weight_durations < durations/weight_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_classification < "durations/ventilation_classification.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_durations < "durations/ventilation_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".crrt_durations < "durations/crrt_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".adenosine_durations < "durations/adenosine_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dobutamine_durations < "durations/dobutamine_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dopamine_durations < "durations/dopamine_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".epinephrine_durations < "durations/epinephrine_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".isuprel_durations < "durations/isuprel_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".milrinone_durations < "durations/milrinone_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".norepinephrine_durations < "durations/norepinephrine_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".phenylephrine_durations < "durations/phenylephrine_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressin_durations < "durations/vasopressin_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressor_durations < "durations/vasopressor_durations.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".weight_durations < "durations/weight_durations.sql" # dose queries for vasopressors -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dobutamine_dose < durations/dobutamine_dose.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dopamine_dose < durations/dopamine_dose.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".epinephrine_dose < durations/epinephrine_dose.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".norepinephrine_dose < durations/norepinephrine_dose.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".phenylephrine_dose < durations/phenylephrine_dose.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressin_dose < durations/vasopressin_dose.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dobutamine_dose < "durations/dobutamine_dose.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dopamine_dose < "durations/dopamine_dose.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".epinephrine_dose < "durations/epinephrine_dose.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".norepinephrine_dose < "durations/norepinephrine_dose.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".phenylephrine_dose < "durations/phenylephrine_dose.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressin_dose < "durations/vasopressin_dose.sql" # "pivoted" tables which have icustay_id / timestamp as the primary key echo 'Directory 3: pivoted tables' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_vital < pivot/pivoted_vital.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_uo < pivot/pivoted_uo.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_rrt < pivot/pivoted_rrt.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_lab < pivot/pivoted_lab.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_invasive_lines < pivot/pivoted_invasive_lines.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_icp < pivot/pivoted_icp.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_height < pivot/pivoted_height.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_gcs < pivot/pivoted_gcs.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_fio2 < pivot/pivoted_fio2.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_bg < pivot/pivoted_bg.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_vital < "pivot/pivoted_vital.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_uo < "pivot/pivoted_uo.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_rrt < "pivot/pivoted_rrt.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_lab < "pivot/pivoted_lab.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_invasive_lines < "pivot/pivoted_invasive_lines.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_icp < "pivot/pivoted_icp.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_height < "pivot/pivoted_height.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_gcs < "pivot/pivoted_gcs.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_fio2 < "pivot/pivoted_fio2.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_bg < "pivot/pivoted_bg.sql" # pivoted_bg_art must be run after pivoted_bg -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_bg_art < pivot/pivoted_bg_art.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_bg_art < "pivot/pivoted_bg_art.sql" # pivoted oasis depends on icustay_hours in demographics -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_oasis < pivot/pivoted_oasis.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_oasis < "pivot/pivoted_oasis.sql" # pivoted sofa depends on many above pivoted views, ventilation_durations, and dose queries -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_sofa < pivot/pivoted_sofa.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_sofa < "pivot/pivoted_sofa.sql" echo 'Directory 4: comorbidity' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_ahrq_v37 < comorbidity/elixhauser_ahrq_v37.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_ahrq_v37_no_drg < comorbidity/elixhauser_ahrq_v37_no_drg.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_quan < comorbidity/elixhauser_quan.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_score_ahrq < comorbidity/elixhauser_score_ahrq.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_score_quan < comorbidity/elixhauser_score_quan.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_ahrq_v37 < "comorbidity/elixhauser_ahrq_v37.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_ahrq_v37_no_drg < "comorbidity/elixhauser_ahrq_v37_no_drg.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_quan < "comorbidity/elixhauser_quan.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_score_ahrq < "comorbidity/elixhauser_score_ahrq.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_score_quan < "comorbidity/elixhauser_score_quan.sql" echo 'Directory 5: firstday' # data which is extracted from a patient's first ICU stay -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".blood_gas_first_day < firstday/blood_gas_first_day.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".blood_gas_first_day_arterial < firstday/blood_gas_first_day_arterial.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".gcs_first_day < firstday/gcs_first_day.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".labs_first_day < firstday/labs_first_day.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".rrt_first_day < firstday/rrt_first_day.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".urine_output_first_day < firstday/urine_output_first_day.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_first_day < firstday/ventilation_first_day.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vitals_first_day < firstday/vitals_first_day.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".weight_first_day < firstday/weight_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".blood_gas_first_day < "firstday/blood_gas_first_day.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".blood_gas_first_day_arterial < "firstday/blood_gas_first_day_arterial.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".gcs_first_day < "firstday/gcs_first_day.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".labs_first_day < "firstday/labs_first_day.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".rrt_first_day < "firstday/rrt_first_day.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".urine_output_first_day < "firstday/urine_output_first_day.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_first_day < "firstday/ventilation_first_day.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vitals_first_day < "firstday/vitals_first_day.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".weight_first_day < "firstday/weight_first_day.sql" echo 'Directory 6: fluid_balance' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".urine_output < fluid_balance/urine_output.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".urine_output < "fluid_balance/urine_output.sql" echo 'Directory 7: sepsis' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".angus < sepsis/angus.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".martin < sepsis/martin.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".explicit < sepsis/explicit.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".angus < "sepsis/angus.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".martin < "sepsis/martin.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".explicit < "sepsis/explicit.sql" # diagnosis mapping using CCS echo 'Directory 8: diagnosis' # load the ccs_multi_dx.csv.gz file into bq bq load --source_format=CSV "${TARGET_DATASET}".ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ccs_dx < diagnosis/ccs_dx.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ccs_dx < "diagnosis/ccs_dx.sql" # Organ failure scores echo 'Directory 9: organfailure' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_creatinine < organfailure/kdigo_creatinine.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_uo < organfailure/kdigo_uo.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages < organfailure/kdigo_stages.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages_7day < organfailure/kdigo_stages_7day.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages_48hr < organfailure/kdigo_stages_48hr.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".meld < organfailure/meld.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_creatinine < "organfailure/kdigo_creatinine.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_uo < "organfailure/kdigo_uo.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages < "organfailure/kdigo_stages.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages_7day < "organfailure/kdigo_stages_7day.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages_48hr < "organfailure/kdigo_stages_48hr.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".meld < "organfailure/meld.sql" # Severity of illness scores (requires many views from above) echo 'Directory 10: severityscores' -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".oasis < severityscores/oasis.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sofa < severityscores/sofa.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".saps < severityscores/saps.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sapsii < severityscores/sapsii.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".apsiii < severityscores/apsiii.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".lods < severityscores/lods.sql -bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sirs < severityscores/sirs.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".oasis < "severityscores/oasis.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sofa < "severityscores/sofa.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".saps < "severityscores/saps.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sapsii < "severityscores/sapsii.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".apsiii < "severityscores/apsiii.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".lods < "severityscores/lods.sql" +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sirs < "severityscores/sirs.sql" echo 'Finished creating concepts.' \ No newline at end of file diff --git a/mimic-iii/concepts_duckdb/duckdb.sql b/mimic-iii/concepts_duckdb/duckdb.sql index c74342eb..2635b2ad 100644 --- a/mimic-iii/concepts_duckdb/duckdb.sql +++ b/mimic-iii/concepts_duckdb/duckdb.sql @@ -93,10 +93,10 @@ .read pivot/pivoted_vital.sql .print 'pivot/pivoted_bg_art.sql' .read pivot/pivoted_bg_art.sql -.print 'pivot/pivoted_sofa.sql' -.read pivot/pivoted_sofa.sql .print 'pivot/pivoted_oasis.sql' .read pivot/pivoted_oasis.sql +.print 'pivot/pivoted_sofa.sql' +.read pivot/pivoted_sofa.sql -- comorbidity .print 'comorbidity/elixhauser_ahrq_v37.sql' diff --git a/mimic-iii/concepts_postgres/postgres-make-concepts.sql b/mimic-iii/concepts_postgres/postgres-make-concepts.sql index 40fe8841..318b3da1 100644 --- a/mimic-iii/concepts_postgres/postgres-make-concepts.sql +++ b/mimic-iii/concepts_postgres/postgres-make-concepts.sql @@ -61,8 +61,8 @@ SET search_path TO mimiciii_derived, mimiciii; \i pivot/pivoted_uo.sql \i pivot/pivoted_vital.sql \i pivot/pivoted_bg_art.sql -\i pivot/pivoted_sofa.sql \i pivot/pivoted_oasis.sql +\i pivot/pivoted_sofa.sql -- comorbidity \i comorbidity/elixhauser_ahrq_v37.sql diff --git a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh index 102e92d8..29a13107 100644 --- a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic4_ed.db)\ " } @@ -94,18 +94,29 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-ed # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (ed) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. " try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index 20e28313..4db098c1 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic4_note.db)\ " } @@ -97,18 +97,23 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-note # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (note) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .." OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -123,5 +128,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index 4ff9fc21..3213a8fd 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic4.db)\ " } @@ -102,18 +102,26 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both uncompressed (.csv) and gzip (.csv.gz). The old '*.csv???' +# pattern only matched names with exactly three chars after ".csv" (i.e. .gz), +# so plain .csv files were silently skipped while the script still reported success. +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in + case "$DIRNAME" in (hosp|icu) ;; # OK (*) continue; esac - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -128,5 +136,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected hosp/ and icu/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/mysql/validate_demo.sql b/mimic-iv/buildmimic/mysql/validate_demo.sql index 575e8413..ebf4f02d 100644 --- a/mimic-iv/buildmimic/mysql/validate_demo.sql +++ b/mimic-iv/buildmimic/mysql/validate_demo.sql @@ -31,14 +31,17 @@ FROM ( SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) exp @@ -64,14 +67,17 @@ INNER JOIN SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM procedureevents ) obs diff --git a/mimic-iv/buildmimic/postgres/validate_demo.sql b/mimic-iv/buildmimic/postgres/validate_demo.sql index 303dd98e..049497e5 100644 --- a/mimic-iv/buildmimic/postgres/validate_demo.sql +++ b/mimic-iv/buildmimic/postgres/validate_demo.sql @@ -22,14 +22,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) @@ -54,14 +57,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM mimiciv_hosp.poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM mimiciv_hosp.prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM mimiciv_hosp.procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM mimiciv_hosp.provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM mimiciv_hosp.services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM mimiciv_hosp.transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM mimiciv_icu.icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM mimiciv_icu.caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM mimiciv_icu.d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.procedureevents ) diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index d3369855..509b369b 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -23,8 +23,11 @@ def _unit(expression: exp.Expression, default: str = "DAY") -> str: # The logic is as follows: # * DAY -> difference of the two calendar dates (date subtraction = whole days) # * YEAR -> difference of the two calendar years +# * MONTH -> year*12 + month difference (calendar month boundaries) # * sub-day units -> truncate both operands to the unit (which makes the # elapsed seconds an exact multiple of the unit) then divide. +# WEEK is intentionally unsupported: BigQuery week starts (SUNDAY/ISO) do not +# match PostgreSQL DATE_TRUNC('week') Monday semantics. # https://cloud.google.com/bigquery/docs/reference/standard-sql/datetime_functions#datetime_diff _SECONDS_PER_UNIT = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600} @@ -39,6 +42,19 @@ def _datetime_diff_sql(self: Postgres.Generator, expression: exp.Expression) -> return f"(CAST({end} AS DATE) - CAST({start} AS DATE))" if unit == "YEAR": return f"CAST(EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start}) AS BIGINT)" + if unit == "MONTH": + # Calendar month boundaries (matches BigQuery DATETIME_DIFF MONTH). + return ( + "CAST((EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start})) * 12 " + "+ (EXTRACT(MONTH FROM {end}) - EXTRACT(MONTH FROM {start})) AS BIGINT)" + ).format(end=end, start=start) + if unit not in _SECONDS_PER_UNIT: + # WEEK (and WEEK(SUNDAY)/ISO) need BigQuery's week-start semantics; + # refuse rather than emit a silently wrong days/7 division. + raise ValueError( + f"Unsupported DATETIME_DIFF unit {unit!r}; " + "expected SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR" + ) lo = unit.lower() factor = _SECONDS_PER_UNIT[unit] diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index d4a65541..29924c18 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -113,7 +113,7 @@ def transpile_file( f"CREATE TABLE {derived_schema}{Path(source_file).stem} AS\n" ) + transpiled_query - with open(destination_file, "w") as write_file: + with open(destination_file, "w", encoding="utf-8") as write_file: write_file.write(transpiled_query) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 952240bc..8907c7ed 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -52,6 +52,9 @@ def t(bq: str, dialect: str) -> str: "SELECT (CAST(a.dischtime AS DATE) - CAST(a.admittime AS DATE)) FROM t AS a"), ("diff_year_pg", "SELECT DATETIME_DIFF(a.b, a.c, YEAR) FROM t a", "postgres", "SELECT CAST(EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c) AS BIGINT) FROM t AS a"), + ("diff_month_pg", "SELECT DATETIME_DIFF(a.b, a.c, MONTH) FROM t a", "postgres", + "SELECT CAST((EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c)) * 12 " + "+ (EXTRACT(MONTH FROM a.b) - EXTRACT(MONTH FROM a.c)) AS BIGINT) FROM t AS a"), # DuckDB handles DATETIME_DIFF natively (boundary count), operands swapped ("diff_hour_duckdb", "SELECT DATETIME_DIFF(a.outtime, a.intime, HOUR) FROM t a", "duckdb", "SELECT DATE_DIFF('HOUR', a.intime, a.outtime) FROM t AS a"), @@ -216,6 +219,7 @@ def test_mimic_iii_concept_transpiles_and_reparses(sql_file, dialect): ("2150-01-02 01:00:00", "2150-01-01 23:00:00", "DAY", 1), # crosses midnight ("2150-01-01 23:00:00", "2150-01-01 01:00:00", "DAY", 0), # same day ("2155-06-15 00:00:00", "2150-01-01 00:00:00", "YEAR", 5), + ("2150-04-01 00:00:00", "2150-01-15 00:00:00", "MONTH", 3), ("2150-01-01 00:01:00", "2150-01-01 00:00:59", "MINUTE", 1), ("2150-01-01 00:00:30", "2150-01-01 00:00:10", "SECOND", 20), ("2150-01-01 01:00:00", "2150-01-01 03:00:00", "HOUR", -2), # negative direction