fix: load paths, imports, transpile, and CI permissions

mimic-iii/buildmimic/duckdb/import_duckdb.sh

@@ -85,9 +85,11 @@ make_table_name () {
 # load data into database
 find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do
     make_table_name "$FILE"
-    echo "Loading $FILE .. \c"
+    # Escape single quotes for SQL string literal
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
+    printf "Loading %s .. " "$FILE"
     try duckdb "$OUTFILE" <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
     echo "done!"
 done && echo "Successfully finished loading data into $OUTFILE."

mimic-iii/buildmimic/postgres/Makefile

@@ -104,7 +104,7 @@ mimic-build-gz:
 	@echo '------------------'
 	@echo ''
 	@sleep 2
-	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir}
+	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)"
 	@echo ''
 	@echo '--------------------'
 	@echo '-- Adding indexes --'
@@ -151,7 +151,7 @@ mimic-build:
 	@echo '------------------'
 	@echo ''
 	@sleep 2
-	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR}
+	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)"
 	@echo ''
 	@echo '--------------------'
 	@echo '-- Adding indexes --'
@@ -184,15 +184,15 @@ ifeq ("$(physionetuser)","")
 	@echo 'Call the makefile again with physionetuser=<USERNAME>'
 	@echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org'
 else
-	wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)"
+	wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)"
 endif
 
 mimic-demo-download:
 	@echo '------------------------------------------'
 	@echo '-- Downloading MIMIC-III from PhysioNet --'
 	@echo '------------------------------------------'
 	@echo ''
-	wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)"
+	wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)"
 
 #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip,
 #but need to maintain compatibility with Windows, which baffling lacks these things

mimic-iii/buildmimic/postgres/create_mimic_user.sh

@@ -20,6 +20,13 @@ else
   echo "User is set to '$MIMIC_USER'";
 fi
 
+# escape ' in password for SQL string literal
+MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g")
+# quote SQL identifiers (double any embedded ")
+sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; }
+MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER")
+MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB")
+
 PSQL='psql'
 
 # add in the host/port, if they were specified (not null, -n)
@@ -58,11 +65,11 @@ fi
 if [ "$MIMIC_USER" != "postgres" ]; then
     # we need to create this user via postgres
     # use SUDO to login as postgres
-    $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';"
+    $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';"
 fi
 
 if [ "$MIMIC_DB" != "postgres" ]; then
   # drop and recreate the database
-  $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;"
-  $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;"
-fi
+  $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;"
+  $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;"
+fi

mimic-iii/concepts_duckdb/duckdb.sql

@@ -93,10 +93,10 @@
 .read pivot/pivoted_vital.sql
 .print 'pivot/pivoted_bg_art.sql'
 .read pivot/pivoted_bg_art.sql
-.print 'pivot/pivoted_sofa.sql'
-.read pivot/pivoted_sofa.sql
 .print 'pivot/pivoted_oasis.sql'
 .read pivot/pivoted_oasis.sql
+.print 'pivot/pivoted_sofa.sql'
+.read pivot/pivoted_sofa.sql
 
 -- comorbidity
 .print 'comorbidity/elixhauser_ahrq_v37.sql'

mimic-iii/concepts_postgres/postgres-make-concepts.sql

@@ -61,8 +61,8 @@ SET search_path TO mimiciii_derived, mimiciii;
 \i pivot/pivoted_uo.sql
 \i pivot/pivoted_vital.sql
 \i pivot/pivoted_bg_art.sql
-\i pivot/pivoted_sofa.sql
 \i pivot/pivoted_oasis.sql
+\i pivot/pivoted_sofa.sql
 
 -- comorbidity
 \i comorbidity/elixhauser_ahrq_v37.sql

mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh

@@ -29,7 +29,7 @@ usage () {
     die "
 USAGE: ./import_duckdb.sh mimic_data_dir [output_db]
 WHERE:
-    mimic_data_dir        directory that contains csv.gz or csv files
+    mimic_data_dir        directory that contains csv.tar.gz or csv files
     output_db: optional   filename for duckdb file (default: mimic4_ed.db)\
 "
 }
@@ -94,18 +94,29 @@ make_table_name () {
 
 
 # load data into database
-find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do
+# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ).
+LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed"
+trap 'rm -f "$LOAD_COUNT_FILE"' EXIT
+: > "$LOAD_COUNT_FILE"
+find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do
     make_table_name "$FILE"
 
     # skip directories which we do not expect in mimic-iv-ed
     # avoids syntax errors if mimic-iv in the same dir
-    case $DIRNAME in
+    case "$DIRNAME" in
       (ed) ;; # OK
       (*) continue;
     esac
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
     echo "Loading $FILE .. "
     try duckdb "$OUTFILE" <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
+    echo x >> "$LOAD_COUNT_FILE"
     echo "done!"
-done && echo "Successfully finished loading data into $OUTFILE."
+done || exit $?
+
+if [ ! -s "$LOAD_COUNT_FILE" ]; then
+    die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)."
+fi
+echo "Successfully finished loading data into $OUTFILE."

mimic-iv-note/buildmimic/duckdb/import_duckdb.sh

@@ -29,7 +29,7 @@ usage () {
     die "
 USAGE: ./import_duckdb.sh mimic_data_dir [output_db]
 WHERE:
-    mimic_data_dir        directory that contains csv.gz or csv files
+    mimic_data_dir        directory that contains csv.tar.gz or csv files
     output_db: optional   filename for duckdb file (default: mimic4_note.db)\
 "
 }
@@ -97,18 +97,23 @@ make_table_name () {
 
 
 # load data into database
-find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do
+# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ).
+LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed"
+trap 'rm -f "$LOAD_COUNT_FILE"' EXIT
+: > "$LOAD_COUNT_FILE"
+find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do
     make_table_name "$FILE"
 
     # skip directories which we do not expect in mimic-iv-note
     # avoids syntax errors if mimic-iv in the same dir
-    case $DIRNAME in
+    case "$DIRNAME" in
       (note) ;; # OK
       (*) continue;
     esac
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
     echo "Loading $FILE .."
     OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
     )
     # If the table is missing in the DB, we emit a warning and continue.
@@ -123,5 +128,11 @@ EOSQL
         yell "$OUTPUT"
         die "Exiting due to load error."
     fi
+    echo x >> "$LOAD_COUNT_FILE"
     echo "done!"
-done && echo "Successfully finished loading data into $OUTFILE."
+done || exit $?
+
+if [ ! -s "$LOAD_COUNT_FILE" ]; then
+    die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)."
+fi
+echo "Successfully finished loading data into $OUTFILE."

mimic-iv/buildmimic/duckdb/import_duckdb.sh

@@ -29,7 +29,7 @@ usage () {
     die "
 USAGE: ./import_duckdb.sh mimic_data_dir [output_db]
 WHERE:
-    mimic_data_dir        directory that contains csv.gz or csv files
+    mimic_data_dir        directory that contains csv.tar.gz or csv files
     output_db: optional   filename for duckdb file (default: mimic4.db)\
 "
 }
@@ -102,18 +102,26 @@ make_table_name () {
 
 
 # load data into database
-find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do
+# Match both uncompressed (.csv) and gzip (.csv.gz). The old '*.csv???'
+# pattern only matched names with exactly three chars after ".csv" (i.e. .gz),
+# so plain .csv files were silently skipped while the script still reported success.
+LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed"
+trap 'rm -f "$LOAD_COUNT_FILE"' EXIT
+: > "$LOAD_COUNT_FILE"
+find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do
     make_table_name "$FILE"
 
     # skip directories which we do not expect in mimic-iv
     # avoids syntax errors if mimic-iv-ed in the same dir
-    case $DIRNAME in
+    case "$DIRNAME" in
       (hosp|icu) ;; # OK
       (*) continue;
     esac
-    echo "Loading $FILE .. \c"
+    # Escape single quotes for SQL string literal
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
+    printf "Loading %s .. " "$FILE"
     OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
     )
     # If the table is missing in the DB, we emit a warning and continue.
@@ -128,5 +136,11 @@ EOSQL
         yell "$OUTPUT"
         die "Exiting due to load error."
     fi
+    echo x >> "$LOAD_COUNT_FILE"
     echo "done!"
-done && echo "Successfully finished loading data into $OUTFILE."
+done || exit $?
+
+if [ ! -s "$LOAD_COUNT_FILE" ]; then
+    die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected hosp/ and icu/)."
+fi
+echo "Successfully finished loading data into $OUTFILE."

mimic-iv/buildmimic/mysql/validate_demo.sql

@@ -31,14 +31,17 @@ FROM (
     SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL
     SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL
     SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL
+    SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL
     SELECT 'services' AS tbl, 319 AS row_count UNION ALL
     SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL
     -- icu data
     SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL
+    SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL
     SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL
     SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL
     SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL
     SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL
+    SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL
     SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL
     SELECT 'procedureevents' AS tbl, 1468 AS row_count
 ) exp
@@ -64,14 +67,17 @@ INNER JOIN
     SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM poe_detail UNION ALL
     SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM prescriptions UNION ALL
     SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM procedures_icd UNION ALL
+    SELECT 'provider' AS tbl, count(*) AS row_count FROM provider UNION ALL
     SELECT 'services' AS tbl, count(*) AS row_count FROM services UNION ALL
     SELECT 'transfers' AS tbl, count(*) AS row_count FROM transfers UNION ALL
     -- icu data
     SELECT 'icustays' AS tbl, count(*) AS row_count FROM icustays UNION ALL
+    SELECT 'caregiver' AS tbl, count(*) AS row_count FROM caregiver UNION ALL
     SELECT 'chartevents' AS tbl, count(*) AS row_count FROM chartevents UNION ALL
     SELECT 'd_items' AS tbl, count(*) AS row_count FROM d_items UNION ALL
     SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM datetimeevents UNION ALL
     SELECT 'inputevents' AS tbl, count(*) AS row_count FROM inputevents UNION ALL
+    SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM ingredientevents UNION ALL
     SELECT 'outputevents' AS tbl, count(*) AS row_count FROM outputevents UNION ALL
     SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM procedureevents
 ) obs

mimic-iv/buildmimic/postgres/validate_demo.sql

@@ -22,14 +22,17 @@ WITH expected AS
     SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL
     SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL
     SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL
+    SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL
     SELECT 'services' AS tbl, 319 AS row_count UNION ALL
     SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL
     -- icu data
     SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL
+    SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL
     SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL
     SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL
     SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL
     SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL
+    SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL
     SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL
     SELECT 'procedureevents' AS tbl, 1468 AS row_count
 )
@@ -54,14 +57,17 @@ WITH expected AS
     SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM mimiciv_hosp.poe_detail UNION ALL
     SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM mimiciv_hosp.prescriptions UNION ALL
     SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM mimiciv_hosp.procedures_icd UNION ALL
+    SELECT 'provider' AS tbl, count(*) AS row_count FROM mimiciv_hosp.provider UNION ALL
     SELECT 'services' AS tbl, count(*) AS row_count FROM mimiciv_hosp.services UNION ALL
     SELECT 'transfers' AS tbl, count(*) AS row_count FROM mimiciv_hosp.transfers UNION ALL
     -- icu data
     SELECT 'icustays' AS tbl, count(*) AS row_count FROM mimiciv_icu.icustays UNION ALL
+    SELECT 'caregiver' AS tbl, count(*) AS row_count FROM mimiciv_icu.caregiver UNION ALL
     SELECT 'chartevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.chartevents UNION ALL
     SELECT 'd_items' AS tbl, count(*) AS row_count FROM mimiciv_icu.d_items UNION ALL
     SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.datetimeevents UNION ALL
     SELECT 'inputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.inputevents UNION ALL
+    SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.ingredientevents UNION ALL
     SELECT 'outputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.outputevents UNION ALL
     SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.procedureevents
 )

src/mimic_utils/sqlglot_dialects/postgres.py

@@ -23,8 +23,11 @@ def _unit(expression: exp.Expression, default: str = "DAY") -> str:
 # The logic is as follows:
 #   * DAY  -> difference of the two calendar dates (date subtraction = whole days)
 #   * YEAR -> difference of the two calendar years
+#   * MONTH -> year*12 + month difference (calendar month boundaries)
 #   * sub-day units -> truncate both operands to the unit (which makes the
 #     elapsed seconds an exact multiple of the unit) then divide.
+# WEEK is intentionally unsupported: BigQuery week starts (SUNDAY/ISO) do not
+# match PostgreSQL DATE_TRUNC('week') Monday semantics.
 # https://cloud.google.com/bigquery/docs/reference/standard-sql/datetime_functions#datetime_diff
 _SECONDS_PER_UNIT = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600}
 
@@ -39,6 +42,19 @@ def _datetime_diff_sql(self: Postgres.Generator, expression: exp.Expression) ->
         return f"(CAST({end} AS DATE) - CAST({start} AS DATE))"
     if unit == "YEAR":
         return f"CAST(EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start}) AS BIGINT)"
+    if unit == "MONTH":
+        # Calendar month boundaries (matches BigQuery DATETIME_DIFF MONTH).
+        return (
+            "CAST((EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start})) * 12 "
+            "+ (EXTRACT(MONTH FROM {end}) - EXTRACT(MONTH FROM {start})) AS BIGINT)"
+        ).format(end=end, start=start)
+    if unit not in _SECONDS_PER_UNIT:
+        # WEEK (and WEEK(SUNDAY)/ISO) need BigQuery's week-start semantics;
+        # refuse rather than emit a silently wrong days/7 division.
+        raise ValueError(
+            f"Unsupported DATETIME_DIFF unit {unit!r}; "
+            "expected SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR"
+        )
 
     lo = unit.lower()
     factor = _SECONDS_PER_UNIT[unit]

src/mimic_utils/transpile.py

@@ -113,7 +113,7 @@ def transpile_file(
         f"CREATE TABLE {derived_schema}{Path(source_file).stem} AS\n"
     ) + transpiled_query
 
-    with open(destination_file, "w") as write_file:
+    with open(destination_file, "w", encoding="utf-8") as write_file:
         write_file.write(transpiled_query)
 
 

tests/test_transpile.py

@@ -52,6 +52,9 @@ def t(bq: str, dialect: str) -> str:
      "SELECT (CAST(a.dischtime AS DATE) - CAST(a.admittime AS DATE)) FROM t AS a"),
     ("diff_year_pg", "SELECT DATETIME_DIFF(a.b, a.c, YEAR) FROM t a", "postgres",
      "SELECT CAST(EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c) AS BIGINT) FROM t AS a"),
+    ("diff_month_pg", "SELECT DATETIME_DIFF(a.b, a.c, MONTH) FROM t a", "postgres",
+     "SELECT CAST((EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c)) * 12 "
+     "+ (EXTRACT(MONTH FROM a.b) - EXTRACT(MONTH FROM a.c)) AS BIGINT) FROM t AS a"),
     # DuckDB handles DATETIME_DIFF natively (boundary count), operands swapped
     ("diff_hour_duckdb", "SELECT DATETIME_DIFF(a.outtime, a.intime, HOUR) FROM t a", "duckdb",
      "SELECT DATE_DIFF('HOUR', a.intime, a.outtime) FROM t AS a"),
@@ -216,6 +219,7 @@ def test_mimic_iii_concept_transpiles_and_reparses(sql_file, dialect):
     ("2150-01-02 01:00:00", "2150-01-01 23:00:00", "DAY", 1),    # crosses midnight
     ("2150-01-01 23:00:00", "2150-01-01 01:00:00", "DAY", 0),    # same day
     ("2155-06-15 00:00:00", "2150-01-01 00:00:00", "YEAR", 5),
+    ("2150-04-01 00:00:00", "2150-01-15 00:00:00", "MONTH", 3),
     ("2150-01-01 00:01:00", "2150-01-01 00:00:59", "MINUTE", 1),
     ("2150-01-01 00:00:30", "2150-01-01 00:00:10", "SECOND", 20),
     ("2150-01-01 01:00:00", "2150-01-01 03:00:00", "HOUR", -2),  # negative direction