From 052ece1617fd0c9151fb52c460ff38afe8dd7633 Mon Sep 17 00:00:00 2001
From: Kristen Liu <kristenliu@verily.com>
Date: Mon, 17 Aug 2026 15:21:25 -0700
Subject: [PATCH 01/10] Replaced all instances of gsutil with gcloud

---
 README.md                                     | 17 +++---
 docs/code.md                                  |  4 +-
 docs/providers/README.md                      |  4 +-
 dsub/lib/param_util.py                        | 12 ++--
 dsub/lib/providers_util.py                    |  2 +-
 dsub/providers/google_batch.py                | 22 ++++----
 dsub/providers/google_utils.py                | 56 +++++++++----------
 dsub/providers/google_v2_base.py              |  8 +--
 dsub/providers/local.py                       | 20 +++----
 examples/custom_scripts/README.md             | 10 ++--
 examples/custom_scripts/submit_one.sh         |  2 +-
 examples/decompress/README.md                 |  6 +-
 examples/fastqc/README.md                     |  4 +-
 examples/samtools/README.md                   |  4 +-
 examples/split_process/README.md              |  4 +-
 examples/split_process/demo_split_process.sh  |  6 +-
 .../e2e_accelerator.google-batch.sh           |  2 +-
 .../e2e_accelerator.google-cls-v2.sh          |  2 +-
 .../e2e_accelerator_vpc_sc.google-batch.sh    |  2 +-
 test/integration/e2e_after.py                 |  2 +-
 test/integration/e2e_after.sh                 |  2 +-
 ...2e_block_external_network.google-cls-v2.sh |  9 ++-
 test/integration/e2e_cleanup.local.sh         |  4 +-
 test/integration/e2e_command_flag.sh          |  2 +-
 test/integration/e2e_env_list.py              |  2 +-
 test/integration/e2e_env_tasks.sh             |  2 +-
 test/integration/e2e_image.sh                 |  2 +-
 test/integration/e2e_input_wildcards.sh       |  6 +-
 test/integration/e2e_io_auto.sh               |  2 +-
 test/integration/e2e_io_gcs_tasks.sh          |  6 +-
 test/integration/e2e_io_mount_dir.local.sh    |  2 +-
 test/integration/e2e_io_recursive.sh          | 10 ++--
 test/integration/e2e_io_tasks.py              |  4 +-
 test/integration/e2e_logging_content.sh       |  4 +-
 test/integration/e2e_runtime.sh               |  2 +-
 test/integration/e2e_skip.sh                  |  8 +--
 test/integration/e2e_skip_tasks.sh            | 12 ++--
 test/integration/io_setup.sh                  |  6 +-
 test/integration/io_tasks_setup.sh            |  4 +-
 .../script_block_external_network.sh          |  4 +-
 test/integration/test_setup_e2e.py            |  8 +--
 test/integration/test_setup_e2e.sh            |  9 ++-
 test/integration/test_util.py                 |  8 +--
 test/integration/unit_skip.test-fails.sh      |  4 +-
 44 files changed, 155 insertions(+), 156 deletions(-)

diff --git a/README.md b/README.md
index 4df7b46..6eff054 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,7 @@ While not used directly by `dsub` for the `google-batch` provider, you are likel
 Cloud SDK](https://cloud.google.com/sdk/).
 
 If you will be using the `local` provider for faster job development,
-you *will* need to install the Google Cloud SDK, which uses `gsutil` to ensure
+you *will* need to install the Google Cloud SDK, which uses `gcloud storage` to ensure
 file operation semantics consistent with the Google `dsub` providers.
 
 1. [Install the Google Cloud SDK](https://cloud.google.com/sdk/)
@@ -182,10 +182,10 @@ The steps for getting started differ slightly as indicated in the steps below:
 
     The dsub logs and output files will be written to a bucket. Create a
     bucket using the [storage browser](https://console.cloud.google.com/storage/browser?project=)
-    or run the command-line utility [gsutil](https://cloud.google.com/storage/docs/gsutil),
+    or run the command-line utility [gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage),
     included in the Cloud SDK.
 
-        gsutil mb gs://my-bucket
+        gcloud storage buckets create gs://my-bucket
 
     Change `my-bucket` to a unique name that follows the
     [bucket-naming conventions](https://cloud.google.com/storage/docs/bucket-naming).
@@ -215,7 +215,7 @@ The steps for getting started differ slightly as indicated in the steps below:
 
 1. View the output file.
 
-        gsutil cat gs://my-bucket/output/out.txt
+        gcloud storage cat gs://my-bucket/output/out.txt
 
 ## Backend providers
 
@@ -351,9 +351,8 @@ by:
 
 To upload the files to Google Cloud Storage, you can use the
 [storage browser](https://console.cloud.google.com/storage/browser?project=) or
-[gsutil](https://cloud.google.com/storage/docs/gsutil). You can also run on data
-that’s public or shared with your service account, an email address that you
-can find in the [Google Cloud Console](https://console.cloud.google.com).
+[gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage). 
+You can also run on data that’s public or shared with your service account, an email address that you can find in the [Google Cloud Console](https://console.cloud.google.com).
 
 #### Files
 
@@ -728,7 +727,9 @@ of the service account will be `sa-name@project-id.iam.gserviceaccount.com`.
 
 2. Grant IAM access on buckets, etc. to the service account.
 
-        gsutil iam ch serviceAccount:sa-name@project-id.iam.gserviceaccount.com:roles/storage.objectAdmin gs://bucket-name
+        gcloud storage buckets add-iam-policy-binding gs://bucket-name
+            --member=serviceAccount:sa-name@project-id.iam.gserviceaccount.com
+            --role=roles/storage.objectAdmin
 
 3. Update your `dsub` command to include `--service-account`
 
diff --git a/docs/code.md b/docs/code.md
index 2e465ec..80c1d29 100644
--- a/docs/code.md
+++ b/docs/code.md
@@ -187,7 +187,7 @@ To run the driver script, first copy `script1.sh` and `script2.sh` to
 cloud storage:
 
 ```
-gsutil cp my-code/script1.sh my-code/script2.sh gs://MY-BUCKET/my-code/
+gcloud storage cp my-code/script1.sh my-code/script2.sh gs://MY-BUCKET/my-code/
 ```
 
 Then launch a dsub job:
@@ -205,7 +205,7 @@ Extending the previous example, you could copy `script1.sh` and `script2.sh`
 to cloud storage with:
 
 ```
-gsutil rsync -r my-code gs://MY-BUCKET/my-code/
+gcloud storage rsync -r my-code gs://MY-BUCKET/my-code/
 ```
 
 and then launch a `dsub` job with:
diff --git a/docs/providers/README.md b/docs/providers/README.md
index c482dc8..131caa7 100644
--- a/docs/providers/README.md
+++ b/docs/providers/README.md
@@ -131,8 +131,8 @@ copying output files.
 
 The copying of files is performed in the host environment, not inside the
 Docker container. This means that for copying to/from Google Cloud Storage,
-the host environment requires a copy of
-[gsutil](https://cloud.google.com/storage/docs/gsutil) to be installed.
+the host environment requires the
+[Google Cloud SDK](https://cloud.google.com/sdk/docs/install) to be installed.
 
 #### Container runtime environment
 
diff --git a/dsub/lib/param_util.py b/dsub/lib/param_util.py
index c5b9dba..ccddf48 100644
--- a/dsub/lib/param_util.py
+++ b/dsub/lib/param_util.py
@@ -827,23 +827,23 @@ def directory_fmt(directory):
 
   Multiple files copy, works as intended in all cases:
       $ touch a.txt b.txt
-      $ gsutil cp ./*.txt gs://mybucket/text_dest
-      $ gsutil ls gs://mybucket/text_dest/
+      $ gcloud storage cp ./*.txt gs://mybucket/text_dest
+      $ gcloud storage ls gs://mybucket/text_dest/
             0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/a.txt
             0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/b.txt
       TOTAL: 2 objects, 0 bytes (0 B)
 
   Single file copy fails to copy into a directory:
       $ touch 1.bam
-      $ gsutil cp ./*.bam gs://mybucket/bad_dest
-      $ gsutil ls gs://mybucket/bad_dest
+      $ gcloud storage cp ./*.bam gs://mybucket/bad_dest
+      $ gcloud storage ls gs://mybucket/bad_dest
                0  2017-07-19T21:46:16Z  gs://mybucket/bad_dest
       TOTAL: 1 objects, 0 bytes (0 B)
 
   Adding a trailing forward slash fixes this:
       $ touch my.sam
-      $ gsutil cp ./*.sam gs://mybucket/good_folder
-      $ gsutil ls gs://mybucket/good_folder
+      $ gcloud storage cp ./*.sam gs://mybucket/good_folder
+      $ gcloud storage ls gs://mybucket/good_folder
                0  2017-07-19T21:46:16Z  gs://mybucket/good_folder/my.sam
       TOTAL: 1 objects, 0 bytes (0 B)
 
diff --git a/dsub/lib/providers_util.py b/dsub/lib/providers_util.py
index 5700ea5..4f17669 100644
--- a/dsub/lib/providers_util.py
+++ b/dsub/lib/providers_util.py
@@ -20,7 +20,7 @@
 from .._dsub_version import DSUB_VERSION
 
 _LOCALIZE_COMMAND_MAP = {
-    job_model.P_GCS: 'gsutil -m rsync -r',
+    job_model.P_GCS: 'gcloud storage rsync -r',
     job_model.P_LOCAL: 'rsync -r',
 }
 
diff --git a/dsub/providers/google_batch.py b/dsub/providers/google_batch.py
index 73a35d4..b96596b 100644
--- a/dsub/providers/google_batch.py
+++ b/dsub/providers/google_batch.py
@@ -145,11 +145,11 @@ def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None
       "${{LOGGING_DIR}}/stderr.txt" \
       "{user_action}"
 
-  gsutil_cp "${{LOGGING_DIR}}/stdout.txt" "${{STDOUT_PATH}}" "text/plain" "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/stdout.txt" "${{STDOUT_PATH}}" "text/plain" "${{USER_PROJECT}}" &
   STDOUT_PID=$!
-  gsutil_cp "${{LOGGING_DIR}}/stderr.txt" "${{STDERR_PATH}}" "text/plain"  "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/stderr.txt" "${{STDERR_PATH}}" "text/plain"  "${{USER_PROJECT}}" &
   STDERR_PID=$!
-  gsutil_cp "${{LOGGING_DIR}}/log.txt" "${{LOGGING_PATH}}" "text/plain" "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/log.txt" "${{LOGGING_PATH}}" "text/plain" "${{USER_PROJECT}}" &
   LOG_PID=$!
 
   wait "${{STDOUT_PID}}"
@@ -168,7 +168,7 @@ def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None
   touch "${{LOGGING_DIR}}/.stop_logging"
 
   {log_msg_fn}
-  {gsutil_cp_fn}
+  {gcloud_cp_fn}
 
   {log_cp}
 """)
@@ -182,7 +182,7 @@ def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None
   readonly LOGGING_DIR="{logging_dir}"
 
   {log_msg_fn}
-  {gsutil_cp_fn}
+  {gcloud_cp_fn}
 
   # Make sure the logging work directory exists
   mkdir -p "${{LOGGING_DIR}}"
@@ -609,7 +609,7 @@ def _create_batch_request(
 
     continuous_logging_cmd = _CONTINUOUS_LOGGING_CMD.format(
         log_msg_fn=google_utils.LOG_MSG_FN,
-        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
+        gcloud_cp_fn=google_utils.GCLOUD_CP_FN,
         log_filter_var=_LOG_FILTER_VAR,
         log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
         python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
@@ -623,7 +623,7 @@ def _create_batch_request(
 
     logging_cmd = _FINAL_LOGGING_CMD.format(
         log_msg_fn=google_utils.LOG_MSG_FN,
-        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
+        gcloud_cp_fn=google_utils.GCLOUD_CP_FN,
         log_filter_var=_LOG_FILTER_VAR,
         log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
         python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
@@ -729,8 +729,8 @@ def _create_batch_request(
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.LOCALIZATION_LOOP,
                 ),
             ],
@@ -778,8 +778,8 @@ def _create_batch_request(
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.DELOCALIZATION_LOOP,
                 ),
             ],
diff --git a/dsub/providers/google_utils.py b/dsub/providers/google_utils.py
index cea1196..8b9862f 100644
--- a/dsub/providers/google_utils.py
+++ b/dsub/providers/google_utils.py
@@ -68,9 +68,9 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
 # pylint: enable=g-complex-comprehension
 
 
-# Action steps that interact with GCS need gsutil and Python.
+# Action steps that interact with GCS need gcloud and Python.
 # Use the 'slim' variant of the cloud-sdk image as it is much smaller.
-CLOUD_SDK_IMAGE = 'gcr.io/google.com/cloudsdktool/cloud-sdk:294.0.0-slim'
+CLOUD_SDK_IMAGE = 'gcr.io/google.com/cloudsdktool/cloud-sdk:499.0.0-slim'
 
 # Name of the data disk
 DATA_DISK_NAME = 'datadisk'
@@ -94,10 +94,10 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
   }
 """)
 
-# Define a bash function for "gsutil cp" to be used by the logging,
+# Define a bash function for "gcloud storage cp" to be used by the logging,
 # localization, and delocalization actions.
-GSUTIL_CP_FN = textwrap.dedent("""\
-  function gsutil_cp() {
+GCLOUD_CP_FN = textwrap.dedent("""\
+  function gcloud_cp() {
     local src="${1}"
     local dst="${2}"
     local content_type="${3}"
@@ -105,33 +105,33 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
 
     local headers=""
     if [[ -n "${content_type}" ]]; then
-      headers="-h Content-Type:${content_type}"
+      headers="--content-type=${content_type}"
     fi
 
     local user_project_flag=""
     if [[ -n "${user_project_name}" ]]; then
-      user_project_flag="-u ${user_project_name}"
+      user_project_flag="--billing-project=${user_project_name}"
     fi
 
     local attempt
     for ((attempt = 0; attempt < 4; attempt++)); do
-      log_info "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
-      if gsutil ${headers} ${user_project_flag} -mq cp "${src}" "${dst}"; then
+      log_info "gcloud storage cp ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
+      if gcloud storage cp ${headers} ${user_project_flag} "${src}" "${dst}"; then
         return
       fi
       if (( attempt < 3 )); then
-        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
-        log_warning "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+        log_warning "Sleeping 10s before the next attempt of failed gcloud command"
+        log_warning "gcloud storage cp ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
         sleep 10s
       fi
     done
 
-    log_error "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+    log_error "gcloud storage cp ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
     exit 1
   }
 """)
 
-LOG_CP_FN = GSUTIL_CP_FN + textwrap.dedent("""\
+LOG_CP_FN = GCLOUD_CP_FN + textwrap.dedent("""\
 
   function log_cp() {
     local src="${1}"
@@ -144,43 +144,43 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
       return
     fi
 
-    # Copy the log files to a local temporary location so that our "gsutil cp" is never
+    # Copy the log files to a local temporary location so that our "gcloud storage cp" is never
     # executed on a file that is changing.
 
     local tmp_path="${tmp}/$(basename ${src})"
     cp "${src}" "${tmp_path}"
 
-    gsutil_cp "${tmp_path}" "${dst}" "text/plain" "${user_project_name}"
+    gcloud_cp "${tmp_path}" "${dst}" "text/plain" "${user_project_name}"
   }
 """)
 
-# Define a bash function for "gsutil rsync" to be used by the logging,
+# Define a bash function for "gcloud rsync" to be used by the logging,
 # localization, and delocalization actions.
-GSUTIL_RSYNC_FN = textwrap.dedent("""\
-  function gsutil_rsync() {
+GCLOUD_RSYNC_FN = textwrap.dedent("""\
+  function gcloud_rsync() {
     local src="${1}"
     local dst="${2}"
     local user_project_name="${3}"
 
     local user_project_flag=""
     if [[ -n "${user_project_name}" ]]; then
-      user_project_flag="-u ${user_project_name}"
+      user_project_flag="--billing-project=${user_project_name}"
     fi
 
     local attempt
     for ((attempt = 0; attempt < 4; attempt++)); do
-      log_info "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
-      if gsutil ${user_project_flag} -mq rsync -r "${src}" "${dst}"; then
+      log_info "gcloud storage rsync -r ${user_project_flag} \"${src}\" \"${dst}\""
+      if gcloud storage rsync -r ${user_project_flag} "${src}" "${dst}"; then
         return
       fi
       if (( attempt < 3 )); then
-        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
-        log_warning "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+        log_warning "Sleeping 10s before the next attempt of failed gcloud command"
+        log_warning "gcloud storage rsync -r ${user_project_flag} \"${src}\" \"${dst}\""
         sleep 10s
       fi
     done
 
-    log_error "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+    log_error "gcloud storage rsync -r ${user_project_flag} \"${src}\" \"${dst}\""
     exit 1
   }
 """)
@@ -198,9 +198,9 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
 
     log_info "Localizing ${!INPUT_VAR}"
     if [[ "${!INPUT_RECURSIVE}" -eq "1" ]]; then
-      gsutil_rsync "${!INPUT_SRC}" "${!INPUT_DST}" "${USER_PROJECT}"
+      gcloud_rsync "${!INPUT_SRC}" "${!INPUT_DST}" "${USER_PROJECT}"
     else
-      gsutil_cp "${!INPUT_SRC}" "${!INPUT_DST}" "" "${USER_PROJECT}"
+      gcloud_cp "${!INPUT_SRC}" "${!INPUT_DST}" "" "${USER_PROJECT}"
     fi
   done
 """)
@@ -218,9 +218,9 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
 
     log_info "Delocalizing ${!OUTPUT_VAR}"
     if [[ "${!OUTPUT_RECURSIVE}" -eq "1" ]]; then
-      gsutil_rsync "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "${USER_PROJECT}"
+      gcloud_rsync "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "${USER_PROJECT}"
     else
-      gsutil_cp "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "" "${USER_PROJECT}"
+      gcloud_cp "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "" "${USER_PROJECT}"
     fi
   done
 """)
diff --git a/dsub/providers/google_v2_base.py b/dsub/providers/google_v2_base.py
index 4147653..781fc16 100644
--- a/dsub/providers/google_v2_base.py
+++ b/dsub/providers/google_v2_base.py
@@ -529,8 +529,8 @@ def _build_pipeline_request(self, task_view):
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.LOCALIZATION_LOOP,
                 ),
             ],
@@ -566,8 +566,8 @@ def _build_pipeline_request(self, task_view):
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.DELOCALIZATION_LOOP,
                 ),
             ],
diff --git a/dsub/providers/local.py b/dsub/providers/local.py
index b4b4814..0935afb 100644
--- a/dsub/providers/local.py
+++ b/dsub/providers/local.py
@@ -712,9 +712,9 @@ def _delocalize_logging_command(self, logging_path, user_project):
     elif logging_path.file_provider == job_model.P_GCS:
       mkdir_cmd = ''
       if user_project:
-        cp_cmd = 'gsutil -u {} -mq cp'.format(user_project)
+        cp_cmd = 'gcloud storage cp --billing-project={}'.format(user_project)
       else:
-        cp_cmd = 'gsutil -mq cp'
+        cp_cmd = 'gcloud storage cp'
     else:
       assert False
 
@@ -773,7 +773,7 @@ def _localize_inputs_recursive_command(self, task_dir, inputs):
     return '\n'.join(provider_commands)
 
   def _get_input_target_path(self, local_file_path):
-    """Returns a directory or file path to be the target for "gsutil cp".
+    """Returns a directory or file path to be the target for "gcloud storage cp".
 
     If the filename contains a wildcard, then the target path must
     be a directory in order to ensure consistency whether the source pattern
@@ -784,7 +784,7 @@ def _get_input_target_path(self, local_file_path):
       local_file_path: A full path terminating in a file or a file wildcard.
 
     Returns:
-      The path to use as the "gsutil cp" target.
+      The path to use as the "gcloud storage cp" target.
     """
 
     path, filename = os.path.split(local_file_path)
@@ -808,17 +808,17 @@ def _localize_inputs_command(self, task_dir, inputs, user_project):
 
       if i.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
         # The semantics that we expect here are implemented consistently in
-        # "gsutil cp", and are a bit different than "cp" when it comes to
+        # "gcloud storage cp", and are a bit different than "cp" when it comes to
         # wildcard handling, so use it for both local and GCS:
         #
         # - `cp path/* dest/` will error if "path" has subdirectories.
         # - `cp "path/*" "dest/"` will fail (it expects wildcard expansion
         #   to come from shell).
         if user_project:
-          command = 'gsutil -u %s -mq cp "%s" "%s"' % (
+          command = 'gcloud storage cp --billing-project=%s "%s" "%s"' % (
               user_project, source_file_path, dest_file_path)
         else:
-          command = 'gsutil -mq cp "%s" "%s"' % (source_file_path,
+          command = 'gcloud storage cp "%s" "%s"' % (source_file_path,
                                                  dest_file_path)
         commands.append(command)
 
@@ -865,13 +865,13 @@ def _delocalize_outputs_commands(self, task_dir, outputs, user_project):
       if o.file_provider == job_model.P_LOCAL:
         commands.append('mkdir -p "%s"' % dest_path)
 
-      # Use gsutil even for local files (explained in _localize_inputs_command).
+      # Use gcloud storage even for local files (explained in _localize_inputs_command).
       if o.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
         if user_project:
-          command = 'gsutil -u %s -mq cp "%s" "%s"' % (user_project, local_path,
+          command = 'gcloud storage cp --billing-project=%s "%s" "%s"' % (user_project, local_path,
                                                        dest_path)
         else:
-          command = 'gsutil -mq cp "%s" "%s"' % (local_path, dest_path)
+          command = 'gcloud storage cp "%s" "%s"' % (local_path, dest_path)
         commands.append(command)
 
     return '\n'.join(commands)
diff --git a/examples/custom_scripts/README.md b/examples/custom_scripts/README.md
index fd1d965..734f057 100644
--- a/examples/custom_scripts/README.md
+++ b/examples/custom_scripts/README.md
@@ -87,7 +87,7 @@ Because the `--wait` flag was set, `dsub` will block until the job completes.
 To list the output, use the command:
 
 ```
-gsutil ls gs://MY-BUCKET/get_vcf_sample_ids.sh/output
+gcloud storage ls gs://MY-BUCKET/get_vcf_sample_ids.sh/output
 ```
 
 Output should look like:
@@ -99,7 +99,7 @@ gs://MY-BUCKET/get_vcf_sample_ids.sh/output/sample_ids.txt
 To see the first few lines of the sample IDs file, run:
 
 ```
-gsutil cat gs://MY-BUCKET/get_vcf_sample_ids.sh/output/sample_ids.txt | head -n 5
+gcloud storage cat gs://MY-BUCKET/get_vcf_sample_ids.sh/output/sample_ids.txt | head -n 5
 ```
 
 Output should look like:
@@ -166,7 +166,7 @@ Because the `--wait` flag was set, `dsub` will block until the job completes.
 To list the output, use the command:
 
 ```
-gsutil ls gs://MY-BUCKET/get_vcf_sample_ids.py/output
+gcloud storage ls gs://MY-BUCKET/get_vcf_sample_ids.py/output
 ```
 
 Output should look like:
@@ -178,7 +178,7 @@ gs://MY-BUCKET/get_vcf_sample_ids.py/output/sample_ids.txt
 To see the first few lines of the sample IDs file, run:
 
 ```
-gsutil cat gs://MY-BUCKET/get_vcf_sample_ids.py/output/sample_ids.txt | head -n 5
+gcloud storage cat gs://MY-BUCKET/get_vcf_sample_ids.py/output/sample_ids.txt | head -n 5
 ```
 
 Output should look like:
@@ -265,7 +265,7 @@ When all tasks for the job have completed, `dsub` will exit.
 To list the output objects, use the command:
 
 ```
-gsutil ls gs://MY-BUCKET/get_vcf_sample_ids/output
+gcloud storage ls gs://MY-BUCKET/get_vcf_sample_ids/output
 ```
 
 Output should look like:
diff --git a/examples/custom_scripts/submit_one.sh b/examples/custom_scripts/submit_one.sh
index 2886bb9..7fb41b6 100755
--- a/examples/custom_scripts/submit_one.sh
+++ b/examples/custom_scripts/submit_one.sh
@@ -73,5 +73,5 @@ dsub \
 
 # Check output
 echo "Check the head of the output file:"
-2>&1 gsutil cat "${OUTPUT_FILE}" | head
+2>&1 gcloud storage cat "${OUTPUT_FILE}" | head
 
diff --git a/examples/decompress/README.md b/examples/decompress/README.md
index 7fa2f81..c319cf7 100644
--- a/examples/decompress/README.md
+++ b/examples/decompress/README.md
@@ -65,7 +65,7 @@ Because the `--wait` flag was set, `dsub` will block until the job completes.
 To list the output, use the command:
 
 ```
-gsutil ls gs://MY-BUCKET/decompress_one/output
+gcloud storage ls gs://MY-BUCKET/decompress_one/output
 ```
 
 Output should look like:
@@ -77,7 +77,7 @@ gs://MY-BUCKET/decompress_one/output/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vc
 To see the first few lines of the decompressed file, run:
 
 ```
-gsutil cat gs://MY-BUCKET/decompress_one/output/*.vcf | head -n 5
+gcloud storage cat gs://MY-BUCKET/decompress_one/output/*.vcf | head -n 5
 ```
 
 Output should look like:
@@ -153,7 +153,7 @@ when all tasks for the job have completed, `dsub` will exit.
 To list the output objects, use the command:
 
 ```
-gsutil ls gs://MY-BUCKET/decompress_list/output
+gcloud storage ls gs://MY-BUCKET/decompress_list/output
 ```
 
 Output should look like:
diff --git a/examples/fastqc/README.md b/examples/fastqc/README.md
index ef02fb7..c9ddea0 100644
--- a/examples/fastqc/README.md
+++ b/examples/fastqc/README.md
@@ -113,7 +113,7 @@ Because the `--wait` flag was set, `dsub` will block until the job completes.
 To list the output, use the command:
 
 ```
-gsutil ls -l gs://MY-BUCKET/fastqc/submit_one/output
+gcloud storage ls -l gs://MY-BUCKET/fastqc/submit_one/output
 ```
 
 Output should look like:
@@ -189,7 +189,7 @@ when all tasks for the job have completed, `dsub` will exit.
 To list the output objects, use the command:
 
 ```
-gsutil ls -l gs://MY-BUCKET/fastqc/submit_list/output
+gcloud storage ls -l gs://MY-BUCKET/fastqc/submit_list/output
 ```
 
 Output should look like:
diff --git a/examples/samtools/README.md b/examples/samtools/README.md
index 5f968ad..eeb7beb 100644
--- a/examples/samtools/README.md
+++ b/examples/samtools/README.md
@@ -77,7 +77,7 @@ Because the `--wait` flag was set, `dsub` will block until the job completes.
 To list the output, use the command:
 
 ```
-gsutil ls -l gs://MY-BUCKET/samtools/submit_one/output
+gcloud storage ls -l gs://MY-BUCKET/samtools/submit_one/output
 ```
 
 Output should look like:
@@ -155,7 +155,7 @@ when all tasks for the job have completed, `dsub` will exit.
 To list the output objects, use the command:
 
 ```
-gsutil ls -l gs://MY-BUCKET/samtools/submit_list/output
+gcloud storage ls -l gs://MY-BUCKET/samtools/submit_list/output
 ```
 
 Output should look like:
diff --git a/examples/split_process/README.md b/examples/split_process/README.md
index ad71d35..32fd1a5 100644
--- a/examples/split_process/README.md
+++ b/examples/split_process/README.md
@@ -44,6 +44,6 @@ rm "${WORKSPACE}/temp/*"
 ```
 WORKSPACE=gs://mybucket/someprefix
 ./demo_split_process.sh input.txt "${WORKSPACE}"
-gsutil ls "${WORKSPACE}/output/"
-gsutil rm "${WORKSPACE}/temp/*"
+gcloud storage ls "${WORKSPACE}/output/"
+gcloud storage rm "${WORKSPACE}/temp/*"
 ```
diff --git a/examples/split_process/demo_split_process.sh b/examples/split_process/demo_split_process.sh
index e33a1af..012745d 100755
--- a/examples/split_process/demo_split_process.sh
+++ b/examples/split_process/demo_split_process.sh
@@ -11,10 +11,10 @@
 # example:
 #   WORKSPACE=gs://mybucket/someprefix
 #   ./demo_split_process.sh input.txt "${WORKSPACE}"
-#   gsutil ls "${WORKSPACE}/output/"
-#   gsutil rm "${WORKSPACE}/temp/*"
+#   gcloud storage ls "${WORKSPACE}/output/"
+#   gcloud storage rm "${WORKSPACE}/temp/*"
 #
-# You need dsub, docker, and gsutil installed.
+# You need dsub, docker, and gcloud installed.
 # Change WORKSPACE to point to a bucket you have write permission to.
 #
 # Since this uses the local provider, you can set WORKSPACE to a local path,
diff --git a/test/integration/e2e_accelerator.google-batch.sh b/test/integration/e2e_accelerator.google-batch.sh
index 9a9a1c6..4f6ae4b 100755
--- a/test/integration/e2e_accelerator.google-batch.sh
+++ b/test/integration/e2e_accelerator.google-batch.sh
@@ -87,7 +87,7 @@ echo
 echo "Checking GPU detection output..."
 
 # Check that GPU was detected and accessible
-RESULT="$(gsutil cat "${STDOUT_LOG}")"
+RESULT="$(gcloud storage cat "${STDOUT_LOG}")"
 
 # Validate GPU hardware was detected
 if ! echo "${RESULT}" | grep -qi "Tesla T4"; then
diff --git a/test/integration/e2e_accelerator.google-cls-v2.sh b/test/integration/e2e_accelerator.google-cls-v2.sh
index 0960efc..3c13a56 100755
--- a/test/integration/e2e_accelerator.google-cls-v2.sh
+++ b/test/integration/e2e_accelerator.google-cls-v2.sh
@@ -45,7 +45,7 @@ echo
 echo "Checking output..."
 
 # Check the results
-RESULT="$(gsutil cat "${STDOUT_LOG}")"
+RESULT="$(gcloud storage cat "${STDOUT_LOG}")"
 if ! echo "${RESULT}" | grep -qi "GPU Memory"; then
   1>&2 echo "GPU Memory not found in the dsub output!"
   1>&2 echo "${RESULT}"
diff --git a/test/integration/e2e_accelerator_vpc_sc.google-batch.sh b/test/integration/e2e_accelerator_vpc_sc.google-batch.sh
index b798992..496c6f7 100755
--- a/test/integration/e2e_accelerator_vpc_sc.google-batch.sh
+++ b/test/integration/e2e_accelerator_vpc_sc.google-batch.sh
@@ -155,7 +155,7 @@ echo
 echo "Checking GPU detection output..."
 
 # Check that GPU was detected and accessible
-RESULT="$(gsutil cat "${STDOUT_LOG}")"
+RESULT="$(gcloud storage cat "${STDOUT_LOG}")"
 
 # Validate GPU hardware was detected
 if ! echo "${RESULT}" | grep -qi "Tesla T4"; then
diff --git a/test/integration/e2e_after.py b/test/integration/e2e_after.py
index 719ede7..a5e09ef 100644
--- a/test/integration/e2e_after.py
+++ b/test/integration/e2e_after.py
@@ -56,7 +56,7 @@
 
 print('\nChecking output...')
 
-RESULT = test_util.gsutil_cat(TEST_FILE_PATH_2)
+RESULT = test_util.gcloud_cat(TEST_FILE_PATH_2)
 if 'hello world' not in RESULT:
   print('Output file does not match expected', file=sys.stderr)
   sys.exit(1)
diff --git a/test/integration/e2e_after.sh b/test/integration/e2e_after.sh
index 9b9c6b7..bb414d6 100755
--- a/test/integration/e2e_after.sh
+++ b/test/integration/e2e_after.sh
@@ -55,7 +55,7 @@ fi
 echo
 echo "Checking output..."
 
-readonly RESULT="$(gsutil cat "${TEST_FILE_PATH_2}")"
+readonly RESULT="$(gcloud storage cat "${TEST_FILE_PATH_2}")"
 if [[ "${RESULT}" != "hello world" ]]; then
   echo "Output file does not match expected"
   echo "Expected: hello world"
diff --git a/test/integration/e2e_block_external_network.google-cls-v2.sh b/test/integration/e2e_block_external_network.google-cls-v2.sh
index feb4889..157549a 100755
--- a/test/integration/e2e_block_external_network.google-cls-v2.sh
+++ b/test/integration/e2e_block_external_network.google-cls-v2.sh
@@ -31,9 +31,8 @@ echo "Launching pipeline..."
 
 set +o errexit
 
-# Run gsutil with Boto:num_retries=0 option. Otherwise, gsutil will retry up to
-# 24 times due to the network error
-# https://stackoverflow.com/questions/44459685/sql-server-agent-job-and-gsutil
+# Run gcloud storage with --no-user-output-enabled and max retries set to 0.
+# Otherwise, gcloud storage will retry due to the network error
 JOB_ID="$(run_dsub \
   --image 'gcr.io/google.com/cloudsdktool/cloud-sdk:327.0.0-slim' \
   --block-external-network \
@@ -54,9 +53,9 @@ readonly ATTEMPT_1_STDERR_LOG="$(dirname "${LOGGING}")/${TEST_NAME}.1-stderr.log
 readonly ATTEMPT_2_STDERR_LOG="$(dirname "${LOGGING}")/${TEST_NAME}.2-stderr.log"
 
 for STDERR_LOG_FILE in "${ATTEMPT_1_STDERR_LOG}" "${ATTEMPT_2_STDERR_LOG}" ; do
-  RESULT="$(gsutil cat "${STDERR_LOG_FILE}")"
+  RESULT="$(gcloud storage cat "${STDERR_LOG_FILE}")"
   if ! echo "${RESULT}" | grep -qi "Unable to find the server at storage.googleapis.com"; then
-    1>&2 echo "Network error from gsutil not found in the dsub stderr log!"
+    1>&2 echo "Network error from gcloud not found in the dsub stderr log!"
     1>&2 echo "${RESULT}"
     exit 1
   fi
diff --git a/test/integration/e2e_cleanup.local.sh b/test/integration/e2e_cleanup.local.sh
index 1677cf9..5064299 100755
--- a/test/integration/e2e_cleanup.local.sh
+++ b/test/integration/e2e_cleanup.local.sh
@@ -26,7 +26,7 @@ readonly SCRIPT_DIR="$(dirname "${0}")"
 source "${SCRIPT_DIR}/test_setup_e2e.sh"
 
 # Stage a test file.
-date | gsutil cp - "${INPUTS}/recursive/deep/today.txt"
+date | gcloud storage cp - "${INPUTS}/recursive/deep/today.txt"
 
 readonly TGT_1="${OUTPUTS}/testfile_1.txt"
 readonly TGT_2="${OUTPUTS}/testfile_2.txt"
@@ -82,7 +82,7 @@ JOB_ID=$(run_dsub \
 check_jobid "${JOB_ID}"
 
 for out in "${TGT_1}" "${TGT_2}" "${TGT_3}"; do
-  if ! gsutil ls "${out}" > /dev/null; then
+  if ! gcloud storage ls "${out}" > /dev/null; then
     echo "Missing output: ${out}"
     exit 1
   fi
diff --git a/test/integration/e2e_command_flag.sh b/test/integration/e2e_command_flag.sh
index da66abc..bc03140 100755
--- a/test/integration/e2e_command_flag.sh
+++ b/test/integration/e2e_command_flag.sh
@@ -57,7 +57,7 @@ VAR5=VAL5
 EOF
 )
 
-readonly RESULT="$(gsutil cat "${STDOUT_LOG}")"
+readonly RESULT="$(gcloud storage cat "${STDOUT_LOG}")"
 if ! diff <(echo "${RESULT_EXPECTED}") <(echo "${RESULT}"); then
   echo "Output file does not match expected"
   exit 1
diff --git a/test/integration/e2e_env_list.py b/test/integration/e2e_env_list.py
index 3099526..fed80d3 100644
--- a/test/integration/e2e_env_list.py
+++ b/test/integration/e2e_env_list.py
@@ -73,7 +73,7 @@
 VAR5=VAL5
 """.lstrip()
 
-RESULT = test_util.gsutil_cat(test.STDOUT_LOG)
+RESULT = test_util.gcloud_cat(test.STDOUT_LOG)
 if not test_util.diff(RESULT_EXPECTED, RESULT):
   print('Output file does not match expected')
   sys.exit(1)
diff --git a/test/integration/e2e_env_tasks.sh b/test/integration/e2e_env_tasks.sh
index 512a287..c1c559e 100755
--- a/test/integration/e2e_env_tasks.sh
+++ b/test/integration/e2e_env_tasks.sh
@@ -101,7 +101,7 @@ for ((TASK_ID=1; TASK_ID <= NUM_TASKS; TASK_ID++)); do
     sed -e 's#^ *##'
   )"
 
-  RESULT="$(gsutil cat "${LOGGING}.${TASK_ID}-stdout.log")"
+  RESULT="$(gcloud storage cat "${LOGGING}.${TASK_ID}-stdout.log")"
   if ! diff <(echo "${RESULT_EXPECTED}") <(echo "${RESULT}"); then
     echo "Output file does not match expected"
     exit 1
diff --git a/test/integration/e2e_image.sh b/test/integration/e2e_image.sh
index 6f38c84..f085307 100755
--- a/test/integration/e2e_image.sh
+++ b/test/integration/e2e_image.sh
@@ -58,7 +58,7 @@ for image in ${IMAGE_ARRAY[@]}; do
   echo "Checking output..."
 
   # Check the results
-  RESULT="$(gsutil cat "${STDOUT_LOG}")"
+  RESULT="$(gcloud storage cat "${STDOUT_LOG}")"
   if ! diff <(echo "${RESULT_EXPECTED}") <(echo "${RESULT}"); then
     echo "Output file does not match expected"
     exit 1
diff --git a/test/integration/e2e_input_wildcards.sh b/test/integration/e2e_input_wildcards.sh
index d7c8d04..638ea14 100755
--- a/test/integration/e2e_input_wildcards.sh
+++ b/test/integration/e2e_input_wildcards.sh
@@ -43,7 +43,7 @@ function exit_handler() {
   # Only cleanup on success
   if [[ "${code}" -eq 0 ]]; then
     rm -rf "${TEST_TMP}"
-    gsutil -mq rm "${INPUTS}/**"
+    gcloud storage rm "${INPUTS}/**"
   fi
 
   return "${code}"
@@ -64,7 +64,7 @@ for INPUT_DIR in "${INPUT_BASIC}" "${INPUT_WITH_SPACE}"; do
   done
 done
 
-gsutil -m rsync -r "${INPUT_ROOT}" "${INPUTS}/"
+gcloud storage rsync -r "${INPUT_ROOT}" "${INPUTS}/"
 
 echo "Launching pipeline..."
 
@@ -94,7 +94,7 @@ FILE_NAME=file.3.txt
 EOF
 )
 
-readonly RESULT="$(gsutil cat "${STDOUT_LOG}")"
+readonly RESULT="$(gcloud storage cat "${STDOUT_LOG}")"
 if ! diff <(echo "${RESULT_EXPECTED}") <(echo "${RESULT}"); then
   echo "Output file does not match expected"
   exit 1
diff --git a/test/integration/e2e_io_auto.sh b/test/integration/e2e_io_auto.sh
index 35b6f94..813b841 100755
--- a/test/integration/e2e_io_auto.sh
+++ b/test/integration/e2e_io_auto.sh
@@ -64,7 +64,7 @@ readonly EXPECTED_FS_OUTPUT_ENTRIES=(
 )
 
 # Get the results- "env" and "find" output is bounded by "BEGIN" and "END"
-readonly RESULT=$(gsutil cat "${STDOUT_LOG}")
+readonly RESULT=$(gcloud storage cat "${STDOUT_LOG}")
 readonly ENV=$(echo "${RESULT}" | sed -n '/^BEGIN: env$/,/^END: env$/p')
 readonly FIND=$(echo "${RESULT}" | sed -n '/^BEGIN: find$/,/^END: find$/p')
 
diff --git a/test/integration/e2e_io_gcs_tasks.sh b/test/integration/e2e_io_gcs_tasks.sh
index 43948f6..a3a4515 100755
--- a/test/integration/e2e_io_gcs_tasks.sh
+++ b/test/integration/e2e_io_gcs_tasks.sh
@@ -44,11 +44,11 @@ io_tasks_setup::write_tasks_file
 
 # Copy the script to GCS to test loading the script remotely
 echo "Copying script to ${DSUB_PARAMS}"
-gsutil cp "${SCRIPT_DIR}/script_io_test.sh" "${DSUB_PARAMS}/"
+gcloud storage cp "${SCRIPT_DIR}/script_io_test.sh" "${DSUB_PARAMS}/"
 
 # Copy the TASKS_FILE to GCS to test loading the tasks file remotely
 echo "Copying tasks file to ${DSUB_PARAMS}"
-gsutil cp "${TASKS_FILE}" "${DSUB_PARAMS}/"
+gcloud storage cp "${TASKS_FILE}" "${DSUB_PARAMS}/"
 
 echo "Launching pipelines..."
 
@@ -62,4 +62,4 @@ io_tasks_setup::check_output
 io_tasks_setup::check_dstat "${JOB_ID}"
 
 # Clean up what we uploaded after the test is done.
-gsutil rm "${DSUB_PARAMS}"/**
+gcloud storage rm "${DSUB_PARAMS}"/**
diff --git a/test/integration/e2e_io_mount_dir.local.sh b/test/integration/e2e_io_mount_dir.local.sh
index 746a2d0..3fb8779 100755
--- a/test/integration/e2e_io_mount_dir.local.sh
+++ b/test/integration/e2e_io_mount_dir.local.sh
@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset
 
 # This test verifies that mounting a local directory works. The test will copy
-# input files via gsutil to the local disk.
+# input files via gcloud to the local disk.
 #
 # The actual operation performed here is to download a BAM and compute
 # the md5, writing it to <filename>.bam.md5.
diff --git a/test/integration/e2e_io_recursive.sh b/test/integration/e2e_io_recursive.sh
index 3bca94c..7163281 100755
--- a/test/integration/e2e_io_recursive.sh
+++ b/test/integration/e2e_io_recursive.sh
@@ -64,7 +64,7 @@ echo "Setting up test inputs"
 echo "Setting up pipeline input..."
 build_recursive_files "${INPUT_DEEP}" "${INPUT_SHALLOW}"
 
-gsutil -m rsync -r "${LOCAL_INPUTS}" "${INPUTS}/"
+gcloud storage rsync -r "${LOCAL_INPUTS}" "${INPUTS}/"
 
 echo "Launching pipeline..."
 
@@ -87,7 +87,7 @@ setup_expected_fs_output_entries "${DOCKER_GCS_OUTPUTS}"
 setup_expected_remote_output_entries "${OUTPUTS}"
 
 # Verify in the stdout file that the expected directories were written
-readonly RESULT=$(gsutil cat "${STDOUT_LOG}")
+readonly RESULT=$(gcloud storage cat "${STDOUT_LOG}")
 
 readonly FS_FIND_IN=$(echo "${RESULT}" | sed -n '/^BEGIN: find$/,/^END: find$/p' \
   | grep --fixed-strings /mnt/data/input/"${DOCKER_GCS_INPUTS}")
@@ -134,11 +134,11 @@ echo
 echo "On-disk output file list matches expected"
 
 # Verify in GCS that the DEEP directory is deep and the SHALLOW directory
-# is shallow. Gsutil prints directories with a trailing "/:" marker that is
+# is shallow. Gcloud storage prints directories with a trailing "/" that is
 # stripped using sed in order to match the output format of the `find` utility.
-readonly GCS_FIND="$(gsutil ls -r "${OUTPUTS}" \
+readonly GCS_FIND="$(gcloud storage ls -r "${OUTPUTS}" \
                      | grep -v '^ *$' \
-                     | sed -e 's#/:$##')"
+                     | sed -e 's#/$##')"
 
 for REC in "${EXPECTED_REMOTE_OUTPUT_ENTRIES[@]}"; do
   if ! echo "${GCS_FIND}" | grep --quiet --fixed-strings "${REC}"; then
diff --git a/test/integration/e2e_io_tasks.py b/test/integration/e2e_io_tasks.py
index 85e422b..604b421 100644
--- a/test/integration/e2e_io_tasks.py
+++ b/test/integration/e2e_io_tasks.py
@@ -104,7 +104,7 @@
       '--output OUTPUT_PATH')
   OUTPUT_FILE = '%s/%s.md5' % (OUTPUT_PATH[:-len('/*.md5')],
                                os.path.basename(INPUT_BAM))
-  RESULT = test_util.gsutil_cat(OUTPUT_FILE)
+  RESULT = test_util.gcloud_cat(OUTPUT_FILE)
 
   if not test_util.diff(RESULT_EXPECTED.strip(), RESULT.strip()):
     print('Output file does not match expected')
@@ -119,7 +119,7 @@
 RESULT_EXPECTED = POPULATION_MD5
 for i in range(TASKS_COUNT):
   OUTPUT_FILE = '%s/TASK_%s.md5' % (test.OUTPUTS, (i + 1))
-  RESULT = test_util.gsutil_cat(OUTPUT_FILE)
+  RESULT = test_util.gcloud_cat(OUTPUT_FILE)
 
   if not test_util.diff(RESULT_EXPECTED.strip(), RESULT.strip()):
     print('Output file does not match expected')
diff --git a/test/integration/e2e_logging_content.sh b/test/integration/e2e_logging_content.sh
index 1aabec0..4e51f9d 100755
--- a/test/integration/e2e_logging_content.sh
+++ b/test/integration/e2e_logging_content.sh
@@ -74,7 +74,7 @@ echo "Checking output..."
 # Check the results
 readonly STDOUT_RESULT_EXPECTED="$(echo -n "${STDOUT_MSG%.}")"
 
-readonly STDOUT_RESULT="$(gsutil cat "${STDOUT_LOG}")"
+readonly STDOUT_RESULT="$(gcloud storage cat "${STDOUT_LOG}")"
 if ! diff <(echo "${STDOUT_RESULT_EXPECTED}") <(echo "${STDOUT_RESULT}"); then
   echo "STDOUT file does not match expected"
   exit 1
@@ -82,7 +82,7 @@ fi
 
 readonly STDERR_RESULT_EXPECTED="$(echo -n "${STDERR_MSG%.}")"
 
-readonly STDERR_RESULT="$(gsutil cat "${STDERR_LOG}")"
+readonly STDERR_RESULT="$(gcloud storage cat "${STDERR_LOG}")"
 if ! diff <(echo "${STDERR_RESULT_EXPECTED}") <(echo "${STDERR_RESULT}"); then
   echo "STDERR file does not match expected"
   exit 1
diff --git a/test/integration/e2e_runtime.sh b/test/integration/e2e_runtime.sh
index fb276b9..4bd9e30 100755
--- a/test/integration/e2e_runtime.sh
+++ b/test/integration/e2e_runtime.sh
@@ -64,7 +64,7 @@ TMPDIR:
 EOF
 )
 
-readonly RESULT="$(gsutil cat "${STDOUT_LOG}")"
+readonly RESULT="$(gcloud storage cat "${STDOUT_LOG}")"
 if ! diff <(echo "${RESULT_EXPECTED}") <(echo "${RESULT}"); then
   echo "Output file does not match expected"
   exit 1
diff --git a/test/integration/e2e_skip.sh b/test/integration/e2e_skip.sh
index 20eb2c0..99d48ed 100755
--- a/test/integration/e2e_skip.sh
+++ b/test/integration/e2e_skip.sh
@@ -29,9 +29,9 @@ source "${SCRIPT_DIR}/test_setup_e2e.sh"
 TEST_FILE_PATH_1="${OUTPUTS}/testfile_1.txt"
 TEST_FILE_PATH_2="${OUTPUTS}/testfile_2.txt"
 
-echo "hello world" | gsutil cp - "${TEST_FILE_PATH_1}"
+echo "hello world" | gcloud storage cp - "${TEST_FILE_PATH_1}"
 
-if gsutil ls "${TEST_FILE_PATH_2}" &> /dev/null; then
+if gcloud storage ls "${TEST_FILE_PATH_2}" &> /dev/null; then
   echo "Unexpected: the output file '${TEST_FILE_PATH_1}' already exists."
   exit 1
 fi
@@ -44,7 +44,7 @@ JOB_ID="$(
     --skip \
     --wait)"
 
-RESULT="$(gsutil cat "${TEST_FILE_PATH_1}")"
+RESULT="$(gcloud storage cat "${TEST_FILE_PATH_1}")"
 if [[ "${RESULT}" != "hello world" ]]; then
   echo "Output file does not match expected (from step 4)"
   echo "Expected: hello world"
@@ -60,7 +60,7 @@ JOB_ID="$(
     --skip \
     --wait)"
 
-RESULT="$(gsutil cat "${TEST_FILE_PATH_2}")"
+RESULT="$(gcloud storage cat "${TEST_FILE_PATH_2}")"
 if [[ "${RESULT}" != "hello from the job" ]]; then
   echo "Output file does not match expected (from step 2)"
   echo "Expected: hello world"
diff --git a/test/integration/e2e_skip_tasks.sh b/test/integration/e2e_skip_tasks.sh
index 4359c78..0ef9ecb 100755
--- a/test/integration/e2e_skip_tasks.sh
+++ b/test/integration/e2e_skip_tasks.sh
@@ -30,7 +30,7 @@ TEST_FILE_PATH_1="${OUTPUTS}/testfile_1.txt"
 TEST_FILE_PATH_2="${OUTPUTS}/testfile_2.txt"
 TEST_FILE_PATH_3="${OUTPUTS}/testfile_3.txt"
 
-if gsutil ls "${TEST_FILE_PATH_1}" &> /dev/null; then
+if gcloud storage ls "${TEST_FILE_PATH_1}" &> /dev/null; then
   echo "Unexpected: the output file '${TEST_FILE_PATH_1}' already exists."
   exit 1
 fi
@@ -51,12 +51,12 @@ JOB_ID="$(
     --skip \
     --wait)"
 
-if ! gsutil ls "${TEST_FILE_PATH_1}" &> /dev/null; then
+if ! gcloud storage ls "${TEST_FILE_PATH_1}" &> /dev/null; then
   echo "Unexpected: the output file '${TEST_FILE_PATH_1}' was not created."
   exit 1
 fi
 
-RESULT="$(gsutil cat "${TEST_FILE_PATH_1}")"
+RESULT="$(gcloud storage cat "${TEST_FILE_PATH_1}")"
 if [[ "${RESULT}" != "hello world" ]]; then
   echo "Output file does not match expected (from step 1)"
   echo "Expected: hello world"
@@ -72,7 +72,7 @@ JOB_ID="$(
     --skip \
     --wait)"
 
-RESULT="$(gsutil cat "${TEST_FILE_PATH_1}")"
+RESULT="$(gcloud storage cat "${TEST_FILE_PATH_1}")"
 if [[ "${RESULT}" != "hello world" ]]; then
   echo "Output file does not match expected (from step 2)"
   echo "Expected: hello world"
@@ -96,14 +96,14 @@ JOB_ID="$(
     --skip \
     --wait)"
 
-RESULT="$(gsutil cat "${TEST_FILE_PATH_1}")"
+RESULT="$(gcloud storage cat "${TEST_FILE_PATH_1}")"
 if [[ "${RESULT}" != "hello world" ]]; then
   echo "Output file does not match expected (from step 3)"
   echo "Expected: hello world"
   echo "Got: ${RESULT}"
   exit 1
 fi
-RESULT="$(gsutil cat "${TEST_FILE_PATH_2}")"
+RESULT="$(gcloud storage cat "${TEST_FILE_PATH_2}")"
 if [[ "${RESULT}" != "hello again from row 2" ]]; then
   echo "Output file does not match expected (from step 3)"
   echo "Expected: hello again from row 2"
diff --git a/test/integration/io_setup.sh b/test/integration/io_setup.sh
index 6cf7caf..db62459 100644
--- a/test/integration/io_setup.sh
+++ b/test/integration/io_setup.sh
@@ -54,10 +54,10 @@ readonly TEST_LOCAL_MOUNT_PARAMETER="file://${TEST_TMP_PATH}"
 function io_setup::mount_local_path_setup() {
   mkdir -p "${TEST_TMP_PATH}"
   if [[ ! -f "${TEST_TMP_PATH}/${POPULATION_FILE}" ]]; then
-    gsutil cp "${POPULATION_FILE_FULL_PATH}" "${TEST_TMP_PATH}/${POPULATION_FILE}"
+    gcloud storage cp "${POPULATION_FILE_FULL_PATH}" "${TEST_TMP_PATH}/${POPULATION_FILE}"
   fi
   if [[ ! -f "${TEST_TMP_PATH}/${INPUT_BAM_FILE}" ]]; then
-    gsutil cp "${INPUT_BAM_FULL_PATH}" "${TEST_TMP_PATH}/${INPUT_BAM_FILE}"
+    gcloud storage cp "${INPUT_BAM_FULL_PATH}" "${TEST_TMP_PATH}/${INPUT_BAM_FILE}"
   fi
 }
 readonly -f io_setup::mount_local_path_setup
@@ -190,7 +190,7 @@ function io_setup::_check_output() {
   local output_file="${1}"
   local result_expected="${2}"
 
-  local result=$(gsutil cat "${output_file}")
+  local result=$(gcloud storage cat "${output_file}")
   if ! diff <(echo "${result_expected}") <(echo "${result}"); then
     echo "Output file does not match expected"
     exit 1
diff --git a/test/integration/io_tasks_setup.sh b/test/integration/io_tasks_setup.sh
index 9b93627..f78d4ec 100644
--- a/test/integration/io_tasks_setup.sh
+++ b/test/integration/io_tasks_setup.sh
@@ -70,7 +70,7 @@ function io_tasks_setup::check_output() {
 
     output_path="$(grep "${input_bam}" "${TASKS_FILE}" | cut -d $'\t' -f 3)"
     output_file="${output_path%/*.md5}/$(basename "${input_bam}").md5"
-    result="$(gsutil cat "${output_file}")"
+    result="$(gcloud storage cat "${output_file}")"
 
     if ! diff <(echo "${expected}") <(echo "${result}"); then
       echo "Output file does not match expected"
@@ -88,7 +88,7 @@ function io_tasks_setup::check_output() {
   expected="${POPULATION_MD5}"
   for ((i=0; i < tasks_count; i++)); do
     output_file="${OUTPUTS}/TASK_$((i+1)).md5"
-    result="$(gsutil cat "${output_file}")"
+    result="$(gcloud storage cat "${output_file}")"
 
     if ! diff <(echo "${expected}") <(echo "${result}"); then
       echo "Output file does not match expected"
diff --git a/test/integration/script_block_external_network.sh b/test/integration/script_block_external_network.sh
index a2b20e5..af1eadf 100755
--- a/test/integration/script_block_external_network.sh
+++ b/test/integration/script_block_external_network.sh
@@ -21,8 +21,8 @@ set -o nounset
 
 RC=0
 
-if ! gsutil -o 'Boto:num_retries=0' ls gs://genomics-public-data; then
-  1>&2 echo "\`gsutil ls\` should not have succeeded"
+if ! gcloud storage ls --retry-max-attempts=0 gs://genomics-public-data; then
+  1>&2 echo "\`gcloud storage ls\` should not have succeeded"
   RC=1
 fi
 
diff --git a/test/integration/test_setup_e2e.py b/test/integration/test_setup_e2e.py
index 898d878..f5df9e8 100644
--- a/test/integration/test_setup_e2e.py
+++ b/test/integration/test_setup_e2e.py
@@ -100,9 +100,9 @@ def _environ():
 print("  Bucket detected as: %s" % DSUB_BUCKET)
 
 print("  Checking if bucket exists")
-if not test_util.gsutil_ls_check("gs://%s" % DSUB_BUCKET):
+if not test_util.gcloud_ls_check("gs://%s" % DSUB_BUCKET):
   print("Bucket does not exist: %s" % DSUB_BUCKET, file=sys.stderr)
-  print("Create the bucket with \"gsutil mb\".", file=sys.stderr)
+  print("Create the bucket with \"gcloud storage buckets create\".", file=sys.stderr)
   sys.exit(1)
 
 # Set standard LOGGING, INPUTS, and OUTPUTS values
@@ -130,10 +130,10 @@ def _environ():
 print("Output path: %s" % OUTPUTS)
 
 print("  Checking if remote test files already exists")
-if test_util.gsutil_ls_check("%s/**" % TEST_GCS_ROOT):
+if test_util.gcloud_ls_check("%s/**" % TEST_GCS_ROOT):
   print("Test files exist: %s" % TEST_GCS_ROOT, file=sys.stderr)
   print("Remove contents:", file=sys.stderr)
-  print("  gsutil -m rm %s/**" % TEST_GCS_ROOT, file=sys.stderr)
+  print("  gcloud storage rm %s/**" % TEST_GCS_ROOT, file=sys.stderr)
   sys.exit(1)
 
 if TASKS_FILE:
diff --git a/test/integration/test_setup_e2e.sh b/test/integration/test_setup_e2e.sh
index c5e7cc1..00d6be9 100755
--- a/test/integration/test_setup_e2e.sh
+++ b/test/integration/test_setup_e2e.sh
@@ -64,13 +64,12 @@ fi
 echo "  Bucket detected as: ${DSUB_BUCKET}"
 
 echo "  Checking if bucket exists"
-if ! gsutil ls "gs://${DSUB_BUCKET}" 2>/dev/null; then
+if ! gcloud storage ls "gs://${DSUB_BUCKET}" 2>/dev/null; then
   1>&2 echo "Bucket does not exist (or we have no access): ${DSUB_BUCKET}"
-  1>&2 echo "Create the bucket with \"gsutil mb\"."
+  1>&2 echo "Create the bucket with \"gcloud storage buckets create\"."
   1>&2 echo "Current gcloud settings:"
   1>&2 echo "  account: $(gcloud config get-value account 2>/dev/null)"
   1>&2 echo "  project: $(gcloud config get-value project 2>/dev/null)"
-  1>&2 echo "  pass_credentials_to_gsutil: $(gcloud config get-value pass_credentials_to_gsutil 2>/dev/null)"
   exit 1
 fi
 
@@ -155,10 +154,10 @@ echo "Output path: ${OUTPUTS}"
 readonly DSUB_PARAMS="${TEST_GCS_ROOT}/params"
 
 echo "  Checking if remote test files already exists"
-if gsutil ls "${TEST_GCS_ROOT}/**" 2>/dev/null; then
+if gcloud storage ls "${TEST_GCS_ROOT}/**" 2>/dev/null; then
   1>&2 echo "Test files exist: ${TEST_GCS_ROOT}"
   1>&2 echo "Remove contents:"
-  1>&2 echo "  gsutil -m rm ${TEST_GCS_ROOT}/**"
+  1>&2 echo "  gcloud storage rm ${TEST_GCS_ROOT}/**"
   exit 1
 fi
 
diff --git a/test/integration/test_util.py b/test/integration/test_util.py
index 8100261..3014e97 100644
--- a/test/integration/test_util.py
+++ b/test/integration/test_util.py
@@ -29,13 +29,13 @@ def to_string(stdoutbytes):
   return stdoutbytes.decode(encoding)
 
 
-def gsutil_ls_check(path):
-  return not subprocess.call('gsutil ls "%s" 2>/dev/null' % path, shell=True)
+def gcloud_ls_check(path):
+  return not subprocess.call('gcloud storage ls "%s" 2>/dev/null' % path, shell=True)
 
 
-def gsutil_cat(path):
+def gcloud_cat(path):
   return to_string(
-      subprocess.check_output('gsutil cat "%s"' % path, shell=True))
+      subprocess.check_output('gcloud storage cat "%s"' % path, shell=True))
 
 
 def diff(str1, str2):
diff --git a/test/integration/unit_skip.test-fails.sh b/test/integration/unit_skip.test-fails.sh
index e80a500..5fd629d 100755
--- a/test/integration/unit_skip.test-fails.sh
+++ b/test/integration/unit_skip.test-fails.sh
@@ -46,8 +46,8 @@ readonly NEWFILE="${OUTPUTS}/newfile"
 readonly OUT_FOLDER_2="${OUTPUTS}/newfolder"
 
 # Create pre-existing output
-echo "test output" | gsutil cp - "${EXISTING}"
-echo "test output" | gsutil cp - "${EXISTING_2}"
+echo "test output" | gcloud storage cp - "${EXISTING}"
+echo "test output" | gcloud storage cp - "${EXISTING_2}"
 
 echo "Job 1 ..."
 

From 4acde36d9235f5f6d03f49a0ec52643c9469aaa2 Mon Sep 17 00:00:00 2001
From: "Kristen Liu (Ong)" <43296048+kvo3@users.noreply.github.com>
Date: Wed, 19 Aug 2026 15:40:02 -0700
Subject: [PATCH 02/10] Update _dsub_version.py

---
 dsub/_dsub_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dsub/_dsub_version.py b/dsub/_dsub_version.py
index 050f045..7b12e0d 100644
--- a/dsub/_dsub_version.py
+++ b/dsub/_dsub_version.py
@@ -27,5 +27,5 @@
 """
 
 
-DSUB_VERSION = '0.5.2'
+DSUB_VERSION = '0.5.3.dev0'
 

From 79dd6ad9c51343d232ec65e10b44919dae7612ea Mon Sep 17 00:00:00 2001
From: Kristen Liu <kristenliu@verily.health>
Date: Fri, 21 Aug 2026 00:43:46 +0000
Subject: [PATCH 03/10] Fixed python3 versioning

---
 .../google_batch-checkpoint.py                | 1068 +++++++++++++++++
 .../google_utils-checkpoint.py                |  502 ++++++++
 dsub/providers/google_batch.py                |    2 +-
 dsub/providers/google_utils.py                |    2 +-
 4 files changed, 1572 insertions(+), 2 deletions(-)
 create mode 100644 dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py
 create mode 100644 dsub/providers/.ipynb_checkpoints/google_utils-checkpoint.py

diff --git a/dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py b/dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py
new file mode 100644
index 0000000..0323f81
--- /dev/null
+++ b/dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py
@@ -0,0 +1,1068 @@
+# Copyright 2022 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Provider for running jobs on Google Cloud Platform.
+
+This module implements job creation, listing, and canceling using the
+Google Batch v1 APIs.
+"""
+
+import ast
+import operator
+import os
+import re
+import sys
+import textwrap
+from typing import Dict, List, Set
+
+from ..lib import dsub_util
+from ..lib import job_model
+from ..lib import param_util
+from ..lib import providers_util
+from . import base
+from . import google_base
+from . import google_batch_operations
+from . import google_custom_machine
+from . import google_utils
+
+# pylint: disable=g-import-not-at-top
+try:
+  from google.cloud import batch_v1
+except ImportError:
+  # TODO: Remove conditional import when batch library is available
+  from . import batch_dummy as batch_v1
+# pylint: enable=g-import-not-at-top
+_PROVIDER_NAME = 'google-batch'
+# Index of the prepare action in the runnable list
+_PREPARE_INDEX = 1
+
+# Create file provider whitelist.
+_SUPPORTED_FILE_PROVIDERS = frozenset([job_model.P_GCS])
+_SUPPORTED_LOGGING_PROVIDERS = _SUPPORTED_FILE_PROVIDERS
+_SUPPORTED_INPUT_PROVIDERS = _SUPPORTED_FILE_PROVIDERS
+_SUPPORTED_OUTPUT_PROVIDERS = _SUPPORTED_FILE_PROVIDERS
+
+# Mount point for the data disk in the user's Docker container
+_VOLUME_MOUNT_POINT = '/mnt/disks/data'
+_DATA_MOUNT_POINT = '/mnt/data'
+
+# These are documented (providers/README.md) as being read/write to user
+# commands.
+_SCRIPT_DIR = f'{_DATA_MOUNT_POINT}/script'
+_TMP_DIR = f'{_DATA_MOUNT_POINT}/tmp'
+_WORKING_DIR = f'{_DATA_MOUNT_POINT}/workingdir'
+
+# These are visible to the user task; not yet documented, as we'd *like* to
+# find a way to have them visible only to the logging tasks.
+_BATCH_LOG_DIR = f'{_VOLUME_MOUNT_POINT}/.logging'
+_LOGGING_DIR = f'{_DATA_MOUNT_POINT}/.logging'
+
+_LOG_FILTER_VAR = '_LOG_FILTER_REPR'
+_LOG_FILTER_SCRIPT_PATH = f'{_DATA_MOUNT_POINT}/.log_filter_script.py'
+
+# _LOG_FILTER_PYTHON is a block of Python code to execute in both the
+# "continuous_logging" and "final_logging" tasks.
+#
+# Batch API will eventually create three log files in the _BATCH_LOG_FILE_PATH
+# directory. They are named:
+#
+# - output-*.log
+# - stdout-*.log
+# - stderr-*.log
+#
+# We will be creating a "staging" location for each of these files.
+#
+# If any of the batch log files don't exist, touch the associated staging file.
+#
+# If the output batch log file exists, copy it directly to the staging
+# location.
+#
+# If the stdout/stderr batch log files exist, copy them to their staging
+# location. Then filter it so that only user-action logs exist and the prefixes
+# are removed. The prefixes look something like:
+# [batch_task_logs]<datetime> ERROR:
+#   [task_id:task/<job_uid>,runnable_index:<action_number>]
+
+# pylint: disable=anomalous-backslash-in-string
+_LOG_FILTER_PYTHON = textwrap.dedent(r"""
+import fileinput
+import glob
+import re
+import shutil
+import sys
+from pathlib import Path
+
+LOGGING_DIR = sys.argv[1]
+LOG_FILE_PATH = sys.argv[2]
+STDOUT_FILE_PATH = sys.argv[3]
+STDERR_FILE_PATH = sys.argv[4]
+USER_TASK = sys.argv[5]
+
+def filter_log_file(staging_path: str, stream_string: str):
+  # Replaces lines in file inplace
+  for line in fileinput.input(staging_path, inplace=True):
+    re_search_string = fr"^\[batch_task_logs\].*{stream_string}: \[task_id:task\/.*runnable_index:{USER_TASK}] (.*)"
+    match = re.search(re_search_string, line)
+    if match:
+      modified_line = match.group(1)
+      print(modified_line)
+
+def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None):
+  # Check if log files exist, and copy to their staging location
+  matching_files = list(Path(LOGGING_DIR).glob(glob_str))
+  if matching_files:
+    assert(len(matching_files) == 1)
+    shutil.copy(matching_files[0], staging_path)
+    if filter_str:
+      filter_log_file(staging_path, filter_str)
+  else:
+    Path(staging_path).touch()
+
+# We know the log file is named output-<job-uid>.log
+# and the stdout/stderr files are named stdout-<job-uid>.log and
+# stderr-<job-uid>.log
+copy_log_to_staging("output-*.log", LOG_FILE_PATH)
+copy_log_to_staging("stdout-*.log", STDOUT_FILE_PATH, filter_str="INFO")
+copy_log_to_staging("stderr-*.log", STDERR_FILE_PATH, filter_str="ERROR")
+""")
+# pylint: enable=anomalous-backslash-in-string
+
+_LOG_CP = textwrap.dedent("""
+  python3 "{log_filter_script_path}" \
+      "${{LOGGING_DIR}}" \
+      "${{LOGGING_DIR}}/log.txt" \
+      "${{LOGGING_DIR}}/stdout.txt" \
+      "${{LOGGING_DIR}}/stderr.txt" \
+      "{user_action}"
+
+  gsutil_cp "${{LOGGING_DIR}}/stdout.txt" "${{STDOUT_PATH}}" "text/plain" "${{USER_PROJECT}}" &
+  STDOUT_PID=$!
+  gsutil_cp "${{LOGGING_DIR}}/stderr.txt" "${{STDERR_PATH}}" "text/plain"  "${{USER_PROJECT}}" &
+  STDERR_PID=$!
+  gsutil_cp "${{LOGGING_DIR}}/log.txt" "${{LOGGING_PATH}}" "text/plain" "${{USER_PROJECT}}" &
+  LOG_PID=$!
+
+  wait "${{STDOUT_PID}}"
+  wait "${{STDERR_PID}}"
+  wait "${{LOG_PID}}"
+""")
+
+_FINAL_LOGGING_CMD = textwrap.dedent("""\
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  readonly LOGGING_DIR="{logging_dir}"
+
+  # Flag the continuous logging command to stop
+  touch "${{LOGGING_DIR}}/.stop_logging"
+
+  {log_msg_fn}
+  {gsutil_cp_fn}
+
+  {log_cp}
+""")
+
+# Keep logging until the final logging action starts
+_CONTINUOUS_LOGGING_CMD = textwrap.dedent("""\
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  readonly LOGGING_DIR="{logging_dir}"
+
+  {log_msg_fn}
+  {gsutil_cp_fn}
+
+  # Make sure the logging work directory exists
+  mkdir -p "${{LOGGING_DIR}}"
+
+  # Prep the log filter script
+  echo "${{{log_filter_var}}}" \
+    | python3 -c '{python_decode_script}' \
+    > "{log_filter_script_path}"
+  chmod a+x "{log_filter_script_path}"
+
+  while [[ ! -e "${{LOGGING_DIR}}/.stop_logging" ]]; do
+    {log_cp}
+
+    sleep "{log_interval}"
+  done
+""")
+
+
+_EVENT_REGEX_MAP = {
+    'scheduled': re.compile('^Job state is set from QUEUED to SCHEDULED'),
+    'start': re.compile('^Job state is set from SCHEDULED to RUNNING'),
+    'ok': re.compile('^Job state is set from RUNNING to SUCCEEDED'),
+    'fail': re.compile('^Job state is set from .+? to FAILED'),
+    'cancellation-in-progress': re.compile(
+        '^Job state is set from .+? to CANCELLATION_IN_PROGRESS'
+    ),
+    'canceled': re.compile('^Job state is set from .+? to CANCELLED'),
+}
+
+
+class GoogleBatchEventMap(object):
+  """Helper for extracing a set of normalized, filtered operation events."""
+
+  def __init__(self, op: batch_v1.types.Job):
+    self._op = op
+
+  def get_filtered_normalized_events(self):
+    """Map and filter the batch API events down to events of interest.
+
+    Returns:
+      A list of maps containing the normalized, filtered events.
+    """
+    events = {}
+    for event in google_batch_operations.get_status_events(self._op):
+      mapped, _ = self._map(event)
+      name = mapped['name']
+
+      events[name] = mapped
+
+    return sorted(list(events.values()), key=operator.itemgetter('event-time'))
+
+  def _map(self, event):
+    """Extract elements from a Batch status event and map to a named event."""
+    description = event.description
+    event_time = event.event_time.rfc3339()
+
+    for name, regex in _EVENT_REGEX_MAP.items():
+      match = regex.match(description)
+      if match:
+        return {'name': name, 'event-time': event_time}, match
+
+    return {'name': description, 'event-time': event_time}, None
+
+
+class GoogleBatchOperation(base.Task):
+  """Task wrapper around a Batch API Job object."""
+
+  def __init__(self, operation_data: batch_v1.types.Job):
+    self._op = operation_data
+    self._job_descriptor = self._try_op_to_job_descriptor()
+
+  def raw_task_data(self):
+    return self._op
+
+  def _try_op_to_job_descriptor(self):
+    # The _META_YAML_REPR field in the 'prepare' action enables reconstructing
+    # the original job descriptor.
+    # We only need the env for the prepare action (runnable) here.
+    env = google_batch_operations.get_environment(self._op, _PREPARE_INDEX)
+    if not env:
+      return
+
+    meta = env.get(google_utils.META_YAML_VARNAME)
+    if not meta:
+      return
+
+    return job_model.JobDescriptor.from_yaml(ast.literal_eval(meta))
+
+  def get_field(self, field: str, default: str = None):
+    """Returns a value from the operation for a specific set of field names.
+
+    This is the implementation of base.Task's abstract get_field method. See
+    base.py get_field for more details.
+
+    Args:
+      field: a dsub-specific job metadata key
+      default: default value to return if field does not exist or is empty.
+
+    Returns:
+      A text string for the field or a list for 'inputs'.
+
+    Raises:
+      ValueError: if the field label is not supported by the operation
+    """
+    value = None
+    if field == 'internal-id':
+      value = self._op.name
+    elif field == 'user-project':
+      if self._job_descriptor:
+        value = self._job_descriptor.job_metadata.get(field)
+    elif field in [
+        'job-id',
+        'job-name',
+        'task-id',
+        'task-attempt',
+        'user-id',
+        'dsub-version',
+    ]:
+      value = google_batch_operations.get_label(self._op, field)
+    elif field == 'task-status':
+      value = self._operation_status()
+    elif field == 'logging':
+      if self._job_descriptor:
+        # The job_resources will contain the "--logging" value.
+        # The task_resources will contain the resolved logging path.
+        # Return the resolved logging path.
+        task_resources = self._job_descriptor.task_descriptors[0].task_resources
+        value = task_resources.logging_path
+    elif field in ['envs', 'labels']:
+      if self._job_descriptor:
+        items = providers_util.get_job_and_task_param(
+            self._job_descriptor.job_params,
+            self._job_descriptor.task_descriptors[0].task_params,
+            field,
+        )
+        value = {item.name: item.value for item in items}
+    elif field in [
+        'inputs',
+        'outputs',
+        'input-recursives',
+        'output-recursives',
+    ]:
+      if self._job_descriptor:
+        value = {}
+        items = providers_util.get_job_and_task_param(
+            self._job_descriptor.job_params,
+            self._job_descriptor.task_descriptors[0].task_params,
+            field,
+        )
+        value.update({item.name: item.value for item in items})
+    elif field == 'mounts':
+      if self._job_descriptor:
+        items = providers_util.get_job_and_task_param(
+            self._job_descriptor.job_params,
+            self._job_descriptor.task_descriptors[0].task_params,
+            field,
+        )
+        value = {item.name: item.value for item in items}
+    elif field == 'provider':
+      return _PROVIDER_NAME
+    elif field == 'provider-attributes':
+      value = {
+          'boot-disk-size': google_batch_operations.get_boot_disk_size(
+              self._op
+          ),
+          'disk-size': google_batch_operations.get_disk_size(self._op),
+          'disk-type': google_batch_operations.get_disk_type(self._op),
+          'machine-type': google_batch_operations.get_machine_type(self._op),
+          'regions': google_batch_operations.get_regions(self._op),
+          'zones': google_batch_operations.get_zones(self._op),
+          'preemptible': google_batch_operations.get_preemptible(self._op),
+      }
+    elif field == 'events':
+      value = GoogleBatchEventMap(self._op).get_filtered_normalized_events()
+    elif field == 'script-name':
+      if self._job_descriptor:
+        value = self._job_descriptor.job_metadata.get(field)
+    elif field == 'script':
+      value = self._try_op_to_script_body()
+    elif field == 'create-time' or field == 'start-time':
+      # TODO: Does Batch offer a start or end-time?
+      # Check http://shortn/_FPYmD1weUF
+      ds = google_batch_operations.get_create_time(self._op)
+      value = google_base.parse_rfc3339_utc_string(ds)
+    elif field == 'end-time' or field == 'last-update':
+      # TODO: Does Batch offer an end-time?
+      # Check http://shortn/_FPYmD1weUF
+      ds = google_batch_operations.get_update_time(self._op)
+      if ds:
+        value = google_base.parse_rfc3339_utc_string(ds)
+    elif field == 'status':
+      value = self._operation_status()
+    elif field == 'status-message':
+      msg, _, _ = self._operation_status_message()
+      value = msg
+    elif field == 'status-detail':
+      # As much detail as we can reasonably get from the operation
+      msg, _, detail = self._operation_status_message()
+      if detail:
+        msg = detail
+      value = msg
+    else:
+      raise ValueError(f'Unsupported field: "{field}"')
+
+    return value if value else default
+
+  def _try_op_to_script_body(self):
+    # We only need the env for the prepare action (runnable) here.
+    env = google_batch_operations.get_environment(self._op, _PREPARE_INDEX)
+    if env:
+      return ast.literal_eval(env.get(google_utils.SCRIPT_VARNAME))
+
+  def _operation_status(self):
+    """Returns the status of this operation.
+
+    Raises:
+      ValueError: if the operation status cannot be determined.
+
+    Returns:
+      A printable status string (RUNNING, SUCCESS, CANCELED or FAILURE).
+    """
+    if not google_batch_operations.is_done(self._op):
+      return 'RUNNING'
+    if google_batch_operations.is_success(self._op):
+      return 'SUCCESS'
+    if google_batch_operations.is_canceled(self._op):
+      return 'CANCELED'
+    if google_batch_operations.is_failed(self._op):
+      return 'FAILURE'
+
+    raise ValueError(
+        'Status for operation {} could not be determined'.format(
+            self._op['name']
+        )
+    )
+
+  def _operation_status_message(self):
+    """Returns the most relevant status string and failed action.
+
+    This string is meant for display only.
+
+    Returns:
+      A triple of:
+      - printable status message
+      - the action that failed (if any)
+      - a detail message (if available)
+    """
+    msg = ''
+    action = None
+    detail = None
+    status_events = google_batch_operations.get_status_events(self._op)
+    if not google_batch_operations.is_done(self._op):
+      msg = 'RUNNING'
+    elif google_batch_operations.is_success(self._op):
+      msg = 'SUCCESS'
+    elif google_batch_operations.is_canceled(self._op):
+      msg = 'CANCELED'
+    elif google_batch_operations.is_failed(self._op):
+      msg = 'FAILURE'
+
+    if status_events:
+      detail = status_events[-1].description
+    return msg, action, detail
+
+
+class GoogleBatchBatchHandler(object):
+  """Implement the HttpBatch interface to enable simple serial batches."""
+
+  def __init__(self, callback):
+    self._cancel_list = []
+    self._response_handler = callback
+
+  def add(self, cancel_fn, request_id):
+    self._cancel_list.append((request_id, cancel_fn))
+
+  def execute(self):
+    for request_id, cancel_fn in self._cancel_list:
+      response = None
+      exception = None
+      try:
+        response = cancel_fn.result()
+      except:  # pylint: disable=bare-except
+        exception = sys.exc_info()[1]
+
+      self._response_handler(request_id, response, exception)
+
+
+class GoogleBatchJobProvider(google_utils.GoogleJobProviderBase):
+  """dsub provider implementation managing Jobs on Google Cloud."""
+
+  def __init__(
+      self, dry_run: bool, project: str, location: str, credentials=None
+  ):
+    storage_service = dsub_util.get_storage_service(credentials=credentials)
+
+    self._dry_run = dry_run
+    self._location = location
+    self._project = project
+    self._storage_service = storage_service
+
+  def _batch_handler_def(self):
+    return GoogleBatchBatchHandler
+
+  def _operations_cancel_api_def(self):
+    return batch_v1.BatchServiceClient().cancel_job
+
+  def _get_provisioning_model(self, task_resources):
+    if task_resources.preemptible:
+      return batch_v1.AllocationPolicy.ProvisioningModel.SPOT
+    else:
+      return batch_v1.AllocationPolicy.ProvisioningModel.STANDARD
+
+  def _get_batch_job_regions(self, regions, zones) -> List[str]:
+    """Returns the list of regions and zones to use for a Batch Job request.
+
+    If neither regions nor zones were specified for the Job, then use the
+    Batch Job API location as the default region.
+
+    Regions need to be prefixed with "regions/" and zones need to be prefixed
+    with "zones/" as documented in
+    https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#LocationPolicy
+
+    Args:
+      regions (str): A space separated list of regions to use for the Job.
+      zones (str): A space separated list of zones to use for the Job.
+    """
+    if regions:
+      regions = [f'regions/{region}' for region in regions]
+    if zones:
+      zones = [f'zones/{zone}' for zone in zones]
+    if not regions and not zones:
+      return [f'regions/{self._location}']
+    return (regions or []) + (zones or [])
+
+  def _get_logging_env(self, logging_uri, user_project, include_filter_script):
+    """Returns the environment for actions that copy logging files."""
+    if not logging_uri.endswith('.log'):
+      raise ValueError('Logging URI must end in ".log": {}'.format(logging_uri))
+
+    logging_prefix = logging_uri[: -len('.log')]
+    env = {
+        'LOGGING_PATH': '{}.log'.format(logging_prefix),
+        'STDOUT_PATH': '{}-stdout.log'.format(logging_prefix),
+        'STDERR_PATH': '{}-stderr.log'.format(logging_prefix),
+        'USER_PROJECT': user_project,
+    }
+    if include_filter_script:
+      env[_LOG_FILTER_VAR] = repr(_LOG_FILTER_PYTHON)
+
+    return env
+
+  def _format_batch_job_id(self, task_metadata, job_metadata) -> str:
+    # Each dsub task is submitted as its own Batch API job, so we
+    # append the dsub task-id and task-attempt to the job-id for the
+    # batch job ID.
+    # For single-task dsub jobs, there is no task-id, so use 0.
+    # Use a '-' character as the delimeter because Batch API job ID
+    # must match regex ^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$
+    task_id = task_metadata.get('task-id') or 0
+    task_attempt = task_metadata.get('task-attempt') or 0
+    batch_job_id = job_metadata.get('job-id')
+    return f'{batch_job_id}-{task_id}-{task_attempt}'
+
+  def _get_gcs_volumes(self, mounts) -> List[batch_v1.types.Volume]:
+    # Return a list of GCS volumes for the Batch Job request.
+    gcs_volumes = []
+    for gcs_mount in param_util.get_gcs_mounts(mounts):
+      mount_path = os.path.join(_VOLUME_MOUNT_POINT, gcs_mount.docker_path)
+      # Normalize mount path because API does not allow trailing slashes
+      normalized_mount_path = os.path.normpath(mount_path)
+      gcs_volume = google_batch_operations.build_gcs_volume(
+          gcs_mount.value[len('gs://') :], normalized_mount_path, ['-o ro']
+      )
+      gcs_volumes.append(gcs_volume)
+    return gcs_volumes
+
+  def _get_gcs_volumes_for_user_command(self, mounts) -> List[str]:
+    # Return a list of GCS volumes to be included with the
+    # user-command runnable
+    user_command_volumes = []
+    for gcs_mount in param_util.get_gcs_mounts(mounts):
+      volume_mount_point = os.path.normpath(
+          os.path.join(_VOLUME_MOUNT_POINT, gcs_mount.docker_path)
+      )
+      data_mount_point = os.path.normpath(
+          os.path.join(_DATA_MOUNT_POINT, gcs_mount.docker_path)
+      )
+      user_command_volumes.append(f'{volume_mount_point}:{data_mount_point}')
+    return user_command_volumes
+
+  def _create_batch_request(
+      self,
+      task_view: job_model.JobDescriptor,
+  ):
+    job_metadata = task_view.job_metadata
+    job_params = task_view.job_params
+    job_resources = task_view.job_resources
+    task_metadata = task_view.task_descriptors[0].task_metadata
+    task_params = task_view.task_descriptors[0].task_params
+    task_resources = task_view.task_descriptors[0].task_resources
+
+    # Set up VM-specific variables
+    datadisk_volume = google_batch_operations.build_volume(
+        disk=google_utils.DATA_DISK_NAME, path=_VOLUME_MOUNT_POINT
+    )
+
+    # Set up the task labels
+    # pylint: disable=g-complex-comprehension
+    labels = {
+        label.name: label.value if label.value else ''
+        for label in google_base.build_pipeline_labels(
+            job_metadata, task_metadata
+        )
+        | job_params['labels']
+        | task_params['labels']
+    }
+    # pylint: enable=g-complex-comprehension
+
+    # Set local variables for the core pipeline values
+    script = task_view.job_metadata['script']
+
+    # Track 0-based runnable indexes for cross-task awareness
+    user_action = 3
+
+    continuous_logging_cmd = _CONTINUOUS_LOGGING_CMD.format(
+        log_msg_fn=google_utils.LOG_MSG_FN,
+        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
+        log_filter_var=_LOG_FILTER_VAR,
+        log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
+        python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
+        logging_dir=_LOGGING_DIR,
+        log_cp=_LOG_CP.format(
+            log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
+            user_action=user_action,
+        ),
+        log_interval=job_resources.log_interval or '60s',
+    )
+
+    logging_cmd = _FINAL_LOGGING_CMD.format(
+        log_msg_fn=google_utils.LOG_MSG_FN,
+        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
+        log_filter_var=_LOG_FILTER_VAR,
+        log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
+        python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
+        logging_dir=_LOGGING_DIR,
+        log_cp=_LOG_CP.format(
+            log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
+            user_action=user_action,
+        ),
+    )
+
+    # Set up command and environments for the prepare, localization, user,
+    # and de-localization actions
+    script_path = os.path.join(_SCRIPT_DIR, script.name)
+    user_project = task_view.job_metadata['user-project'] or ''
+
+    prepare_command = google_utils.PREPARE_CMD.format(
+        log_msg_fn=google_utils.LOG_MSG_FN,
+        mk_runtime_dirs=google_utils.make_runtime_dirs_command(
+            _SCRIPT_DIR, _TMP_DIR, _WORKING_DIR
+        ),
+        script_var=google_utils.SCRIPT_VARNAME,
+        python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
+        script_path=script_path,
+        mk_io_dirs=google_utils.MK_IO_DIRS,
+    )
+    # pylint: disable=line-too-long
+
+    continuous_logging_env = google_batch_operations.build_environment(
+        self._get_logging_env(
+            task_resources.logging_path.uri, user_project, True
+        )
+    )
+    final_logging_env = google_batch_operations.build_environment(
+        self._get_logging_env(
+            task_resources.logging_path.uri, user_project, False
+        )
+    )
+
+    envs = job_params['envs'] | task_params['envs']
+    inputs = job_params['inputs'] | task_params['inputs']
+    outputs = job_params['outputs'] | task_params['outputs']
+    mounts = job_params['mounts']
+    gcs_volumes = self._get_gcs_volumes(mounts)
+
+    prepare_env = google_batch_operations.build_environment(
+        self._get_prepare_env(
+            script, task_view, inputs, outputs, mounts, _DATA_MOUNT_POINT
+        )
+    )
+    localization_env = google_batch_operations.build_environment(
+        self._get_localization_env(inputs, user_project, _DATA_MOUNT_POINT)
+    )
+    user_environment = google_batch_operations.build_environment(
+        self._build_user_environment(
+            envs, inputs, outputs, mounts, _DATA_MOUNT_POINT
+        )
+    )
+    delocalization_env = google_batch_operations.build_environment(
+        self._get_delocalization_env(outputs, user_project, _DATA_MOUNT_POINT)
+    )
+
+    # Build the list of runnables (aka actions)
+    runnables = []
+
+    runnables.append(
+        # logging
+        google_batch_operations.build_runnable(
+            run_in_background=True,
+            always_run=False,
+            image_uri=google_utils.CLOUD_SDK_IMAGE,
+            environment=continuous_logging_env,
+            entrypoint='/bin/bash',
+            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}'],
+            commands=['-c', continuous_logging_cmd],
+            options=None
+        )
+    )
+
+    runnables.append(
+        # prepare
+        google_batch_operations.build_runnable(
+            run_in_background=False,
+            always_run=False,
+            image_uri=google_utils.CLOUD_SDK_IMAGE,
+            environment=prepare_env,
+            entrypoint='/bin/bash',
+            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}'],
+            commands=['-c', prepare_command],
+            options=None
+        )
+    )
+
+    runnables.append(
+        # localization
+        google_batch_operations.build_runnable(
+            run_in_background=False,
+            always_run=False,
+            image_uri=google_utils.CLOUD_SDK_IMAGE,
+            environment=localization_env,
+            entrypoint='/bin/bash',
+            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}'],
+            commands=[
+                '-c',
+                google_utils.LOCALIZATION_CMD.format(
+                    log_msg_fn=google_utils.LOG_MSG_FN,
+                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
+                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    cp_loop=google_utils.LOCALIZATION_LOOP,
+                ),
+            ],
+            options=None
+        )
+    )
+
+    user_command_volumes = [f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}']
+    for gcs_volume in self._get_gcs_volumes_for_user_command(mounts):
+      user_command_volumes.append(gcs_volume)
+    # Add --gpus all option for GPU-enabled containers
+    container_options = '--gpus all' if job_resources.accelerator_type and job_resources.accelerator_type.startswith('nvidia') else None
+    runnables.append(
+        # user-command
+        google_batch_operations.build_runnable(
+            run_in_background=False,
+            always_run=False,
+            image_uri=job_resources.image,
+            environment=user_environment,
+            entrypoint='/usr/bin/env',
+            volumes=user_command_volumes,
+            commands=[
+                'bash',
+                '-c',
+                google_utils.USER_CMD.format(
+                    tmp_dir=_TMP_DIR,
+                    working_dir=_WORKING_DIR,
+                    user_script=script_path,
+                ),
+            ],
+            options=container_options,
+        )
+    )
+
+    runnables.append(
+        # delocalization
+        google_batch_operations.build_runnable(
+            run_in_background=False,
+            always_run=False,
+            image_uri=google_utils.CLOUD_SDK_IMAGE,
+            environment=delocalization_env,
+            entrypoint='/bin/bash',
+            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}:ro'],
+            commands=[
+                '-c',
+                google_utils.LOCALIZATION_CMD.format(
+                    log_msg_fn=google_utils.LOG_MSG_FN,
+                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
+                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    cp_loop=google_utils.DELOCALIZATION_LOOP,
+                ),
+            ],
+            options=None
+        )
+    )
+
+    runnables.append(
+        # final_logging
+        google_batch_operations.build_runnable(
+            run_in_background=False,
+            always_run=True,
+            image_uri=google_utils.CLOUD_SDK_IMAGE,
+            environment=final_logging_env,
+            entrypoint='/bin/bash',
+            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}'],
+            commands=['-c', logging_cmd],
+            options=None
+        ),
+    )
+
+    # Prepare the VM (resources) configuration. The InstancePolicy describes an
+    # instance type and resources attached to each VM. The AllocationPolicy
+    # describes when, where, and how compute resources should be allocated
+    # for the Job.
+    boot_disk_size = (
+        job_resources.boot_disk_size if job_resources.boot_disk_size else 0
+    )
+    # Determine boot disk image: use user-specified value, or default to batch-debian for GPU jobs
+    if job_resources.boot_disk_image:
+      boot_disk_image = job_resources.boot_disk_image
+    elif job_resources.accelerator_type and job_resources.accelerator_type.startswith('nvidia'):
+      boot_disk_image = 'batch-debian'
+    else:
+      boot_disk_image = None
+
+    boot_disk = google_batch_operations.build_persistent_disk(
+        size_gb=max(boot_disk_size, job_model.LARGE_BOOT_DISK_SIZE),
+        disk_type=job_model.DEFAULT_DISK_TYPE,
+        image=boot_disk_image,
+    )
+    disk = google_batch_operations.build_persistent_disk(
+        size_gb=job_resources.disk_size,
+        disk_type=job_resources.disk_type or job_model.DEFAULT_DISK_TYPE,
+        image=None
+    )
+    attached_disk = google_batch_operations.build_attached_disk(
+        disk=disk, device_name=google_utils.DATA_DISK_NAME
+    )
+
+    if job_resources.machine_type:
+      machine_type = job_resources.machine_type
+    elif job_resources.min_cores or job_resources.min_ram:
+      machine_type = (
+          google_custom_machine.GoogleCustomMachine.build_machine_type(
+              job_resources.min_cores, job_resources.min_ram
+          )
+      )
+    else:
+      machine_type = job_model.DEFAULT_MACHINE_TYPE
+
+    instance_policy = google_batch_operations.build_instance_policy(
+        boot_disk=boot_disk,
+        disks=attached_disk,
+        machine_type=machine_type,
+        accelerators=google_batch_operations.build_accelerators(
+            accelerator_type=job_resources.accelerator_type,
+            accelerator_count=job_resources.accelerator_count,
+        ),
+        provisioning_model=self._get_provisioning_model(task_resources),
+    )
+
+    # Determine whether to install GPU drivers: use user-specified value, or default to True for GPU jobs
+    if job_resources.install_gpu_drivers is not None:
+      install_gpu_drivers = job_resources.install_gpu_drivers
+    else:
+      install_gpu_drivers = job_resources.accelerator_type is not None
+
+    ipt = google_batch_operations.build_instance_policy_or_template(
+        instance_policy=instance_policy,
+        install_gpu_drivers=install_gpu_drivers,
+    )
+
+    if job_resources.service_account:
+      scopes = job_resources.scopes or google_base.DEFAULT_SCOPES
+      service_account = google_batch_operations.build_service_account(
+          service_account_email=job_resources.service_account, scopes=scopes
+      )
+    else:
+      service_account = None
+
+    network_policy = google_batch_operations.build_network_policy(
+        network=job_resources.network,
+        subnetwork=job_resources.subnetwork,
+        no_external_ip_address=job_resources.use_private_address,
+    )
+
+    location_policy = google_batch_operations.build_location_policy(
+        allowed_locations=self._get_batch_job_regions(
+            regions=job_resources.regions, zones=job_resources.zones
+        ),
+    )
+
+    allocation_policy = google_batch_operations.build_allocation_policy(
+        ipts=[ipt],
+        service_account=service_account,
+        network_policy=network_policy,
+        location_policy=location_policy,
+    )
+
+    logs_policy = google_batch_operations.build_logs_policy(
+        # Explicitly end the logging path with a slash.
+        # This will prompt Batch API to create the log, stdout, and stderr
+        # files in the specified directory.
+        batch_v1.LogsPolicy.Destination.PATH,
+        _BATCH_LOG_DIR + '/',
+    )
+
+    # Bring together the task definition(s) and build the Job request.
+    task_spec = google_batch_operations.build_task_spec(
+        runnables=runnables, volumes=([datadisk_volume] + gcs_volumes), max_run_duration=job_resources.timeout
+    )
+    task_group = google_batch_operations.build_task_group(
+        task_spec, task_count=1, task_count_per_node=1
+    )
+
+    job = google_batch_operations.build_job(
+        [task_group], allocation_policy, labels, logs_policy
+    )
+
+    batch_job_id = self._format_batch_job_id(task_metadata, job_metadata)
+
+    job_request = batch_v1.CreateJobRequest(
+        parent=f'projects/{self._project}/locations/{self._location}',
+        job=job,
+        job_id=batch_job_id,
+    )
+    # pylint: enable=line-too-long
+    return job_request
+
+  def _submit_batch_job(self, request) -> str:
+    client = batch_v1.BatchServiceClient()
+    job_response = client.create_job(request=request)
+    op = GoogleBatchOperation(job_response)
+    print(f'Provider internal-id (operation): {job_response.name}')
+    return op.get_field('task-id')
+
+  def submit_job(
+      self,
+      job_descriptor: job_model.JobDescriptor,
+      skip_if_output_present: bool,
+  ) -> Dict[str, any]:
+    # Validate task data and resources.
+    param_util.validate_submit_args_or_fail(
+        job_descriptor,
+        provider_name=_PROVIDER_NAME,
+        input_providers=_SUPPORTED_INPUT_PROVIDERS,
+        output_providers=_SUPPORTED_OUTPUT_PROVIDERS,
+        logging_providers=_SUPPORTED_LOGGING_PROVIDERS,
+    )
+
+    # Prepare and submit jobs.
+    launched_tasks = []
+    requests = []
+
+    for task_view in job_model.task_view_generator(job_descriptor):
+
+      job_params = task_view.job_params
+      task_params = task_view.task_descriptors[0].task_params
+
+      outputs = job_params['outputs'] | task_params['outputs']
+      if skip_if_output_present:
+        # check whether the output's already there
+        if dsub_util.outputs_are_present(outputs, self._storage_service):
+          print('Skipping task because its outputs are present')
+          continue
+
+      request = self._create_batch_request(task_view)
+      if self._dry_run:
+        requests.append(request)
+      else:
+        task_id = self._submit_batch_job(request)
+        launched_tasks.append(task_id)
+
+    # If this is a dry-run, emit all the batch request objects
+    if self._dry_run:
+      # Each request is a google.cloud.batch_v1.types.batch.CreateJobRequest
+      # object. The __repr__ method for this object outputs something that
+      # closely resembles yaml, but can't actually be serialized into yaml.
+      # Ideally, we could serialize these request objects to yaml or json.
+      print(requests)
+
+    if not requests and not launched_tasks:
+      return {'job-id': dsub_util.NO_JOB}
+
+    return {
+        'job-id': job_descriptor.job_metadata['job-id'],
+        'user-id': job_descriptor.job_metadata.get('user-id'),
+        'task-id': [task_id for task_id in launched_tasks if task_id],
+    }
+
+  def delete_jobs(
+      self,
+      user_ids,
+      job_ids,
+      task_ids,
+      labels,
+      create_time_min=None,
+      create_time_max=None,
+  ):
+    """Kills the operations associated with the specified job or job.task.
+
+    Args:
+      user_ids: List of user ids who "own" the job(s) to cancel.
+      job_ids: List of job_ids to cancel.
+      task_ids: List of task-ids to cancel.
+      labels: List of LabelParam, each must match the job(s) to be canceled.
+      create_time_min: a timezone-aware datetime value for the earliest create
+        time of a task, inclusive.
+      create_time_max: a timezone-aware datetime value for the most recent
+        create time of a task, inclusive.
+
+    Returns:
+      A list of tasks canceled and a list of error messages.
+    """
+    # Look up the job(s)
+    tasks = list(
+        self.lookup_job_tasks(
+            {'RUNNING'},
+            user_ids=user_ids,
+            job_ids=job_ids,
+            task_ids=task_ids,
+            labels=labels,
+            create_time_min=create_time_min,
+            create_time_max=create_time_max,
+        )
+    )
+
+    print('Found %d tasks to delete.' % len(tasks))
+    return google_base.cancel(
+        self._batch_handler_def(), self._operations_cancel_api_def(), tasks
+    )
+
+  def lookup_job_tasks(
+      self,
+      statuses: Set[str],
+      user_ids=None,
+      job_ids=None,
+      job_names=None,
+      task_ids=None,
+      task_attempts=None,
+      labels=None,
+      create_time_min=None,
+      create_time_max=None,
+      max_tasks=0,
+      page_size=0,
+  ):
+    client = batch_v1.BatchServiceClient()
+    ops_filter = self._build_query_filter(
+        statuses,
+        user_ids,
+        job_ids,
+        job_names,
+        task_ids,
+        task_attempts,
+        labels,
+        create_time_min,
+        create_time_max,
+    )
+    # Initialize request argument(s)
+    request = batch_v1.ListJobsRequest(
+        parent=f'projects/{self._project}/locations/{self._location}',
+        filter=ops_filter,
+    )
+
+    # Make the request
+    response = client.list_jobs(request=request)
+    # Sort the operations by create-time to match sort of other providers
+    operations = [GoogleBatchOperation(page) for page in response]
+    operations.sort(key=lambda op: op.get_field('create-time'), reverse=True)
+    for op in operations:
+      yield op
+
+  def get_tasks_completion_messages(self, tasks):
+    # TODO: This needs to return a list of error messages for each task
+    pass
\ No newline at end of file
diff --git a/dsub/providers/.ipynb_checkpoints/google_utils-checkpoint.py b/dsub/providers/.ipynb_checkpoints/google_utils-checkpoint.py
new file mode 100644
index 0000000..6a47e96
--- /dev/null
+++ b/dsub/providers/.ipynb_checkpoints/google_utils-checkpoint.py
@@ -0,0 +1,502 @@
+# Copyright 2022 Verily Life Sciences Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions to be used by the Google providers.
+
+This module holds constants and methods useful to google-cls-v2
+and google-batch providers.
+"""
+import os
+import textwrap
+from typing import Dict
+
+from . import base
+
+from ..lib import job_model
+from ..lib import providers_util
+
+STATUS_FILTER_MAP = {
+    'RUNNING': 'status.state="RUNNING" OR status.state="QUEUED" OR status.state="SCHEDULED"',
+    'CANCELED': 'status.state="CANCELLED"',
+    'FAILURE': 'status.state="FAILED"',
+    'SUCCESS': 'status.state="SUCCEEDED"',
+}
+
+
+def prepare_query_label_value(labels):
+  """Converts the label strings to contain label-appropriate characters.
+
+  Args:
+    labels: A set of strings to be converted.
+
+  Returns:
+    A list of converted strings.
+  """
+  if not labels:
+    return None
+  return [job_model.convert_to_label_chars(label) for label in labels]
+
+
+def label_filter(label_key, label_value):
+  """Return a valid label filter for operations.list()."""
+  return 'labels."{}" = "{}"'.format(label_key, label_value)
+
+
+def create_time_filter(create_time, comparator):
+  """Return a valid createTime filter for operations.list()."""
+  return 'createTime {} "{}"'.format(comparator, create_time.isoformat())
+
+
+# Generate command to create the directories for the dsub user environment
+# pylint: disable=g-complex-comprehension
+def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
+                              working_dir: str) -> str:
+  return '\n'.join('mkdir -m 777 -p "%s" ' % dir
+                   for dir in [script_dir, tmp_dir, working_dir])
+
+
+# pylint: enable=g-complex-comprehension
+
+
+# Action steps that interact with GCS need gsutil and Python.
+# Use the 'slim' variant of the cloud-sdk image as it is much smaller.
+CLOUD_SDK_IMAGE = 'gcr.io/google.com/cloudsdktool/cloud-sdk:294.0.0-slim'
+
+# Name of the data disk
+DATA_DISK_NAME = 'datadisk'
+
+# Define a bash function for "echo" that includes timestamps
+LOG_MSG_FN = textwrap.dedent("""\
+  function get_datestamp() {
+    date "+%Y-%m-%d %H:%M:%S"
+  }
+
+  function log_info() {
+    echo "$(get_datestamp) INFO: $@"
+  }
+
+  function log_warning() {
+    1>&2 echo "$(get_datestamp) WARNING: $@"
+  }
+
+  function log_error() {
+    1>&2 echo "$(get_datestamp) ERROR: $@"
+  }
+""")
+
+# Define a bash function for "gsutil cp" to be used by the logging,
+# localization, and delocalization actions.
+GSUTIL_CP_FN = textwrap.dedent("""\
+  function gsutil_cp() {
+    local src="${1}"
+    local dst="${2}"
+    local content_type="${3}"
+    local user_project_name="${4}"
+
+    local headers=""
+    if [[ -n "${content_type}" ]]; then
+      headers="-h Content-Type:${content_type}"
+    fi
+
+    local user_project_flag=""
+    if [[ -n "${user_project_name}" ]]; then
+      user_project_flag="-u ${user_project_name}"
+    fi
+
+    local attempt
+    for ((attempt = 0; attempt < 4; attempt++)); do
+      log_info "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+      if gsutil ${headers} ${user_project_flag} -mq cp "${src}" "${dst}"; then
+        return
+      fi
+      if (( attempt < 3 )); then
+        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
+        log_warning "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+        sleep 10s
+      fi
+    done
+
+    log_error "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+    exit 1
+  }
+""")
+
+LOG_CP_FN = GSUTIL_CP_FN + textwrap.dedent("""\
+
+  function log_cp() {
+    local src="${1}"
+    local dst="${2}"
+    local tmp="${3}"
+    local check_src="${4}"
+    local user_project_name="${5}"
+
+    if [[ "${check_src}" == "true" ]] && [[ ! -e "${src}" ]]; then
+      return
+    fi
+
+    # Copy the log files to a local temporary location so that our "gsutil cp" is never
+    # executed on a file that is changing.
+
+    local tmp_path="${tmp}/$(basename ${src})"
+    cp "${src}" "${tmp_path}"
+
+    gsutil_cp "${tmp_path}" "${dst}" "text/plain" "${user_project_name}"
+  }
+""")
+
+# Define a bash function for "gsutil rsync" to be used by the logging,
+# localization, and delocalization actions.
+GSUTIL_RSYNC_FN = textwrap.dedent("""\
+  function gsutil_rsync() {
+    local src="${1}"
+    local dst="${2}"
+    local user_project_name="${3}"
+
+    local user_project_flag=""
+    if [[ -n "${user_project_name}" ]]; then
+      user_project_flag="-u ${user_project_name}"
+    fi
+
+    local attempt
+    for ((attempt = 0; attempt < 4; attempt++)); do
+      log_info "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+      if gsutil ${user_project_flag} -mq rsync -r "${src}" "${dst}"; then
+        return
+      fi
+      if (( attempt < 3 )); then
+        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
+        log_warning "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+        sleep 10s
+      fi
+    done
+
+    log_error "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+    exit 1
+  }
+""")
+
+LOCALIZATION_LOOP = textwrap.dedent("""\
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  for ((i=0; i < INPUT_COUNT; i++)); do
+    INPUT_VAR="INPUT_${i}"
+    INPUT_RECURSIVE="INPUT_RECURSIVE_${i}"
+    INPUT_SRC="INPUT_SRC_${i}"
+    INPUT_DST="INPUT_DST_${i}"
+
+    log_info "Localizing ${!INPUT_VAR}"
+    if [[ "${!INPUT_RECURSIVE}" -eq "1" ]]; then
+      gsutil_rsync "${!INPUT_SRC}" "${!INPUT_DST}" "${USER_PROJECT}"
+    else
+      gsutil_cp "${!INPUT_SRC}" "${!INPUT_DST}" "" "${USER_PROJECT}"
+    fi
+  done
+""")
+
+DELOCALIZATION_LOOP = textwrap.dedent("""\
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  for ((i=0; i < OUTPUT_COUNT; i++)); do
+    OUTPUT_VAR="OUTPUT_${i}"
+    OUTPUT_RECURSIVE="OUTPUT_RECURSIVE_${i}"
+    OUTPUT_SRC="OUTPUT_SRC_${i}"
+    OUTPUT_DST="OUTPUT_DST_${i}"
+
+    log_info "Delocalizing ${!OUTPUT_VAR}"
+    if [[ "${!OUTPUT_RECURSIVE}" -eq "1" ]]; then
+      gsutil_rsync "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "${USER_PROJECT}"
+    else
+      gsutil_cp "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "" "${USER_PROJECT}"
+    fi
+  done
+""")
+
+LOCALIZATION_CMD = textwrap.dedent("""\
+  {log_msg_fn}
+  {recursive_cp_fn}
+  {cp_fn}
+
+  {cp_loop}
+""")
+
+# The user's script or command is made available to the container in
+#   /mnt/data/script/<script-name>
+#
+# To get it there, it is passed in through the environment in the "prepare"
+# action and "echo"-ed to a file.
+#
+# Google APIs use Docker environment files which do not support
+# multi-line environment variables, so we encode the script using Python's
+# repr() function and then decoded it using ast.literal_eval().
+# This has the advantage over other encoding schemes (such as base64) of being
+# user-readable in the LifeSciences "operation" or Batch "Job" object.
+SCRIPT_VARNAME = '_SCRIPT_REPR'
+META_YAML_VARNAME = '_META_YAML_REPR'
+
+PYTHON_DECODE_SCRIPT = textwrap.dedent("""\
+  import ast
+  import sys
+
+  sys.stdout.write(ast.literal_eval(sys.stdin.read()))
+""")
+
+MK_IO_DIRS = textwrap.dedent("""\
+  for ((i=0; i < DIR_COUNT; i++)); do
+    DIR_VAR="DIR_${i}"
+
+    log_info "mkdir -m 777 -p \"${!DIR_VAR}\""
+    mkdir -m 777 -p "${!DIR_VAR}"
+  done
+""")
+
+PREPARE_CMD = textwrap.dedent("""\
+  #!/bin/bash
+
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  {log_msg_fn}
+  {mk_runtime_dirs}
+
+  echo "${{{script_var}}}" \
+    | python3 -c '{python_decode_script}' \
+    > "{script_path}"
+  chmod a+x "{script_path}"
+
+  {mk_io_dirs}
+""")
+
+USER_CMD = textwrap.dedent("""\
+  export TMPDIR="{tmp_dir}"
+  cd {working_dir}
+
+  "{user_script}"
+""")
+
+
+class GoogleJobProviderBase(base.JobProvider):
+  """dsub provider implementation managing Jobs on Google Cloud."""
+
+  def _get_prepare_env(self, script, job_descriptor, inputs, outputs, mounts,
+                       mount_point) -> Dict[str, str]:
+    """Return a dict with variables for the 'prepare' action."""
+
+    # Add the _SCRIPT_REPR with the repr(script) contents
+    # Add the _META_YAML_REPR with the repr(meta) contents
+
+    # Add variables for directories that need to be created, for example:
+    # DIR_COUNT: 2
+    # DIR_0: /mnt/data/input/gs/bucket/path1/
+    # DIR_1: /mnt/data/output/gs/bucket/path2
+
+    # List the directories in sorted order so that they are created in that
+    # order. This is primarily to ensure that permissions are set as we create
+    # each directory.
+    # For example:
+    #   mkdir -m 777 -p /root/first/second
+    #   mkdir -m 777 -p /root/first
+    # *may* not actually set 777 on /root/first
+
+    docker_paths = sorted([
+        var.docker_path if var.recursive else os.path.dirname(var.docker_path)
+        for var in inputs | outputs | mounts
+        if var.value
+    ])
+
+    env = {
+        SCRIPT_VARNAME: repr(script.value),
+        META_YAML_VARNAME: repr(job_descriptor.to_yaml()),
+        'DIR_COUNT': str(len(docker_paths))
+    }
+
+    for idx, path in enumerate(docker_paths):
+      env['DIR_{}'.format(idx)] = os.path.join(mount_point, path)
+
+    return env
+
+  def _get_localization_env(self, inputs, user_project,
+                            mount_point) -> Dict[str, str]:
+    """Return a dict with variables for the 'localization' action."""
+
+    # Add variables for paths that need to be localized, for example:
+    # INPUT_COUNT: 1
+    # INPUT_0: MY_INPUT_FILE
+    # INPUT_RECURSIVE_0: 0
+    # INPUT_SRC_0: gs://mybucket/mypath/myfile
+    # INPUT_DST_0: /mnt/data/inputs/mybucket/mypath/myfile
+
+    non_empty_inputs = [var for var in inputs if var.value]
+    env = {'INPUT_COUNT': str(len(non_empty_inputs))}
+
+    for idx, var in enumerate(non_empty_inputs):
+      env['INPUT_{}'.format(idx)] = var.name
+      env['INPUT_RECURSIVE_{}'.format(idx)] = str(int(var.recursive))
+      env['INPUT_SRC_{}'.format(idx)] = var.value
+
+      # For wildcard paths, the destination must be a directory
+      dst = os.path.join(mount_point, var.docker_path)
+      path, filename = os.path.split(dst)
+      if '*' in filename:
+        dst = '{}/'.format(path)
+      env['INPUT_DST_{}'.format(idx)] = dst
+
+    env['USER_PROJECT'] = user_project
+
+    return env
+
+  def _get_delocalization_env(self, outputs, user_project,
+                              mount_point) -> Dict[str, str]:
+    """Return a dict with variables for the 'delocalization' action."""
+
+    # Add variables for paths that need to be delocalized, for example:
+    # OUTPUT_COUNT: 1
+    # OUTPUT_0: MY_OUTPUT_FILE
+    # OUTPUT_RECURSIVE_0: 0
+    # OUTPUT_SRC_0: gs://mybucket/mypath/myfile
+    # OUTPUT_DST_0: /mnt/data/outputs/mybucket/mypath/myfile
+
+    non_empty_outputs = [var for var in outputs if var.value]
+    env = {'OUTPUT_COUNT': str(len(non_empty_outputs))}
+
+    for idx, var in enumerate(non_empty_outputs):
+      env['OUTPUT_{}'.format(idx)] = var.name
+      env['OUTPUT_RECURSIVE_{}'.format(idx)] = str(int(var.recursive))
+      env['OUTPUT_SRC_{}'.format(idx)] = os.path.join(mount_point,
+                                                      var.docker_path)
+
+      # For wildcard paths, the destination must be a directory
+      if '*' in var.uri.basename:
+        dst = var.uri.path
+      else:
+        dst = var.uri
+      env['OUTPUT_DST_{}'.format(idx)] = dst
+
+    env['USER_PROJECT'] = user_project
+
+    return env
+
+  def _build_user_environment(self, envs, inputs, outputs, mounts,
+                              mount_point) -> Dict[str, str]:
+    """Returns a dictionary of for the user container environment."""
+    envs = {env.name: env.value for env in envs}
+    envs.update(
+        providers_util.get_file_environment_variables(inputs, mount_point))
+    envs.update(
+        providers_util.get_file_environment_variables(outputs, mount_point))
+    envs.update(
+        providers_util.get_file_environment_variables(mounts, mount_point))
+    return envs
+
+  def prepare_job_metadata(self, script: str, job_name: str,
+                           user_id: str) -> Dict[str, str]:
+    return providers_util.prepare_job_metadata(script, job_name, user_id)
+
+  def _get_label_filters(self, label_key, values):
+    if not values or values == {'*'}:
+      return None
+
+    return [label_filter(label_key, v) for v in values]
+
+  def _get_labels_filters(self, labels):
+    if not labels:
+      return None
+
+    return [label_filter(l.name, l.value) for l in labels]
+
+  def _get_status_filters(self, statuses):
+    if not statuses or statuses == {'*'}:
+      return None
+
+    return [STATUS_FILTER_MAP[s] for s in statuses]
+
+  def _get_user_id_filter_value(self, user_ids):
+    if not user_ids or user_ids == {'*'}:
+      return None
+
+    return prepare_query_label_value(user_ids)
+
+  def _get_create_time_filters(self, create_time_min, create_time_max):
+    filters = []
+    for create_time, comparator in [(create_time_min, '>='),
+                                    (create_time_max, '<=')]:
+      if not create_time:
+        continue
+
+      filters.append(create_time_filter(create_time, comparator))
+    return filters
+
+  def _build_query_filter(self,
+                          statuses,
+                          user_ids=None,
+                          job_ids=None,
+                          job_names=None,
+                          task_ids=None,
+                          task_attempts=None,
+                          labels=None,
+                          create_time_min=None,
+                          create_time_max=None):
+    # The Google APIs allows for building fairly elaborate filter
+    # clauses. We can group (). We can AND, OR, and NOT.
+    #
+    # The first set of filters, labeled here as OR filters are elements
+    # where more than one value cannot be true at the same time. For example,
+    # an operation cannot have a status of both RUNNING and CANCELED.
+    #
+    # The second set of filters, labeled here as AND filters are elements
+    # where more than one value can be true. For example,
+    # an operation can have a label with key1=value2 AND key2=value2.
+
+    # Translate the semantic requests into a google-specific filter.
+
+    # 'OR' filtering arguments.
+    status_filters = self._get_status_filters(statuses)
+    user_id_filters = self._get_label_filters(
+        'user-id', self._get_user_id_filter_value(user_ids))
+    job_id_filters = self._get_label_filters('job-id', job_ids)
+    job_name_filters = self._get_label_filters(
+        'job-name', prepare_query_label_value(job_names))
+    task_id_filters = self._get_label_filters('task-id', task_ids)
+    task_attempt_filters = self._get_label_filters('task-attempt',
+                                                   task_attempts)
+    # 'AND' filtering arguments.
+    label_filters = self._get_labels_filters(labels)
+    create_time_filters = self._get_create_time_filters(create_time_min,
+                                                        create_time_max)
+
+    if job_id_filters and job_name_filters:
+      raise ValueError(
+          'Filtering by both job IDs and job names is not supported')
+
+    # Now build up the full text filter.
+    # OR all of the OR filter arguments together.
+    # AND all of the AND arguments together.
+    or_arguments = []
+    for or_filters in [
+        status_filters, user_id_filters, job_id_filters, job_name_filters,
+        task_id_filters, task_attempt_filters
+    ]:
+      if or_filters:
+        or_arguments.append('(' + ' OR '.join(or_filters) + ')')
+
+    and_arguments = []
+    for and_filters in [label_filters, create_time_filters]:
+      if and_filters:
+        and_arguments.append('(' + ' AND '.join(and_filters) + ')')
+
+    # Now and all of these arguments together.
+    return ' AND '.join(or_arguments + and_arguments)
diff --git a/dsub/providers/google_batch.py b/dsub/providers/google_batch.py
index b96596b..2938730 100644
--- a/dsub/providers/google_batch.py
+++ b/dsub/providers/google_batch.py
@@ -189,7 +189,7 @@ def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None
 
   # Prep the log filter script
   echo "${{{log_filter_var}}}" \
-    | python -c '{python_decode_script}' \
+    | python3 -c '{python_decode_script}' \
     > "{log_filter_script_path}"
   chmod a+x "{log_filter_script_path}"
 
diff --git a/dsub/providers/google_utils.py b/dsub/providers/google_utils.py
index 8b9862f..a73a901 100644
--- a/dsub/providers/google_utils.py
+++ b/dsub/providers/google_utils.py
@@ -274,7 +274,7 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
   {mk_runtime_dirs}
 
   echo "${{{script_var}}}" \
-    | python -c '{python_decode_script}' \
+    | python3 -c '{python_decode_script}' \
     > "{script_path}"
   chmod a+x "{script_path}"
 

From ae679601166af4bcc9cdb08a297fe3a21216e2a0 Mon Sep 17 00:00:00 2001
From: Kristen Liu <kristenliu@verily.health>
Date: Fri, 21 Aug 2026 06:11:11 +0000
Subject: [PATCH 04/10] Modified examples to use google-batch

---
 examples/custom_scripts/README.md             |   8 +-
 examples/custom_scripts/submit_list.sh        |   2 +-
 examples/custom_scripts/submit_one.sh         |   2 +-
 .../.ipynb_checkpoints/README-checkpoint.md   | 167 ++++++++++++++++++
 examples/decompress/README.md                 |   8 +-
 examples/decompress/submit_list.sh            |   4 +-
 examples/decompress/submit_one.sh             |   4 +-
 examples/fastqc/README.md                     |  16 +-
 examples/fastqc/submit_list.sh                |   4 +-
 examples/fastqc/submit_one.sh                 |   4 +-
 examples/samtools/README.md                   |  16 +-
 examples/samtools/submit_list.sh              |   4 +-
 examples/samtools/submit_one.sh               |   4 +-
 13 files changed, 205 insertions(+), 38 deletions(-)
 create mode 100644 examples/decompress/.ipynb_checkpoints/README-checkpoint.md

diff --git a/examples/custom_scripts/README.md b/examples/custom_scripts/README.md
index 734f057..c471591 100644
--- a/examples/custom_scripts/README.md
+++ b/examples/custom_scripts/README.md
@@ -55,7 +55,7 @@ To run a Bash script to decompress the VCF file, type:
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
   --regions "us-central1" \
   --logging "gs://MY-BUCKET/get_vcf_sample_ids.sh/logging" \
@@ -134,7 +134,7 @@ To run a Python script to decompress the VCF file, type:
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
   --regions "us-central1" \
   --logging "gs://MY-BUCKET/get_vcf_sample_ids.py/logging" \
@@ -221,7 +221,7 @@ Run either of the following commands:
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
   --regions "us-central1" \
   --logging "gs://MY-BUCKET/get_vcf_sample_ids/logging" \
@@ -234,7 +234,7 @@ dsub \
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
   --regions "us-central1" \
   --logging "gs://MY-BUCKET/get_vcf_sample_ids/logging" \
diff --git a/examples/custom_scripts/submit_list.sh b/examples/custom_scripts/submit_list.sh
index 4168dc3..754a05b 100755
--- a/examples/custom_scripts/submit_list.sh
+++ b/examples/custom_scripts/submit_list.sh
@@ -57,7 +57,7 @@ echo
 
 # Launch the task
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project "${MY_PROJECT}" \
   --regions "us-central1" \
   --logging "${LOGGING}" \
diff --git a/examples/custom_scripts/submit_one.sh b/examples/custom_scripts/submit_one.sh
index 7fb41b6..f6c15a3 100755
--- a/examples/custom_scripts/submit_one.sh
+++ b/examples/custom_scripts/submit_one.sh
@@ -60,7 +60,7 @@ echo
 
 # Launch the task
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project "${MY_PROJECT}" \
   --regions "us-central1" \
   --logging "${LOGGING}" \
diff --git a/examples/decompress/.ipynb_checkpoints/README-checkpoint.md b/examples/decompress/.ipynb_checkpoints/README-checkpoint.md
new file mode 100644
index 0000000..00a15a2
--- /dev/null
+++ b/examples/decompress/.ipynb_checkpoints/README-checkpoint.md
@@ -0,0 +1,167 @@
+# Decompress with dsub
+
+This example demonstrates how to decompress files stored in a Google
+Cloud Storage bucket by submitting a simple command from a shell prompt
+on your laptop. The job executes in the cloud. As input, we start with a
+single compressed variant call format (VCF) file from the
+[1000 Genomes Project](http://www.internationalgenome.org/).
+
+We then proceed to an example that demonstrates processing multiple files,
+using a small list of VCFs.
+All of the source VCF files are stored in a public bucket at
+[gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/](https://console.cloud.google.com/storage/browser/genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/):
+
+*   20130723_phase3_wg/cornell/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf.gz
+*   20140708_previous_phase3/v2_vcfs/ALL.chr21.phase3_shapeit2_mvncall_integrated_v2.20130502.genotypes.vcf.gz
+*   20140708_previous_phase3/v1_vcfs/ALL.chr21.phase3_shapeit2_mvncall_integrated.20130502.genotype.vcf.gz
+*   20110721_exome_call_sets/bcm/ALL.BCM_Illumina_Mosaik_ontarget_plus50bp_822.20110521.snp.exome.genotypes.vcf.gz
+
+## Setup
+
+* Follow the [dsub geting started](../../README.md#getting-started)
+instructions.
+
+## Decompress one file
+
+### Submit the job
+
+The following command will submit a job to decompress the first input file
+from the list above and write the decompressed file to a Cloud Storage bucket
+you have write access to.
+
+To run a command to decompress the VCF file, type:
+
+```
+dsub \
+  --provider google-batch \
+  --project MY-PROJECT \
+  --regions us-central1 \
+  --logging "gs://MY-BUCKET/decompress_one/logging" \
+  --disk-size 200 \
+  --image ubuntu:14.04 \
+  --input INPUT_VCF="gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/20130723_phase3_wg/cornell/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf.gz" \
+  --output OUTPUT_VCF="gs://MY-BUCKET/decompress_one/output/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf" \
+  --command 'gunzip ${INPUT_VCF} && \
+             mv ${INPUT_VCF%.gz} $(dirname ${OUTPUT_VCF})' \
+  --wait
+```
+
+Set MY-PROJECT to your cloud project name, and set MY-BUCKET to a cloud bucket
+on which you have write privileges.
+
+You should see output like:
+
+```
+Job: gunzip--<userid>--170224-114336-37
+Launched job-id: gunzip--<userid>--170224-114336-37
+        user-id: <userid>
+Waiting for jobs to complete...
+```
+
+Because the `--wait` flag was set, `dsub` will block until the job completes.
+
+### Check the results
+
+To list the output, use the command:
+
+```
+gsutil ls gs://MY-BUCKET/decompress_one/output
+```
+
+Output should look like:
+
+```
+gs://MY-BUCKET/decompress_one/output/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf
+```
+
+To see the first few lines of the decompressed file, run:
+
+```
+gsutil cat gs://MY-BUCKET/decompress_one/output/*.vcf | head -n 5
+```
+
+Output should look like:
+
+```
+##fileformat=VCFv4.1
+##FILTER=<ID=LowQual,Description="Low quality">
+##FILTER=<ID=PASS,Description="Passing basic quality fiters">
+##FORMAT=<ID=AD,Number=.,Type=Integer,Description="Allelic depths for the ref and alt alleles in the order listed">
+##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Approximate read depth (reads with MQ=255 or with bad mates are filtered)">
+```
+
+## Decompress multiple files
+
+`dsub` allows you to define a batch of tasks to submit together using a
+tab-separated values (TSV) file listing the inputs and outputs.
+Each line lists the inputs and outputs for a separate task.
+
+More on dsub batch jobs can be found in the
+[README](../../README#submitting-a-batch-job).
+
+### Create a TSV file
+
+Open an editor and create a file `submit_list.tsv`:
+
+<pre>
+--input INPUT_VCF&#9;--output OUTPUT_VCF
+gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/20140708_previous_phase3/v2_vcfs/ALL.chr21.phase3_shapeit2_mvncall_integrated_v2.20130502.genotypes.vcf.gz&#9;gs://MY-BUCKET/decompress_list/output/*.vcf
+gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/20140708_previous_phase3/v1_vcfs/ALL.chr21.phase3_shapeit2_mvncall_integrated.20130502.genotype.vcf.gz&#9;gs://MY-BUCKET/decompress_list/output/*.vcf
+gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/20110721_exome_call_sets/bcm/ALL.BCM_Illumina_Mosaik_ontarget_plus50bp_822.20110521.snp.exome.genotypes.vcf.gz&#9;gs://MY-BUCKET/decompress_list/output/*.vcf
+</pre>
+
+The first line of the file lists the input and output parameter names.
+Each subsequent line lists the parameter values.
+Replace MY-BUCKET with a Cloud bucket on which you have write privileges.
+
+Note that for the output parameter, for simplicity, we used wildcards to match
+the 1 VCF file each task outputs instead of explicitly listing the complete
+output file name.
+
+### Submit the job
+
+```
+dsub \
+  --provider google-batch \
+  --project MY-PROJECT \
+  --regions us-central1 \
+  --logging "gs://MY-BUCKET/decompress_list/logging" \
+  --disk-size 200 \
+  --image ubuntu:14.04 \
+  --command 'gunzip ${INPUT_VCF} && \
+             mv ${INPUT_VCF%.gz} $(dirname ${OUTPUT_VCF})' \
+  --tasks submit_list.tsv \
+  --wait
+```
+
+Output should look like:
+
+```
+Job: gunzip--<userid>--170224-122223-54
+Launched job-id: gunzip--<userid>--170224-122223-54
+        user-id: <userid>
+  Task: task-1
+  Task: task-2
+  Task: task-3
+Waiting for jobs to complete...
+```
+
+when all tasks for the job have completed, `dsub` will exit.
+
+### Check the results
+
+To list the output objects, use the command:
+
+```
+gsutil ls gs://MY-BUCKET/decompress_list/output
+```
+
+Output should look like:
+
+```
+gs://MY-BUCKET/decompress_list/output/ALL.BCM_Illumina_Mosaik_ontarget_plus50bp_822.20110521.snp.exome.genotypes.vcf
+gs://MY-BUCKET/decompress_list/output/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf
+gs://MY-BUCKET/decompress_list/output/ALL.chr21.phase3_shapeit2_mvncall_integrated.20130502.genotype.vcf
+gs://MY-BUCKET/decompress_list/output/ALL.chr21.phase3_shapeit2_mvncall_integrated_v2.20130502.genotypes.vcf
+```
+
diff --git a/examples/decompress/README.md b/examples/decompress/README.md
index c319cf7..f8750ef 100644
--- a/examples/decompress/README.md
+++ b/examples/decompress/README.md
@@ -33,9 +33,9 @@ To run a command to decompress the VCF file, type:
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "gs://MY-BUCKET/decompress_one/logging" \
   --disk-size 200 \
   --image ubuntu:14.04 \
@@ -122,9 +122,9 @@ output file name.
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "gs://MY-BUCKET/decompress_list/logging" \
   --disk-size 200 \
   --image ubuntu:14.04 \
diff --git a/examples/decompress/submit_list.sh b/examples/decompress/submit_list.sh
index 334d6d9..4a880c1 100755
--- a/examples/decompress/submit_list.sh
+++ b/examples/decompress/submit_list.sh
@@ -34,9 +34,9 @@ readonly SCRIPT_DIR="$(dirname "${0}")"
 
 # Launch the task
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project "${MY_PROJECT}" \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "${MY_BUCKET}/decompress_list/logging/" \
   --disk-size 200 \
   --image ubuntu:14.04 \
diff --git a/examples/decompress/submit_one.sh b/examples/decompress/submit_one.sh
index 0870e61..ff85417 100755
--- a/examples/decompress/submit_one.sh
+++ b/examples/decompress/submit_one.sh
@@ -35,9 +35,9 @@ readonly SCRIPT_DIR="$(dirname "${0}")"
 
 # Launch the task
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project "${MY_PROJECT}" \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "${MY_BUCKET_PATH}"/logging/ \
   --disk-size 200 \
   --image ubuntu:14.04 \
diff --git a/examples/fastqc/README.md b/examples/fastqc/README.md
index c9ddea0..a50e227 100644
--- a/examples/fastqc/README.md
+++ b/examples/fastqc/README.md
@@ -77,9 +77,9 @@ To run FastQC on the BAM file, type:
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "gs://MY-BUCKET/fastqc/submit_one/logging" \
   --disk-size 200 \
   --name "fastqc" \
@@ -99,9 +99,9 @@ You should see output like:
 Job: fastqc--<userid>--170619-105212-67
 Launched job-id: fastqc--<userid>--170619-105212-67
 To check the status, run:
-  dstat --provider google-cls-v2 --project MY-PROJECT --jobs fastqc--<userid>--170619-105212-67 --status '*'
+  dstat --provider google-batch --project MY-PROJECT --jobs fastqc--<userid>--170619-105212-67 --status '*'
 To cancel the job, run:
-  ddel --provider google-cls-v2 --project MY-PROJECT --jobs fastqc--<userid>--170619-105212-67
+  ddel --provider google-batch --project MY-PROJECT --jobs fastqc--<userid>--170619-105212-67
 Waiting for job to complete...
 Waiting for: fastqc--<userid>--170619-105212-67.
 ```
@@ -156,9 +156,9 @@ output file name.
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "gs://MY-BUCKET/samtools/submit_list/logging" \
   --disk-size 200 \
   --name "fastqc" \
@@ -175,9 +175,9 @@ Job: fastqc--<userid>--170522-154943-70
 Launched job-id: fastqc--<userid>--170522-154943-70
 3 task(s)
 To check the status, run:
-  dstat --provider google-cls-v2 --project MY-PROJECT --jobs fastqc--<userid>--170522-154943-70 --status '*'
+  dstat --provider google-batch --project MY-PROJECT --jobs fastqc--<userid>--170522-154943-70 --status '*'
 To cancel the job, run:
-  ddel --provider google-cls-v2 --project MY-PROJECT --jobs fastqc--<userid>--170522-154943-70
+  ddel --provider google-batch --project MY-PROJECT --jobs fastqc--<userid>--170522-154943-70
 Waiting for job to complete...
 Waiting for: fastqc--<userid>--170522-154943-70.
 ```
diff --git a/examples/fastqc/submit_list.sh b/examples/fastqc/submit_list.sh
index 2421cf3..8163aac 100755
--- a/examples/fastqc/submit_list.sh
+++ b/examples/fastqc/submit_list.sh
@@ -40,9 +40,9 @@ gcloud builds submit "${SCRIPT_DIR}" \
 
 # Launch the task
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project "${MY_PROJECT}" \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "${OUTPUT_ROOT}/logging/" \
   --disk-size 200 \
   --name "fastqc" \
diff --git a/examples/fastqc/submit_one.sh b/examples/fastqc/submit_one.sh
index bb848fb..9b4141e 100755
--- a/examples/fastqc/submit_one.sh
+++ b/examples/fastqc/submit_one.sh
@@ -41,9 +41,9 @@ gcloud builds submit "${SCRIPT_DIR}" \
 
 # Launch the task
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project "${MY_PROJECT}" \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "${OUTPUT_ROOT}/logging" \
   --disk-size 200 \
   --name "fastqc" \
diff --git a/examples/samtools/README.md b/examples/samtools/README.md
index eeb7beb..93e30de 100644
--- a/examples/samtools/README.md
+++ b/examples/samtools/README.md
@@ -38,9 +38,9 @@ To run a command to index the BAM file, type:
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "gs://MY-BUCKET/samtools/submit_one/logging" \
   --disk-size 200 \
   --name "samtools index" \
@@ -63,9 +63,9 @@ You should see output like:
 Job: samtools-i--<userid>--170522-153810-14
 Launched job-id: samtools-i--<userid>--170522-153810-14
 To check the status, run:
-  dstat --provider google-cls-v2 --project MY-PROJECT --jobs samtools-i--<userid>--170522-153810-14 --status '*'
+  dstat --provider google-batch --project MY-PROJECT --jobs samtools-i--<userid>--170522-153810-14 --status '*'
 To cancel the job, run:
-  ddel --provider google-cls-v2 --project MY-PROJECT --jobs samtools-i--<userid>--170522-153810-14
+  ddel --provider google-batch --project MY-PROJECT --jobs samtools-i--<userid>--170522-153810-14
 Waiting for job to complete...
 Waiting for: samtools-i--<userid>--170522-153810-14.
 ```
@@ -119,9 +119,9 @@ output file name.
 
 ```
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project MY-PROJECT \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "gs://MY-BUCKET/samtools/submit_list/logging" \
   --disk-size 200 \
   --name "samtools index" \
@@ -141,9 +141,9 @@ Job: samtools-i--<userid>--170522-154943-70
 Launched job-id: samtools-i--<userid>--170522-154943-70
 3 task(s)
 To check the status, run:
-  dstat --provider google-cls-v2 --project MY-PROJECT --jobs samtools-i--<userid>--170522-154943-70 --status '*'
+  dstat --provider google-batch --project MY-PROJECT --jobs samtools-i--<userid>--170522-154943-70 --status '*'
 To cancel the job, run:
-  ddel --provider google-cls-v2 --project MY-PROJECT --jobs samtools-i--<userid>--170522-154943-70
+  ddel --provider google-batch --project MY-PROJECT --jobs samtools-i--<userid>--170522-154943-70
 Waiting for job to complete...
 Waiting for: samtools-i--<userid>--170522-154943-70.
 ```
diff --git a/examples/samtools/submit_list.sh b/examples/samtools/submit_list.sh
index 82ca243..77efdaf 100755
--- a/examples/samtools/submit_list.sh
+++ b/examples/samtools/submit_list.sh
@@ -36,9 +36,9 @@ readonly SCRIPT_DIR="$(dirname "${0}")"
 
 # Launch the task
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project "${MY_PROJECT}" \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "${OUTPUT_ROOT}/logging/" \
   --disk-size 200 \
   --name "samtools index" \
diff --git a/examples/samtools/submit_one.sh b/examples/samtools/submit_one.sh
index c3e46ed..4a322a8 100755
--- a/examples/samtools/submit_one.sh
+++ b/examples/samtools/submit_one.sh
@@ -37,9 +37,9 @@ readonly SCRIPT_DIR="$(dirname "${0}")"
 
 # Launch the task
 dsub \
-  --provider google-cls-v2 \
+  --provider google-batch \
   --project "${MY_PROJECT}" \
-  --zones "us-central1-*" \
+  --regions us-central1 \
   --logging "${OUTPUT_ROOT}"/logging \
   --disk-size 200 \
   --name "samtools index" \

From 72db0c3be25ef990c438dc596bfeac18816dd8e9 Mon Sep 17 00:00:00 2001
From: "Kristen Liu (Ong)" <43296048+kvo3@users.noreply.github.com>
Date: Fri, 21 Aug 2026 09:34:11 -0700
Subject: [PATCH 05/10] Delete
 dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py

---
 .../google_batch-checkpoint.py                | 1068 -----------------
 1 file changed, 1068 deletions(-)
 delete mode 100644 dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py

diff --git a/dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py b/dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py
deleted file mode 100644
index 0323f81..0000000
--- a/dsub/providers/.ipynb_checkpoints/google_batch-checkpoint.py
+++ /dev/null
@@ -1,1068 +0,0 @@
-# Copyright 2022 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Provider for running jobs on Google Cloud Platform.
-
-This module implements job creation, listing, and canceling using the
-Google Batch v1 APIs.
-"""
-
-import ast
-import operator
-import os
-import re
-import sys
-import textwrap
-from typing import Dict, List, Set
-
-from ..lib import dsub_util
-from ..lib import job_model
-from ..lib import param_util
-from ..lib import providers_util
-from . import base
-from . import google_base
-from . import google_batch_operations
-from . import google_custom_machine
-from . import google_utils
-
-# pylint: disable=g-import-not-at-top
-try:
-  from google.cloud import batch_v1
-except ImportError:
-  # TODO: Remove conditional import when batch library is available
-  from . import batch_dummy as batch_v1
-# pylint: enable=g-import-not-at-top
-_PROVIDER_NAME = 'google-batch'
-# Index of the prepare action in the runnable list
-_PREPARE_INDEX = 1
-
-# Create file provider whitelist.
-_SUPPORTED_FILE_PROVIDERS = frozenset([job_model.P_GCS])
-_SUPPORTED_LOGGING_PROVIDERS = _SUPPORTED_FILE_PROVIDERS
-_SUPPORTED_INPUT_PROVIDERS = _SUPPORTED_FILE_PROVIDERS
-_SUPPORTED_OUTPUT_PROVIDERS = _SUPPORTED_FILE_PROVIDERS
-
-# Mount point for the data disk in the user's Docker container
-_VOLUME_MOUNT_POINT = '/mnt/disks/data'
-_DATA_MOUNT_POINT = '/mnt/data'
-
-# These are documented (providers/README.md) as being read/write to user
-# commands.
-_SCRIPT_DIR = f'{_DATA_MOUNT_POINT}/script'
-_TMP_DIR = f'{_DATA_MOUNT_POINT}/tmp'
-_WORKING_DIR = f'{_DATA_MOUNT_POINT}/workingdir'
-
-# These are visible to the user task; not yet documented, as we'd *like* to
-# find a way to have them visible only to the logging tasks.
-_BATCH_LOG_DIR = f'{_VOLUME_MOUNT_POINT}/.logging'
-_LOGGING_DIR = f'{_DATA_MOUNT_POINT}/.logging'
-
-_LOG_FILTER_VAR = '_LOG_FILTER_REPR'
-_LOG_FILTER_SCRIPT_PATH = f'{_DATA_MOUNT_POINT}/.log_filter_script.py'
-
-# _LOG_FILTER_PYTHON is a block of Python code to execute in both the
-# "continuous_logging" and "final_logging" tasks.
-#
-# Batch API will eventually create three log files in the _BATCH_LOG_FILE_PATH
-# directory. They are named:
-#
-# - output-*.log
-# - stdout-*.log
-# - stderr-*.log
-#
-# We will be creating a "staging" location for each of these files.
-#
-# If any of the batch log files don't exist, touch the associated staging file.
-#
-# If the output batch log file exists, copy it directly to the staging
-# location.
-#
-# If the stdout/stderr batch log files exist, copy them to their staging
-# location. Then filter it so that only user-action logs exist and the prefixes
-# are removed. The prefixes look something like:
-# [batch_task_logs]<datetime> ERROR:
-#   [task_id:task/<job_uid>,runnable_index:<action_number>]
-
-# pylint: disable=anomalous-backslash-in-string
-_LOG_FILTER_PYTHON = textwrap.dedent(r"""
-import fileinput
-import glob
-import re
-import shutil
-import sys
-from pathlib import Path
-
-LOGGING_DIR = sys.argv[1]
-LOG_FILE_PATH = sys.argv[2]
-STDOUT_FILE_PATH = sys.argv[3]
-STDERR_FILE_PATH = sys.argv[4]
-USER_TASK = sys.argv[5]
-
-def filter_log_file(staging_path: str, stream_string: str):
-  # Replaces lines in file inplace
-  for line in fileinput.input(staging_path, inplace=True):
-    re_search_string = fr"^\[batch_task_logs\].*{stream_string}: \[task_id:task\/.*runnable_index:{USER_TASK}] (.*)"
-    match = re.search(re_search_string, line)
-    if match:
-      modified_line = match.group(1)
-      print(modified_line)
-
-def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None):
-  # Check if log files exist, and copy to their staging location
-  matching_files = list(Path(LOGGING_DIR).glob(glob_str))
-  if matching_files:
-    assert(len(matching_files) == 1)
-    shutil.copy(matching_files[0], staging_path)
-    if filter_str:
-      filter_log_file(staging_path, filter_str)
-  else:
-    Path(staging_path).touch()
-
-# We know the log file is named output-<job-uid>.log
-# and the stdout/stderr files are named stdout-<job-uid>.log and
-# stderr-<job-uid>.log
-copy_log_to_staging("output-*.log", LOG_FILE_PATH)
-copy_log_to_staging("stdout-*.log", STDOUT_FILE_PATH, filter_str="INFO")
-copy_log_to_staging("stderr-*.log", STDERR_FILE_PATH, filter_str="ERROR")
-""")
-# pylint: enable=anomalous-backslash-in-string
-
-_LOG_CP = textwrap.dedent("""
-  python3 "{log_filter_script_path}" \
-      "${{LOGGING_DIR}}" \
-      "${{LOGGING_DIR}}/log.txt" \
-      "${{LOGGING_DIR}}/stdout.txt" \
-      "${{LOGGING_DIR}}/stderr.txt" \
-      "{user_action}"
-
-  gsutil_cp "${{LOGGING_DIR}}/stdout.txt" "${{STDOUT_PATH}}" "text/plain" "${{USER_PROJECT}}" &
-  STDOUT_PID=$!
-  gsutil_cp "${{LOGGING_DIR}}/stderr.txt" "${{STDERR_PATH}}" "text/plain"  "${{USER_PROJECT}}" &
-  STDERR_PID=$!
-  gsutil_cp "${{LOGGING_DIR}}/log.txt" "${{LOGGING_PATH}}" "text/plain" "${{USER_PROJECT}}" &
-  LOG_PID=$!
-
-  wait "${{STDOUT_PID}}"
-  wait "${{STDERR_PID}}"
-  wait "${{LOG_PID}}"
-""")
-
-_FINAL_LOGGING_CMD = textwrap.dedent("""\
-  set -o errexit
-  set -o nounset
-  set -o pipefail
-
-  readonly LOGGING_DIR="{logging_dir}"
-
-  # Flag the continuous logging command to stop
-  touch "${{LOGGING_DIR}}/.stop_logging"
-
-  {log_msg_fn}
-  {gsutil_cp_fn}
-
-  {log_cp}
-""")
-
-# Keep logging until the final logging action starts
-_CONTINUOUS_LOGGING_CMD = textwrap.dedent("""\
-  set -o errexit
-  set -o nounset
-  set -o pipefail
-
-  readonly LOGGING_DIR="{logging_dir}"
-
-  {log_msg_fn}
-  {gsutil_cp_fn}
-
-  # Make sure the logging work directory exists
-  mkdir -p "${{LOGGING_DIR}}"
-
-  # Prep the log filter script
-  echo "${{{log_filter_var}}}" \
-    | python3 -c '{python_decode_script}' \
-    > "{log_filter_script_path}"
-  chmod a+x "{log_filter_script_path}"
-
-  while [[ ! -e "${{LOGGING_DIR}}/.stop_logging" ]]; do
-    {log_cp}
-
-    sleep "{log_interval}"
-  done
-""")
-
-
-_EVENT_REGEX_MAP = {
-    'scheduled': re.compile('^Job state is set from QUEUED to SCHEDULED'),
-    'start': re.compile('^Job state is set from SCHEDULED to RUNNING'),
-    'ok': re.compile('^Job state is set from RUNNING to SUCCEEDED'),
-    'fail': re.compile('^Job state is set from .+? to FAILED'),
-    'cancellation-in-progress': re.compile(
-        '^Job state is set from .+? to CANCELLATION_IN_PROGRESS'
-    ),
-    'canceled': re.compile('^Job state is set from .+? to CANCELLED'),
-}
-
-
-class GoogleBatchEventMap(object):
-  """Helper for extracing a set of normalized, filtered operation events."""
-
-  def __init__(self, op: batch_v1.types.Job):
-    self._op = op
-
-  def get_filtered_normalized_events(self):
-    """Map and filter the batch API events down to events of interest.
-
-    Returns:
-      A list of maps containing the normalized, filtered events.
-    """
-    events = {}
-    for event in google_batch_operations.get_status_events(self._op):
-      mapped, _ = self._map(event)
-      name = mapped['name']
-
-      events[name] = mapped
-
-    return sorted(list(events.values()), key=operator.itemgetter('event-time'))
-
-  def _map(self, event):
-    """Extract elements from a Batch status event and map to a named event."""
-    description = event.description
-    event_time = event.event_time.rfc3339()
-
-    for name, regex in _EVENT_REGEX_MAP.items():
-      match = regex.match(description)
-      if match:
-        return {'name': name, 'event-time': event_time}, match
-
-    return {'name': description, 'event-time': event_time}, None
-
-
-class GoogleBatchOperation(base.Task):
-  """Task wrapper around a Batch API Job object."""
-
-  def __init__(self, operation_data: batch_v1.types.Job):
-    self._op = operation_data
-    self._job_descriptor = self._try_op_to_job_descriptor()
-
-  def raw_task_data(self):
-    return self._op
-
-  def _try_op_to_job_descriptor(self):
-    # The _META_YAML_REPR field in the 'prepare' action enables reconstructing
-    # the original job descriptor.
-    # We only need the env for the prepare action (runnable) here.
-    env = google_batch_operations.get_environment(self._op, _PREPARE_INDEX)
-    if not env:
-      return
-
-    meta = env.get(google_utils.META_YAML_VARNAME)
-    if not meta:
-      return
-
-    return job_model.JobDescriptor.from_yaml(ast.literal_eval(meta))
-
-  def get_field(self, field: str, default: str = None):
-    """Returns a value from the operation for a specific set of field names.
-
-    This is the implementation of base.Task's abstract get_field method. See
-    base.py get_field for more details.
-
-    Args:
-      field: a dsub-specific job metadata key
-      default: default value to return if field does not exist or is empty.
-
-    Returns:
-      A text string for the field or a list for 'inputs'.
-
-    Raises:
-      ValueError: if the field label is not supported by the operation
-    """
-    value = None
-    if field == 'internal-id':
-      value = self._op.name
-    elif field == 'user-project':
-      if self._job_descriptor:
-        value = self._job_descriptor.job_metadata.get(field)
-    elif field in [
-        'job-id',
-        'job-name',
-        'task-id',
-        'task-attempt',
-        'user-id',
-        'dsub-version',
-    ]:
-      value = google_batch_operations.get_label(self._op, field)
-    elif field == 'task-status':
-      value = self._operation_status()
-    elif field == 'logging':
-      if self._job_descriptor:
-        # The job_resources will contain the "--logging" value.
-        # The task_resources will contain the resolved logging path.
-        # Return the resolved logging path.
-        task_resources = self._job_descriptor.task_descriptors[0].task_resources
-        value = task_resources.logging_path
-    elif field in ['envs', 'labels']:
-      if self._job_descriptor:
-        items = providers_util.get_job_and_task_param(
-            self._job_descriptor.job_params,
-            self._job_descriptor.task_descriptors[0].task_params,
-            field,
-        )
-        value = {item.name: item.value for item in items}
-    elif field in [
-        'inputs',
-        'outputs',
-        'input-recursives',
-        'output-recursives',
-    ]:
-      if self._job_descriptor:
-        value = {}
-        items = providers_util.get_job_and_task_param(
-            self._job_descriptor.job_params,
-            self._job_descriptor.task_descriptors[0].task_params,
-            field,
-        )
-        value.update({item.name: item.value for item in items})
-    elif field == 'mounts':
-      if self._job_descriptor:
-        items = providers_util.get_job_and_task_param(
-            self._job_descriptor.job_params,
-            self._job_descriptor.task_descriptors[0].task_params,
-            field,
-        )
-        value = {item.name: item.value for item in items}
-    elif field == 'provider':
-      return _PROVIDER_NAME
-    elif field == 'provider-attributes':
-      value = {
-          'boot-disk-size': google_batch_operations.get_boot_disk_size(
-              self._op
-          ),
-          'disk-size': google_batch_operations.get_disk_size(self._op),
-          'disk-type': google_batch_operations.get_disk_type(self._op),
-          'machine-type': google_batch_operations.get_machine_type(self._op),
-          'regions': google_batch_operations.get_regions(self._op),
-          'zones': google_batch_operations.get_zones(self._op),
-          'preemptible': google_batch_operations.get_preemptible(self._op),
-      }
-    elif field == 'events':
-      value = GoogleBatchEventMap(self._op).get_filtered_normalized_events()
-    elif field == 'script-name':
-      if self._job_descriptor:
-        value = self._job_descriptor.job_metadata.get(field)
-    elif field == 'script':
-      value = self._try_op_to_script_body()
-    elif field == 'create-time' or field == 'start-time':
-      # TODO: Does Batch offer a start or end-time?
-      # Check http://shortn/_FPYmD1weUF
-      ds = google_batch_operations.get_create_time(self._op)
-      value = google_base.parse_rfc3339_utc_string(ds)
-    elif field == 'end-time' or field == 'last-update':
-      # TODO: Does Batch offer an end-time?
-      # Check http://shortn/_FPYmD1weUF
-      ds = google_batch_operations.get_update_time(self._op)
-      if ds:
-        value = google_base.parse_rfc3339_utc_string(ds)
-    elif field == 'status':
-      value = self._operation_status()
-    elif field == 'status-message':
-      msg, _, _ = self._operation_status_message()
-      value = msg
-    elif field == 'status-detail':
-      # As much detail as we can reasonably get from the operation
-      msg, _, detail = self._operation_status_message()
-      if detail:
-        msg = detail
-      value = msg
-    else:
-      raise ValueError(f'Unsupported field: "{field}"')
-
-    return value if value else default
-
-  def _try_op_to_script_body(self):
-    # We only need the env for the prepare action (runnable) here.
-    env = google_batch_operations.get_environment(self._op, _PREPARE_INDEX)
-    if env:
-      return ast.literal_eval(env.get(google_utils.SCRIPT_VARNAME))
-
-  def _operation_status(self):
-    """Returns the status of this operation.
-
-    Raises:
-      ValueError: if the operation status cannot be determined.
-
-    Returns:
-      A printable status string (RUNNING, SUCCESS, CANCELED or FAILURE).
-    """
-    if not google_batch_operations.is_done(self._op):
-      return 'RUNNING'
-    if google_batch_operations.is_success(self._op):
-      return 'SUCCESS'
-    if google_batch_operations.is_canceled(self._op):
-      return 'CANCELED'
-    if google_batch_operations.is_failed(self._op):
-      return 'FAILURE'
-
-    raise ValueError(
-        'Status for operation {} could not be determined'.format(
-            self._op['name']
-        )
-    )
-
-  def _operation_status_message(self):
-    """Returns the most relevant status string and failed action.
-
-    This string is meant for display only.
-
-    Returns:
-      A triple of:
-      - printable status message
-      - the action that failed (if any)
-      - a detail message (if available)
-    """
-    msg = ''
-    action = None
-    detail = None
-    status_events = google_batch_operations.get_status_events(self._op)
-    if not google_batch_operations.is_done(self._op):
-      msg = 'RUNNING'
-    elif google_batch_operations.is_success(self._op):
-      msg = 'SUCCESS'
-    elif google_batch_operations.is_canceled(self._op):
-      msg = 'CANCELED'
-    elif google_batch_operations.is_failed(self._op):
-      msg = 'FAILURE'
-
-    if status_events:
-      detail = status_events[-1].description
-    return msg, action, detail
-
-
-class GoogleBatchBatchHandler(object):
-  """Implement the HttpBatch interface to enable simple serial batches."""
-
-  def __init__(self, callback):
-    self._cancel_list = []
-    self._response_handler = callback
-
-  def add(self, cancel_fn, request_id):
-    self._cancel_list.append((request_id, cancel_fn))
-
-  def execute(self):
-    for request_id, cancel_fn in self._cancel_list:
-      response = None
-      exception = None
-      try:
-        response = cancel_fn.result()
-      except:  # pylint: disable=bare-except
-        exception = sys.exc_info()[1]
-
-      self._response_handler(request_id, response, exception)
-
-
-class GoogleBatchJobProvider(google_utils.GoogleJobProviderBase):
-  """dsub provider implementation managing Jobs on Google Cloud."""
-
-  def __init__(
-      self, dry_run: bool, project: str, location: str, credentials=None
-  ):
-    storage_service = dsub_util.get_storage_service(credentials=credentials)
-
-    self._dry_run = dry_run
-    self._location = location
-    self._project = project
-    self._storage_service = storage_service
-
-  def _batch_handler_def(self):
-    return GoogleBatchBatchHandler
-
-  def _operations_cancel_api_def(self):
-    return batch_v1.BatchServiceClient().cancel_job
-
-  def _get_provisioning_model(self, task_resources):
-    if task_resources.preemptible:
-      return batch_v1.AllocationPolicy.ProvisioningModel.SPOT
-    else:
-      return batch_v1.AllocationPolicy.ProvisioningModel.STANDARD
-
-  def _get_batch_job_regions(self, regions, zones) -> List[str]:
-    """Returns the list of regions and zones to use for a Batch Job request.
-
-    If neither regions nor zones were specified for the Job, then use the
-    Batch Job API location as the default region.
-
-    Regions need to be prefixed with "regions/" and zones need to be prefixed
-    with "zones/" as documented in
-    https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#LocationPolicy
-
-    Args:
-      regions (str): A space separated list of regions to use for the Job.
-      zones (str): A space separated list of zones to use for the Job.
-    """
-    if regions:
-      regions = [f'regions/{region}' for region in regions]
-    if zones:
-      zones = [f'zones/{zone}' for zone in zones]
-    if not regions and not zones:
-      return [f'regions/{self._location}']
-    return (regions or []) + (zones or [])
-
-  def _get_logging_env(self, logging_uri, user_project, include_filter_script):
-    """Returns the environment for actions that copy logging files."""
-    if not logging_uri.endswith('.log'):
-      raise ValueError('Logging URI must end in ".log": {}'.format(logging_uri))
-
-    logging_prefix = logging_uri[: -len('.log')]
-    env = {
-        'LOGGING_PATH': '{}.log'.format(logging_prefix),
-        'STDOUT_PATH': '{}-stdout.log'.format(logging_prefix),
-        'STDERR_PATH': '{}-stderr.log'.format(logging_prefix),
-        'USER_PROJECT': user_project,
-    }
-    if include_filter_script:
-      env[_LOG_FILTER_VAR] = repr(_LOG_FILTER_PYTHON)
-
-    return env
-
-  def _format_batch_job_id(self, task_metadata, job_metadata) -> str:
-    # Each dsub task is submitted as its own Batch API job, so we
-    # append the dsub task-id and task-attempt to the job-id for the
-    # batch job ID.
-    # For single-task dsub jobs, there is no task-id, so use 0.
-    # Use a '-' character as the delimeter because Batch API job ID
-    # must match regex ^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$
-    task_id = task_metadata.get('task-id') or 0
-    task_attempt = task_metadata.get('task-attempt') or 0
-    batch_job_id = job_metadata.get('job-id')
-    return f'{batch_job_id}-{task_id}-{task_attempt}'
-
-  def _get_gcs_volumes(self, mounts) -> List[batch_v1.types.Volume]:
-    # Return a list of GCS volumes for the Batch Job request.
-    gcs_volumes = []
-    for gcs_mount in param_util.get_gcs_mounts(mounts):
-      mount_path = os.path.join(_VOLUME_MOUNT_POINT, gcs_mount.docker_path)
-      # Normalize mount path because API does not allow trailing slashes
-      normalized_mount_path = os.path.normpath(mount_path)
-      gcs_volume = google_batch_operations.build_gcs_volume(
-          gcs_mount.value[len('gs://') :], normalized_mount_path, ['-o ro']
-      )
-      gcs_volumes.append(gcs_volume)
-    return gcs_volumes
-
-  def _get_gcs_volumes_for_user_command(self, mounts) -> List[str]:
-    # Return a list of GCS volumes to be included with the
-    # user-command runnable
-    user_command_volumes = []
-    for gcs_mount in param_util.get_gcs_mounts(mounts):
-      volume_mount_point = os.path.normpath(
-          os.path.join(_VOLUME_MOUNT_POINT, gcs_mount.docker_path)
-      )
-      data_mount_point = os.path.normpath(
-          os.path.join(_DATA_MOUNT_POINT, gcs_mount.docker_path)
-      )
-      user_command_volumes.append(f'{volume_mount_point}:{data_mount_point}')
-    return user_command_volumes
-
-  def _create_batch_request(
-      self,
-      task_view: job_model.JobDescriptor,
-  ):
-    job_metadata = task_view.job_metadata
-    job_params = task_view.job_params
-    job_resources = task_view.job_resources
-    task_metadata = task_view.task_descriptors[0].task_metadata
-    task_params = task_view.task_descriptors[0].task_params
-    task_resources = task_view.task_descriptors[0].task_resources
-
-    # Set up VM-specific variables
-    datadisk_volume = google_batch_operations.build_volume(
-        disk=google_utils.DATA_DISK_NAME, path=_VOLUME_MOUNT_POINT
-    )
-
-    # Set up the task labels
-    # pylint: disable=g-complex-comprehension
-    labels = {
-        label.name: label.value if label.value else ''
-        for label in google_base.build_pipeline_labels(
-            job_metadata, task_metadata
-        )
-        | job_params['labels']
-        | task_params['labels']
-    }
-    # pylint: enable=g-complex-comprehension
-
-    # Set local variables for the core pipeline values
-    script = task_view.job_metadata['script']
-
-    # Track 0-based runnable indexes for cross-task awareness
-    user_action = 3
-
-    continuous_logging_cmd = _CONTINUOUS_LOGGING_CMD.format(
-        log_msg_fn=google_utils.LOG_MSG_FN,
-        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
-        log_filter_var=_LOG_FILTER_VAR,
-        log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
-        python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
-        logging_dir=_LOGGING_DIR,
-        log_cp=_LOG_CP.format(
-            log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
-            user_action=user_action,
-        ),
-        log_interval=job_resources.log_interval or '60s',
-    )
-
-    logging_cmd = _FINAL_LOGGING_CMD.format(
-        log_msg_fn=google_utils.LOG_MSG_FN,
-        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
-        log_filter_var=_LOG_FILTER_VAR,
-        log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
-        python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
-        logging_dir=_LOGGING_DIR,
-        log_cp=_LOG_CP.format(
-            log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
-            user_action=user_action,
-        ),
-    )
-
-    # Set up command and environments for the prepare, localization, user,
-    # and de-localization actions
-    script_path = os.path.join(_SCRIPT_DIR, script.name)
-    user_project = task_view.job_metadata['user-project'] or ''
-
-    prepare_command = google_utils.PREPARE_CMD.format(
-        log_msg_fn=google_utils.LOG_MSG_FN,
-        mk_runtime_dirs=google_utils.make_runtime_dirs_command(
-            _SCRIPT_DIR, _TMP_DIR, _WORKING_DIR
-        ),
-        script_var=google_utils.SCRIPT_VARNAME,
-        python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
-        script_path=script_path,
-        mk_io_dirs=google_utils.MK_IO_DIRS,
-    )
-    # pylint: disable=line-too-long
-
-    continuous_logging_env = google_batch_operations.build_environment(
-        self._get_logging_env(
-            task_resources.logging_path.uri, user_project, True
-        )
-    )
-    final_logging_env = google_batch_operations.build_environment(
-        self._get_logging_env(
-            task_resources.logging_path.uri, user_project, False
-        )
-    )
-
-    envs = job_params['envs'] | task_params['envs']
-    inputs = job_params['inputs'] | task_params['inputs']
-    outputs = job_params['outputs'] | task_params['outputs']
-    mounts = job_params['mounts']
-    gcs_volumes = self._get_gcs_volumes(mounts)
-
-    prepare_env = google_batch_operations.build_environment(
-        self._get_prepare_env(
-            script, task_view, inputs, outputs, mounts, _DATA_MOUNT_POINT
-        )
-    )
-    localization_env = google_batch_operations.build_environment(
-        self._get_localization_env(inputs, user_project, _DATA_MOUNT_POINT)
-    )
-    user_environment = google_batch_operations.build_environment(
-        self._build_user_environment(
-            envs, inputs, outputs, mounts, _DATA_MOUNT_POINT
-        )
-    )
-    delocalization_env = google_batch_operations.build_environment(
-        self._get_delocalization_env(outputs, user_project, _DATA_MOUNT_POINT)
-    )
-
-    # Build the list of runnables (aka actions)
-    runnables = []
-
-    runnables.append(
-        # logging
-        google_batch_operations.build_runnable(
-            run_in_background=True,
-            always_run=False,
-            image_uri=google_utils.CLOUD_SDK_IMAGE,
-            environment=continuous_logging_env,
-            entrypoint='/bin/bash',
-            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}'],
-            commands=['-c', continuous_logging_cmd],
-            options=None
-        )
-    )
-
-    runnables.append(
-        # prepare
-        google_batch_operations.build_runnable(
-            run_in_background=False,
-            always_run=False,
-            image_uri=google_utils.CLOUD_SDK_IMAGE,
-            environment=prepare_env,
-            entrypoint='/bin/bash',
-            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}'],
-            commands=['-c', prepare_command],
-            options=None
-        )
-    )
-
-    runnables.append(
-        # localization
-        google_batch_operations.build_runnable(
-            run_in_background=False,
-            always_run=False,
-            image_uri=google_utils.CLOUD_SDK_IMAGE,
-            environment=localization_env,
-            entrypoint='/bin/bash',
-            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}'],
-            commands=[
-                '-c',
-                google_utils.LOCALIZATION_CMD.format(
-                    log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
-                    cp_loop=google_utils.LOCALIZATION_LOOP,
-                ),
-            ],
-            options=None
-        )
-    )
-
-    user_command_volumes = [f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}']
-    for gcs_volume in self._get_gcs_volumes_for_user_command(mounts):
-      user_command_volumes.append(gcs_volume)
-    # Add --gpus all option for GPU-enabled containers
-    container_options = '--gpus all' if job_resources.accelerator_type and job_resources.accelerator_type.startswith('nvidia') else None
-    runnables.append(
-        # user-command
-        google_batch_operations.build_runnable(
-            run_in_background=False,
-            always_run=False,
-            image_uri=job_resources.image,
-            environment=user_environment,
-            entrypoint='/usr/bin/env',
-            volumes=user_command_volumes,
-            commands=[
-                'bash',
-                '-c',
-                google_utils.USER_CMD.format(
-                    tmp_dir=_TMP_DIR,
-                    working_dir=_WORKING_DIR,
-                    user_script=script_path,
-                ),
-            ],
-            options=container_options,
-        )
-    )
-
-    runnables.append(
-        # delocalization
-        google_batch_operations.build_runnable(
-            run_in_background=False,
-            always_run=False,
-            image_uri=google_utils.CLOUD_SDK_IMAGE,
-            environment=delocalization_env,
-            entrypoint='/bin/bash',
-            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}:ro'],
-            commands=[
-                '-c',
-                google_utils.LOCALIZATION_CMD.format(
-                    log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
-                    cp_loop=google_utils.DELOCALIZATION_LOOP,
-                ),
-            ],
-            options=None
-        )
-    )
-
-    runnables.append(
-        # final_logging
-        google_batch_operations.build_runnable(
-            run_in_background=False,
-            always_run=True,
-            image_uri=google_utils.CLOUD_SDK_IMAGE,
-            environment=final_logging_env,
-            entrypoint='/bin/bash',
-            volumes=[f'{_VOLUME_MOUNT_POINT}:{_DATA_MOUNT_POINT}'],
-            commands=['-c', logging_cmd],
-            options=None
-        ),
-    )
-
-    # Prepare the VM (resources) configuration. The InstancePolicy describes an
-    # instance type and resources attached to each VM. The AllocationPolicy
-    # describes when, where, and how compute resources should be allocated
-    # for the Job.
-    boot_disk_size = (
-        job_resources.boot_disk_size if job_resources.boot_disk_size else 0
-    )
-    # Determine boot disk image: use user-specified value, or default to batch-debian for GPU jobs
-    if job_resources.boot_disk_image:
-      boot_disk_image = job_resources.boot_disk_image
-    elif job_resources.accelerator_type and job_resources.accelerator_type.startswith('nvidia'):
-      boot_disk_image = 'batch-debian'
-    else:
-      boot_disk_image = None
-
-    boot_disk = google_batch_operations.build_persistent_disk(
-        size_gb=max(boot_disk_size, job_model.LARGE_BOOT_DISK_SIZE),
-        disk_type=job_model.DEFAULT_DISK_TYPE,
-        image=boot_disk_image,
-    )
-    disk = google_batch_operations.build_persistent_disk(
-        size_gb=job_resources.disk_size,
-        disk_type=job_resources.disk_type or job_model.DEFAULT_DISK_TYPE,
-        image=None
-    )
-    attached_disk = google_batch_operations.build_attached_disk(
-        disk=disk, device_name=google_utils.DATA_DISK_NAME
-    )
-
-    if job_resources.machine_type:
-      machine_type = job_resources.machine_type
-    elif job_resources.min_cores or job_resources.min_ram:
-      machine_type = (
-          google_custom_machine.GoogleCustomMachine.build_machine_type(
-              job_resources.min_cores, job_resources.min_ram
-          )
-      )
-    else:
-      machine_type = job_model.DEFAULT_MACHINE_TYPE
-
-    instance_policy = google_batch_operations.build_instance_policy(
-        boot_disk=boot_disk,
-        disks=attached_disk,
-        machine_type=machine_type,
-        accelerators=google_batch_operations.build_accelerators(
-            accelerator_type=job_resources.accelerator_type,
-            accelerator_count=job_resources.accelerator_count,
-        ),
-        provisioning_model=self._get_provisioning_model(task_resources),
-    )
-
-    # Determine whether to install GPU drivers: use user-specified value, or default to True for GPU jobs
-    if job_resources.install_gpu_drivers is not None:
-      install_gpu_drivers = job_resources.install_gpu_drivers
-    else:
-      install_gpu_drivers = job_resources.accelerator_type is not None
-
-    ipt = google_batch_operations.build_instance_policy_or_template(
-        instance_policy=instance_policy,
-        install_gpu_drivers=install_gpu_drivers,
-    )
-
-    if job_resources.service_account:
-      scopes = job_resources.scopes or google_base.DEFAULT_SCOPES
-      service_account = google_batch_operations.build_service_account(
-          service_account_email=job_resources.service_account, scopes=scopes
-      )
-    else:
-      service_account = None
-
-    network_policy = google_batch_operations.build_network_policy(
-        network=job_resources.network,
-        subnetwork=job_resources.subnetwork,
-        no_external_ip_address=job_resources.use_private_address,
-    )
-
-    location_policy = google_batch_operations.build_location_policy(
-        allowed_locations=self._get_batch_job_regions(
-            regions=job_resources.regions, zones=job_resources.zones
-        ),
-    )
-
-    allocation_policy = google_batch_operations.build_allocation_policy(
-        ipts=[ipt],
-        service_account=service_account,
-        network_policy=network_policy,
-        location_policy=location_policy,
-    )
-
-    logs_policy = google_batch_operations.build_logs_policy(
-        # Explicitly end the logging path with a slash.
-        # This will prompt Batch API to create the log, stdout, and stderr
-        # files in the specified directory.
-        batch_v1.LogsPolicy.Destination.PATH,
-        _BATCH_LOG_DIR + '/',
-    )
-
-    # Bring together the task definition(s) and build the Job request.
-    task_spec = google_batch_operations.build_task_spec(
-        runnables=runnables, volumes=([datadisk_volume] + gcs_volumes), max_run_duration=job_resources.timeout
-    )
-    task_group = google_batch_operations.build_task_group(
-        task_spec, task_count=1, task_count_per_node=1
-    )
-
-    job = google_batch_operations.build_job(
-        [task_group], allocation_policy, labels, logs_policy
-    )
-
-    batch_job_id = self._format_batch_job_id(task_metadata, job_metadata)
-
-    job_request = batch_v1.CreateJobRequest(
-        parent=f'projects/{self._project}/locations/{self._location}',
-        job=job,
-        job_id=batch_job_id,
-    )
-    # pylint: enable=line-too-long
-    return job_request
-
-  def _submit_batch_job(self, request) -> str:
-    client = batch_v1.BatchServiceClient()
-    job_response = client.create_job(request=request)
-    op = GoogleBatchOperation(job_response)
-    print(f'Provider internal-id (operation): {job_response.name}')
-    return op.get_field('task-id')
-
-  def submit_job(
-      self,
-      job_descriptor: job_model.JobDescriptor,
-      skip_if_output_present: bool,
-  ) -> Dict[str, any]:
-    # Validate task data and resources.
-    param_util.validate_submit_args_or_fail(
-        job_descriptor,
-        provider_name=_PROVIDER_NAME,
-        input_providers=_SUPPORTED_INPUT_PROVIDERS,
-        output_providers=_SUPPORTED_OUTPUT_PROVIDERS,
-        logging_providers=_SUPPORTED_LOGGING_PROVIDERS,
-    )
-
-    # Prepare and submit jobs.
-    launched_tasks = []
-    requests = []
-
-    for task_view in job_model.task_view_generator(job_descriptor):
-
-      job_params = task_view.job_params
-      task_params = task_view.task_descriptors[0].task_params
-
-      outputs = job_params['outputs'] | task_params['outputs']
-      if skip_if_output_present:
-        # check whether the output's already there
-        if dsub_util.outputs_are_present(outputs, self._storage_service):
-          print('Skipping task because its outputs are present')
-          continue
-
-      request = self._create_batch_request(task_view)
-      if self._dry_run:
-        requests.append(request)
-      else:
-        task_id = self._submit_batch_job(request)
-        launched_tasks.append(task_id)
-
-    # If this is a dry-run, emit all the batch request objects
-    if self._dry_run:
-      # Each request is a google.cloud.batch_v1.types.batch.CreateJobRequest
-      # object. The __repr__ method for this object outputs something that
-      # closely resembles yaml, but can't actually be serialized into yaml.
-      # Ideally, we could serialize these request objects to yaml or json.
-      print(requests)
-
-    if not requests and not launched_tasks:
-      return {'job-id': dsub_util.NO_JOB}
-
-    return {
-        'job-id': job_descriptor.job_metadata['job-id'],
-        'user-id': job_descriptor.job_metadata.get('user-id'),
-        'task-id': [task_id for task_id in launched_tasks if task_id],
-    }
-
-  def delete_jobs(
-      self,
-      user_ids,
-      job_ids,
-      task_ids,
-      labels,
-      create_time_min=None,
-      create_time_max=None,
-  ):
-    """Kills the operations associated with the specified job or job.task.
-
-    Args:
-      user_ids: List of user ids who "own" the job(s) to cancel.
-      job_ids: List of job_ids to cancel.
-      task_ids: List of task-ids to cancel.
-      labels: List of LabelParam, each must match the job(s) to be canceled.
-      create_time_min: a timezone-aware datetime value for the earliest create
-        time of a task, inclusive.
-      create_time_max: a timezone-aware datetime value for the most recent
-        create time of a task, inclusive.
-
-    Returns:
-      A list of tasks canceled and a list of error messages.
-    """
-    # Look up the job(s)
-    tasks = list(
-        self.lookup_job_tasks(
-            {'RUNNING'},
-            user_ids=user_ids,
-            job_ids=job_ids,
-            task_ids=task_ids,
-            labels=labels,
-            create_time_min=create_time_min,
-            create_time_max=create_time_max,
-        )
-    )
-
-    print('Found %d tasks to delete.' % len(tasks))
-    return google_base.cancel(
-        self._batch_handler_def(), self._operations_cancel_api_def(), tasks
-    )
-
-  def lookup_job_tasks(
-      self,
-      statuses: Set[str],
-      user_ids=None,
-      job_ids=None,
-      job_names=None,
-      task_ids=None,
-      task_attempts=None,
-      labels=None,
-      create_time_min=None,
-      create_time_max=None,
-      max_tasks=0,
-      page_size=0,
-  ):
-    client = batch_v1.BatchServiceClient()
-    ops_filter = self._build_query_filter(
-        statuses,
-        user_ids,
-        job_ids,
-        job_names,
-        task_ids,
-        task_attempts,
-        labels,
-        create_time_min,
-        create_time_max,
-    )
-    # Initialize request argument(s)
-    request = batch_v1.ListJobsRequest(
-        parent=f'projects/{self._project}/locations/{self._location}',
-        filter=ops_filter,
-    )
-
-    # Make the request
-    response = client.list_jobs(request=request)
-    # Sort the operations by create-time to match sort of other providers
-    operations = [GoogleBatchOperation(page) for page in response]
-    operations.sort(key=lambda op: op.get_field('create-time'), reverse=True)
-    for op in operations:
-      yield op
-
-  def get_tasks_completion_messages(self, tasks):
-    # TODO: This needs to return a list of error messages for each task
-    pass
\ No newline at end of file

From f342e8d61af64c4e969c04986eae0a5955b70365 Mon Sep 17 00:00:00 2001
From: "Kristen Liu (Ong)" <43296048+kvo3@users.noreply.github.com>
Date: Fri, 21 Aug 2026 09:34:26 -0700
Subject: [PATCH 06/10] Delete dsub/providers/.ipynb_checkpoints directory

---
 .../google_utils-checkpoint.py                | 502 ------------------
 1 file changed, 502 deletions(-)
 delete mode 100644 dsub/providers/.ipynb_checkpoints/google_utils-checkpoint.py

diff --git a/dsub/providers/.ipynb_checkpoints/google_utils-checkpoint.py b/dsub/providers/.ipynb_checkpoints/google_utils-checkpoint.py
deleted file mode 100644
index 6a47e96..0000000
--- a/dsub/providers/.ipynb_checkpoints/google_utils-checkpoint.py
+++ /dev/null
@@ -1,502 +0,0 @@
-# Copyright 2022 Verily Life Sciences Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Utility functions to be used by the Google providers.
-
-This module holds constants and methods useful to google-cls-v2
-and google-batch providers.
-"""
-import os
-import textwrap
-from typing import Dict
-
-from . import base
-
-from ..lib import job_model
-from ..lib import providers_util
-
-STATUS_FILTER_MAP = {
-    'RUNNING': 'status.state="RUNNING" OR status.state="QUEUED" OR status.state="SCHEDULED"',
-    'CANCELED': 'status.state="CANCELLED"',
-    'FAILURE': 'status.state="FAILED"',
-    'SUCCESS': 'status.state="SUCCEEDED"',
-}
-
-
-def prepare_query_label_value(labels):
-  """Converts the label strings to contain label-appropriate characters.
-
-  Args:
-    labels: A set of strings to be converted.
-
-  Returns:
-    A list of converted strings.
-  """
-  if not labels:
-    return None
-  return [job_model.convert_to_label_chars(label) for label in labels]
-
-
-def label_filter(label_key, label_value):
-  """Return a valid label filter for operations.list()."""
-  return 'labels."{}" = "{}"'.format(label_key, label_value)
-
-
-def create_time_filter(create_time, comparator):
-  """Return a valid createTime filter for operations.list()."""
-  return 'createTime {} "{}"'.format(comparator, create_time.isoformat())
-
-
-# Generate command to create the directories for the dsub user environment
-# pylint: disable=g-complex-comprehension
-def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
-                              working_dir: str) -> str:
-  return '\n'.join('mkdir -m 777 -p "%s" ' % dir
-                   for dir in [script_dir, tmp_dir, working_dir])
-
-
-# pylint: enable=g-complex-comprehension
-
-
-# Action steps that interact with GCS need gsutil and Python.
-# Use the 'slim' variant of the cloud-sdk image as it is much smaller.
-CLOUD_SDK_IMAGE = 'gcr.io/google.com/cloudsdktool/cloud-sdk:294.0.0-slim'
-
-# Name of the data disk
-DATA_DISK_NAME = 'datadisk'
-
-# Define a bash function for "echo" that includes timestamps
-LOG_MSG_FN = textwrap.dedent("""\
-  function get_datestamp() {
-    date "+%Y-%m-%d %H:%M:%S"
-  }
-
-  function log_info() {
-    echo "$(get_datestamp) INFO: $@"
-  }
-
-  function log_warning() {
-    1>&2 echo "$(get_datestamp) WARNING: $@"
-  }
-
-  function log_error() {
-    1>&2 echo "$(get_datestamp) ERROR: $@"
-  }
-""")
-
-# Define a bash function for "gsutil cp" to be used by the logging,
-# localization, and delocalization actions.
-GSUTIL_CP_FN = textwrap.dedent("""\
-  function gsutil_cp() {
-    local src="${1}"
-    local dst="${2}"
-    local content_type="${3}"
-    local user_project_name="${4}"
-
-    local headers=""
-    if [[ -n "${content_type}" ]]; then
-      headers="-h Content-Type:${content_type}"
-    fi
-
-    local user_project_flag=""
-    if [[ -n "${user_project_name}" ]]; then
-      user_project_flag="-u ${user_project_name}"
-    fi
-
-    local attempt
-    for ((attempt = 0; attempt < 4; attempt++)); do
-      log_info "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
-      if gsutil ${headers} ${user_project_flag} -mq cp "${src}" "${dst}"; then
-        return
-      fi
-      if (( attempt < 3 )); then
-        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
-        log_warning "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
-        sleep 10s
-      fi
-    done
-
-    log_error "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
-    exit 1
-  }
-""")
-
-LOG_CP_FN = GSUTIL_CP_FN + textwrap.dedent("""\
-
-  function log_cp() {
-    local src="${1}"
-    local dst="${2}"
-    local tmp="${3}"
-    local check_src="${4}"
-    local user_project_name="${5}"
-
-    if [[ "${check_src}" == "true" ]] && [[ ! -e "${src}" ]]; then
-      return
-    fi
-
-    # Copy the log files to a local temporary location so that our "gsutil cp" is never
-    # executed on a file that is changing.
-
-    local tmp_path="${tmp}/$(basename ${src})"
-    cp "${src}" "${tmp_path}"
-
-    gsutil_cp "${tmp_path}" "${dst}" "text/plain" "${user_project_name}"
-  }
-""")
-
-# Define a bash function for "gsutil rsync" to be used by the logging,
-# localization, and delocalization actions.
-GSUTIL_RSYNC_FN = textwrap.dedent("""\
-  function gsutil_rsync() {
-    local src="${1}"
-    local dst="${2}"
-    local user_project_name="${3}"
-
-    local user_project_flag=""
-    if [[ -n "${user_project_name}" ]]; then
-      user_project_flag="-u ${user_project_name}"
-    fi
-
-    local attempt
-    for ((attempt = 0; attempt < 4; attempt++)); do
-      log_info "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
-      if gsutil ${user_project_flag} -mq rsync -r "${src}" "${dst}"; then
-        return
-      fi
-      if (( attempt < 3 )); then
-        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
-        log_warning "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
-        sleep 10s
-      fi
-    done
-
-    log_error "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
-    exit 1
-  }
-""")
-
-LOCALIZATION_LOOP = textwrap.dedent("""\
-  set -o errexit
-  set -o nounset
-  set -o pipefail
-
-  for ((i=0; i < INPUT_COUNT; i++)); do
-    INPUT_VAR="INPUT_${i}"
-    INPUT_RECURSIVE="INPUT_RECURSIVE_${i}"
-    INPUT_SRC="INPUT_SRC_${i}"
-    INPUT_DST="INPUT_DST_${i}"
-
-    log_info "Localizing ${!INPUT_VAR}"
-    if [[ "${!INPUT_RECURSIVE}" -eq "1" ]]; then
-      gsutil_rsync "${!INPUT_SRC}" "${!INPUT_DST}" "${USER_PROJECT}"
-    else
-      gsutil_cp "${!INPUT_SRC}" "${!INPUT_DST}" "" "${USER_PROJECT}"
-    fi
-  done
-""")
-
-DELOCALIZATION_LOOP = textwrap.dedent("""\
-  set -o errexit
-  set -o nounset
-  set -o pipefail
-
-  for ((i=0; i < OUTPUT_COUNT; i++)); do
-    OUTPUT_VAR="OUTPUT_${i}"
-    OUTPUT_RECURSIVE="OUTPUT_RECURSIVE_${i}"
-    OUTPUT_SRC="OUTPUT_SRC_${i}"
-    OUTPUT_DST="OUTPUT_DST_${i}"
-
-    log_info "Delocalizing ${!OUTPUT_VAR}"
-    if [[ "${!OUTPUT_RECURSIVE}" -eq "1" ]]; then
-      gsutil_rsync "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "${USER_PROJECT}"
-    else
-      gsutil_cp "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "" "${USER_PROJECT}"
-    fi
-  done
-""")
-
-LOCALIZATION_CMD = textwrap.dedent("""\
-  {log_msg_fn}
-  {recursive_cp_fn}
-  {cp_fn}
-
-  {cp_loop}
-""")
-
-# The user's script or command is made available to the container in
-#   /mnt/data/script/<script-name>
-#
-# To get it there, it is passed in through the environment in the "prepare"
-# action and "echo"-ed to a file.
-#
-# Google APIs use Docker environment files which do not support
-# multi-line environment variables, so we encode the script using Python's
-# repr() function and then decoded it using ast.literal_eval().
-# This has the advantage over other encoding schemes (such as base64) of being
-# user-readable in the LifeSciences "operation" or Batch "Job" object.
-SCRIPT_VARNAME = '_SCRIPT_REPR'
-META_YAML_VARNAME = '_META_YAML_REPR'
-
-PYTHON_DECODE_SCRIPT = textwrap.dedent("""\
-  import ast
-  import sys
-
-  sys.stdout.write(ast.literal_eval(sys.stdin.read()))
-""")
-
-MK_IO_DIRS = textwrap.dedent("""\
-  for ((i=0; i < DIR_COUNT; i++)); do
-    DIR_VAR="DIR_${i}"
-
-    log_info "mkdir -m 777 -p \"${!DIR_VAR}\""
-    mkdir -m 777 -p "${!DIR_VAR}"
-  done
-""")
-
-PREPARE_CMD = textwrap.dedent("""\
-  #!/bin/bash
-
-  set -o errexit
-  set -o nounset
-  set -o pipefail
-
-  {log_msg_fn}
-  {mk_runtime_dirs}
-
-  echo "${{{script_var}}}" \
-    | python3 -c '{python_decode_script}' \
-    > "{script_path}"
-  chmod a+x "{script_path}"
-
-  {mk_io_dirs}
-""")
-
-USER_CMD = textwrap.dedent("""\
-  export TMPDIR="{tmp_dir}"
-  cd {working_dir}
-
-  "{user_script}"
-""")
-
-
-class GoogleJobProviderBase(base.JobProvider):
-  """dsub provider implementation managing Jobs on Google Cloud."""
-
-  def _get_prepare_env(self, script, job_descriptor, inputs, outputs, mounts,
-                       mount_point) -> Dict[str, str]:
-    """Return a dict with variables for the 'prepare' action."""
-
-    # Add the _SCRIPT_REPR with the repr(script) contents
-    # Add the _META_YAML_REPR with the repr(meta) contents
-
-    # Add variables for directories that need to be created, for example:
-    # DIR_COUNT: 2
-    # DIR_0: /mnt/data/input/gs/bucket/path1/
-    # DIR_1: /mnt/data/output/gs/bucket/path2
-
-    # List the directories in sorted order so that they are created in that
-    # order. This is primarily to ensure that permissions are set as we create
-    # each directory.
-    # For example:
-    #   mkdir -m 777 -p /root/first/second
-    #   mkdir -m 777 -p /root/first
-    # *may* not actually set 777 on /root/first
-
-    docker_paths = sorted([
-        var.docker_path if var.recursive else os.path.dirname(var.docker_path)
-        for var in inputs | outputs | mounts
-        if var.value
-    ])
-
-    env = {
-        SCRIPT_VARNAME: repr(script.value),
-        META_YAML_VARNAME: repr(job_descriptor.to_yaml()),
-        'DIR_COUNT': str(len(docker_paths))
-    }
-
-    for idx, path in enumerate(docker_paths):
-      env['DIR_{}'.format(idx)] = os.path.join(mount_point, path)
-
-    return env
-
-  def _get_localization_env(self, inputs, user_project,
-                            mount_point) -> Dict[str, str]:
-    """Return a dict with variables for the 'localization' action."""
-
-    # Add variables for paths that need to be localized, for example:
-    # INPUT_COUNT: 1
-    # INPUT_0: MY_INPUT_FILE
-    # INPUT_RECURSIVE_0: 0
-    # INPUT_SRC_0: gs://mybucket/mypath/myfile
-    # INPUT_DST_0: /mnt/data/inputs/mybucket/mypath/myfile
-
-    non_empty_inputs = [var for var in inputs if var.value]
-    env = {'INPUT_COUNT': str(len(non_empty_inputs))}
-
-    for idx, var in enumerate(non_empty_inputs):
-      env['INPUT_{}'.format(idx)] = var.name
-      env['INPUT_RECURSIVE_{}'.format(idx)] = str(int(var.recursive))
-      env['INPUT_SRC_{}'.format(idx)] = var.value
-
-      # For wildcard paths, the destination must be a directory
-      dst = os.path.join(mount_point, var.docker_path)
-      path, filename = os.path.split(dst)
-      if '*' in filename:
-        dst = '{}/'.format(path)
-      env['INPUT_DST_{}'.format(idx)] = dst
-
-    env['USER_PROJECT'] = user_project
-
-    return env
-
-  def _get_delocalization_env(self, outputs, user_project,
-                              mount_point) -> Dict[str, str]:
-    """Return a dict with variables for the 'delocalization' action."""
-
-    # Add variables for paths that need to be delocalized, for example:
-    # OUTPUT_COUNT: 1
-    # OUTPUT_0: MY_OUTPUT_FILE
-    # OUTPUT_RECURSIVE_0: 0
-    # OUTPUT_SRC_0: gs://mybucket/mypath/myfile
-    # OUTPUT_DST_0: /mnt/data/outputs/mybucket/mypath/myfile
-
-    non_empty_outputs = [var for var in outputs if var.value]
-    env = {'OUTPUT_COUNT': str(len(non_empty_outputs))}
-
-    for idx, var in enumerate(non_empty_outputs):
-      env['OUTPUT_{}'.format(idx)] = var.name
-      env['OUTPUT_RECURSIVE_{}'.format(idx)] = str(int(var.recursive))
-      env['OUTPUT_SRC_{}'.format(idx)] = os.path.join(mount_point,
-                                                      var.docker_path)
-
-      # For wildcard paths, the destination must be a directory
-      if '*' in var.uri.basename:
-        dst = var.uri.path
-      else:
-        dst = var.uri
-      env['OUTPUT_DST_{}'.format(idx)] = dst
-
-    env['USER_PROJECT'] = user_project
-
-    return env
-
-  def _build_user_environment(self, envs, inputs, outputs, mounts,
-                              mount_point) -> Dict[str, str]:
-    """Returns a dictionary of for the user container environment."""
-    envs = {env.name: env.value for env in envs}
-    envs.update(
-        providers_util.get_file_environment_variables(inputs, mount_point))
-    envs.update(
-        providers_util.get_file_environment_variables(outputs, mount_point))
-    envs.update(
-        providers_util.get_file_environment_variables(mounts, mount_point))
-    return envs
-
-  def prepare_job_metadata(self, script: str, job_name: str,
-                           user_id: str) -> Dict[str, str]:
-    return providers_util.prepare_job_metadata(script, job_name, user_id)
-
-  def _get_label_filters(self, label_key, values):
-    if not values or values == {'*'}:
-      return None
-
-    return [label_filter(label_key, v) for v in values]
-
-  def _get_labels_filters(self, labels):
-    if not labels:
-      return None
-
-    return [label_filter(l.name, l.value) for l in labels]
-
-  def _get_status_filters(self, statuses):
-    if not statuses or statuses == {'*'}:
-      return None
-
-    return [STATUS_FILTER_MAP[s] for s in statuses]
-
-  def _get_user_id_filter_value(self, user_ids):
-    if not user_ids or user_ids == {'*'}:
-      return None
-
-    return prepare_query_label_value(user_ids)
-
-  def _get_create_time_filters(self, create_time_min, create_time_max):
-    filters = []
-    for create_time, comparator in [(create_time_min, '>='),
-                                    (create_time_max, '<=')]:
-      if not create_time:
-        continue
-
-      filters.append(create_time_filter(create_time, comparator))
-    return filters
-
-  def _build_query_filter(self,
-                          statuses,
-                          user_ids=None,
-                          job_ids=None,
-                          job_names=None,
-                          task_ids=None,
-                          task_attempts=None,
-                          labels=None,
-                          create_time_min=None,
-                          create_time_max=None):
-    # The Google APIs allows for building fairly elaborate filter
-    # clauses. We can group (). We can AND, OR, and NOT.
-    #
-    # The first set of filters, labeled here as OR filters are elements
-    # where more than one value cannot be true at the same time. For example,
-    # an operation cannot have a status of both RUNNING and CANCELED.
-    #
-    # The second set of filters, labeled here as AND filters are elements
-    # where more than one value can be true. For example,
-    # an operation can have a label with key1=value2 AND key2=value2.
-
-    # Translate the semantic requests into a google-specific filter.
-
-    # 'OR' filtering arguments.
-    status_filters = self._get_status_filters(statuses)
-    user_id_filters = self._get_label_filters(
-        'user-id', self._get_user_id_filter_value(user_ids))
-    job_id_filters = self._get_label_filters('job-id', job_ids)
-    job_name_filters = self._get_label_filters(
-        'job-name', prepare_query_label_value(job_names))
-    task_id_filters = self._get_label_filters('task-id', task_ids)
-    task_attempt_filters = self._get_label_filters('task-attempt',
-                                                   task_attempts)
-    # 'AND' filtering arguments.
-    label_filters = self._get_labels_filters(labels)
-    create_time_filters = self._get_create_time_filters(create_time_min,
-                                                        create_time_max)
-
-    if job_id_filters and job_name_filters:
-      raise ValueError(
-          'Filtering by both job IDs and job names is not supported')
-
-    # Now build up the full text filter.
-    # OR all of the OR filter arguments together.
-    # AND all of the AND arguments together.
-    or_arguments = []
-    for or_filters in [
-        status_filters, user_id_filters, job_id_filters, job_name_filters,
-        task_id_filters, task_attempt_filters
-    ]:
-      if or_filters:
-        or_arguments.append('(' + ' OR '.join(or_filters) + ')')
-
-    and_arguments = []
-    for and_filters in [label_filters, create_time_filters]:
-      if and_filters:
-        and_arguments.append('(' + ' AND '.join(and_filters) + ')')
-
-    # Now and all of these arguments together.
-    return ' AND '.join(or_arguments + and_arguments)

From 0839c576c1285135f15651c354e2abd63d4ad299 Mon Sep 17 00:00:00 2001
From: "Kristen Liu (Ong)" <43296048+kvo3@users.noreply.github.com>
Date: Fri, 21 Aug 2026 09:36:32 -0700
Subject: [PATCH 07/10] Delete examples/decompress/.ipynb_checkpoints directory

---
 .../.ipynb_checkpoints/README-checkpoint.md   | 167 ------------------
 1 file changed, 167 deletions(-)
 delete mode 100644 examples/decompress/.ipynb_checkpoints/README-checkpoint.md

diff --git a/examples/decompress/.ipynb_checkpoints/README-checkpoint.md b/examples/decompress/.ipynb_checkpoints/README-checkpoint.md
deleted file mode 100644
index 00a15a2..0000000
--- a/examples/decompress/.ipynb_checkpoints/README-checkpoint.md
+++ /dev/null
@@ -1,167 +0,0 @@
-# Decompress with dsub
-
-This example demonstrates how to decompress files stored in a Google
-Cloud Storage bucket by submitting a simple command from a shell prompt
-on your laptop. The job executes in the cloud. As input, we start with a
-single compressed variant call format (VCF) file from the
-[1000 Genomes Project](http://www.internationalgenome.org/).
-
-We then proceed to an example that demonstrates processing multiple files,
-using a small list of VCFs.
-All of the source VCF files are stored in a public bucket at
-[gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/](https://console.cloud.google.com/storage/browser/genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/):
-
-*   20130723_phase3_wg/cornell/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf.gz
-*   20140708_previous_phase3/v2_vcfs/ALL.chr21.phase3_shapeit2_mvncall_integrated_v2.20130502.genotypes.vcf.gz
-*   20140708_previous_phase3/v1_vcfs/ALL.chr21.phase3_shapeit2_mvncall_integrated.20130502.genotype.vcf.gz
-*   20110721_exome_call_sets/bcm/ALL.BCM_Illumina_Mosaik_ontarget_plus50bp_822.20110521.snp.exome.genotypes.vcf.gz
-
-## Setup
-
-* Follow the [dsub geting started](../../README.md#getting-started)
-instructions.
-
-## Decompress one file
-
-### Submit the job
-
-The following command will submit a job to decompress the first input file
-from the list above and write the decompressed file to a Cloud Storage bucket
-you have write access to.
-
-To run a command to decompress the VCF file, type:
-
-```
-dsub \
-  --provider google-batch \
-  --project MY-PROJECT \
-  --regions us-central1 \
-  --logging "gs://MY-BUCKET/decompress_one/logging" \
-  --disk-size 200 \
-  --image ubuntu:14.04 \
-  --input INPUT_VCF="gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/20130723_phase3_wg/cornell/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf.gz" \
-  --output OUTPUT_VCF="gs://MY-BUCKET/decompress_one/output/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf" \
-  --command 'gunzip ${INPUT_VCF} && \
-             mv ${INPUT_VCF%.gz} $(dirname ${OUTPUT_VCF})' \
-  --wait
-```
-
-Set MY-PROJECT to your cloud project name, and set MY-BUCKET to a cloud bucket
-on which you have write privileges.
-
-You should see output like:
-
-```
-Job: gunzip--<userid>--170224-114336-37
-Launched job-id: gunzip--<userid>--170224-114336-37
-        user-id: <userid>
-Waiting for jobs to complete...
-```
-
-Because the `--wait` flag was set, `dsub` will block until the job completes.
-
-### Check the results
-
-To list the output, use the command:
-
-```
-gsutil ls gs://MY-BUCKET/decompress_one/output
-```
-
-Output should look like:
-
-```
-gs://MY-BUCKET/decompress_one/output/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf
-```
-
-To see the first few lines of the decompressed file, run:
-
-```
-gsutil cat gs://MY-BUCKET/decompress_one/output/*.vcf | head -n 5
-```
-
-Output should look like:
-
-```
-##fileformat=VCFv4.1
-##FILTER=<ID=LowQual,Description="Low quality">
-##FILTER=<ID=PASS,Description="Passing basic quality fiters">
-##FORMAT=<ID=AD,Number=.,Type=Integer,Description="Allelic depths for the ref and alt alleles in the order listed">
-##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Approximate read depth (reads with MQ=255 or with bad mates are filtered)">
-```
-
-## Decompress multiple files
-
-`dsub` allows you to define a batch of tasks to submit together using a
-tab-separated values (TSV) file listing the inputs and outputs.
-Each line lists the inputs and outputs for a separate task.
-
-More on dsub batch jobs can be found in the
-[README](../../README#submitting-a-batch-job).
-
-### Create a TSV file
-
-Open an editor and create a file `submit_list.tsv`:
-
-<pre>
---input INPUT_VCF&#9;--output OUTPUT_VCF
-gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/20140708_previous_phase3/v2_vcfs/ALL.chr21.phase3_shapeit2_mvncall_integrated_v2.20130502.genotypes.vcf.gz&#9;gs://MY-BUCKET/decompress_list/output/*.vcf
-gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/20140708_previous_phase3/v1_vcfs/ALL.chr21.phase3_shapeit2_mvncall_integrated.20130502.genotype.vcf.gz&#9;gs://MY-BUCKET/decompress_list/output/*.vcf
-gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/working/20110721_exome_call_sets/bcm/ALL.BCM_Illumina_Mosaik_ontarget_plus50bp_822.20110521.snp.exome.genotypes.vcf.gz&#9;gs://MY-BUCKET/decompress_list/output/*.vcf
-</pre>
-
-The first line of the file lists the input and output parameter names.
-Each subsequent line lists the parameter values.
-Replace MY-BUCKET with a Cloud bucket on which you have write privileges.
-
-Note that for the output parameter, for simplicity, we used wildcards to match
-the 1 VCF file each task outputs instead of explicitly listing the complete
-output file name.
-
-### Submit the job
-
-```
-dsub \
-  --provider google-batch \
-  --project MY-PROJECT \
-  --regions us-central1 \
-  --logging "gs://MY-BUCKET/decompress_list/logging" \
-  --disk-size 200 \
-  --image ubuntu:14.04 \
-  --command 'gunzip ${INPUT_VCF} && \
-             mv ${INPUT_VCF%.gz} $(dirname ${OUTPUT_VCF})' \
-  --tasks submit_list.tsv \
-  --wait
-```
-
-Output should look like:
-
-```
-Job: gunzip--<userid>--170224-122223-54
-Launched job-id: gunzip--<userid>--170224-122223-54
-        user-id: <userid>
-  Task: task-1
-  Task: task-2
-  Task: task-3
-Waiting for jobs to complete...
-```
-
-when all tasks for the job have completed, `dsub` will exit.
-
-### Check the results
-
-To list the output objects, use the command:
-
-```
-gsutil ls gs://MY-BUCKET/decompress_list/output
-```
-
-Output should look like:
-
-```
-gs://MY-BUCKET/decompress_list/output/ALL.BCM_Illumina_Mosaik_ontarget_plus50bp_822.20110521.snp.exome.genotypes.vcf
-gs://MY-BUCKET/decompress_list/output/ALL.ChrY.Cornell.20130502.SNPs.Genotypes.vcf
-gs://MY-BUCKET/decompress_list/output/ALL.chr21.phase3_shapeit2_mvncall_integrated.20130502.genotype.vcf
-gs://MY-BUCKET/decompress_list/output/ALL.chr21.phase3_shapeit2_mvncall_integrated_v2.20130502.genotypes.vcf
-```
-

From 54b65bfc8ddab1638e7727e255ffd4a3f99a3dbb Mon Sep 17 00:00:00 2001
From: Kristen Liu <kristenliu@verily.com>
Date: Fri, 21 Aug 2026 13:00:34 -0700
Subject: [PATCH 08/10] Fixed requested changes with gcloud storage cp,
 retry-max-attempts flag, and gcs header stripping

---
 dsub/providers/local.py                       |  28 +++++++++++-------
 test/.DS_Store                                | Bin 0 -> 8196 bytes
 test/integration/e2e_io_recursive.sh          |   8 +++--
 .../script_block_external_network.sh          |   2 +-
 4 files changed, 25 insertions(+), 13 deletions(-)
 create mode 100644 test/.DS_Store

diff --git a/dsub/providers/local.py b/dsub/providers/local.py
index 0935afb..3f3e71e 100644
--- a/dsub/providers/local.py
+++ b/dsub/providers/local.py
@@ -806,14 +806,10 @@ def _localize_inputs_command(self, task_dir, inputs, user_project):
 
       commands.append('mkdir -p "%s"' % os.path.dirname(local_file_path))
 
-      if i.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
-        # The semantics that we expect here are implemented consistently in
-        # "gcloud storage cp", and are a bit different than "cp" when it comes to
-        # wildcard handling, so use it for both local and GCS:
-        #
-        # - `cp path/* dest/` will error if "path" has subdirectories.
-        # - `cp "path/*" "dest/"` will fail (it expects wildcard expansion
-        #   to come from shell).
+      if i.file_provider == job_model.P_GCS:
+        # `cp path/* dest/` will error if "path" has subdirectories, and
+        # `cp "path/*" "dest/"` will fail (it expects wildcard expansion
+        # to come from shell), so use "gcloud storage cp" instead.
         if user_project:
           command = 'gcloud storage cp --billing-project=%s "%s" "%s"' % (
               user_project, source_file_path, dest_file_path)
@@ -821,6 +817,14 @@ def _localize_inputs_command(self, task_dir, inputs, user_project):
           command = 'gcloud storage cp "%s" "%s"' % (source_file_path,
                                                  dest_file_path)
         commands.append(command)
+      elif i.file_provider == job_model.P_LOCAL:
+        # "gcloud storage cp" does not support local-to-local copies, so use
+        # rsync (without "-r", since this path is non-recursive) instead.
+        # It matches the wildcard semantics we need: like the old "gsutil
+        # cp", it silently skips subdirectories rather than erroring like
+        # plain "cp" does.
+        command = 'rsync "%s" "%s"' % (source_file_path, dest_file_path)
+        commands.append(command)
 
     return '\n'.join(commands)
 
@@ -865,14 +869,18 @@ def _delocalize_outputs_commands(self, task_dir, outputs, user_project):
       if o.file_provider == job_model.P_LOCAL:
         commands.append('mkdir -p "%s"' % dest_path)
 
-      # Use gcloud storage even for local files (explained in _localize_inputs_command).
-      if o.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
+      if o.file_provider == job_model.P_GCS:
         if user_project:
           command = 'gcloud storage cp --billing-project=%s "%s" "%s"' % (user_project, local_path,
                                                        dest_path)
         else:
           command = 'gcloud storage cp "%s" "%s"' % (local_path, dest_path)
         commands.append(command)
+      elif o.file_provider == job_model.P_LOCAL:
+        # "gcloud storage cp" does not support local-to-local copies (see
+        # _localize_inputs_command), so use rsync instead.
+        command = 'rsync "%s" "%s"' % (local_path, dest_path)
+        commands.append(command)
 
     return '\n'.join(commands)
 
diff --git a/test/.DS_Store b/test/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..ac9251cc4645ef64fa9c5ff14c7e4aff2a002d8c
GIT binary patch
literal 8196
zcmeHMTWl3Y7@lui=<IUa)&gDd(4!SBrG-+;9rg5rT#6QHX-l~r_MEfO4cpyvcF!qD
zh!rmnYKVy%O;lpMJc>c1F-DChJ`goN7@}gL@sjwW4?gf@qWI6w9?};2Vu&GOCz<)@
z|7ZT0oo~K>=gc0)7}^WDm5kLe#yDjm^%+!LBZ+pAuS!Z}DJKc?XPHrdB;$A)(%-+T
zcWj6eh!Kbph!Kbph!MCE5TG+#BtFHtFI(d>Mj%GurbIxzACi=XOhs}`%J9`eMR)}u
zDXsuvqPi|BB$JU$MRH6^YM}(_N>aKax?(`OlRX*rOGR=_O6krJT|N-ajOd1faCRC$
z8Fpt#Ng0<h0x<$tBOt=3oE4bMY!-~azh_1q+Yg$XzkyO#K5cphui!KI{+=Okv?nlw
zg5PfD_X@4o^PEhnZf5#yYc#D_cX)1K+OCxk9qgP%UH0|cZq6HR_X=*#4}E)N16R44
z)@x3jSlPU?G1;=F^<-o6#QK)z#$+qiPpW)w-HMI7(np+O*Ly-P0nuxK&6>2NC&#bH
z@rxbudqmm1(t5rttw+^n7=8T%T3W9v(nEKT1h(h;eOBHlnbzm{w76|q&)u1~M0<bM
za|*+*pVlk0rjso=W?;2BPLKVd<qrhbu|U)YeJ>w$gdGQL@)fqUt`~f3mqiYC(dr~O
zzk1R`Lp?ilZsPVO%bHhjxTkaLj-3}~&7PxcRa$j_(e~|(W9=TbbHUIa)6ZIN&UOzE
z5^`=}AF{2yT8S>T*ENT&3-jjJ=*9x$)~UR>UdIOt8T)9#4vxz?YK_}e{awB<N6&Si
zmlGrA!WN^}Sj_oeVeiOWR7$f}Z#1a7$$NcTnjE4@TcPT{+IxhySGu-ntz6HkyB#xQ
zInuONThH|_wZACwmBvll-MkWw>Oj^T9;Ws9(%fcraQ!?#Am-j-`a$=I<(8hR&Dg^E
zpg&~ht?p6cSIMJ`^Kj3|E?hR}?7+)QKx&O{&i9DH+HyG>N1tCa?}1&0K^tScMVr`6
z8!kz$f6Y0sOEt*-2|=w(HA_vU6Ut!A=_K6B_A!f%u;c70_A)!e-em8wkJ)GJD|V56
z&wgM(vR~OB>=OG6<(P&Fa8zOe79xQ~ScX=t#u~Jv3p;Q>y3vdMIDmsNaR?qpkcW@s
zIE6>>7#_!IJdYRfB3{B9IE%M%4sYXAe2y>hCBDT^_!+<868^?NxU8s3m9kj5Q>j;$
zDh<jiWu4L{A6+R$%iu)fDea_RiGGTf=<A(w^jgu9xOdCe8zWjS&J^)cS#?YG+(mWu
zO)YCSrmj#(lCH$F@%#-bJAvpSWy4*b%r%N6W5Gf#v70z;9h)3|rTEe-1bexlnNge6
z68bb^y;Sd5OzL!Eyi}JqBoewpfhN`ED-xPsMq#9ezN;EYMWn1<r)hdQ1(a;tbQh^i
z#JJSk+G!*a9{+);=hz4A6Ly|mAfo=perJEO%S6uEs3u}AK|PVP5luwSb=ZK7*o4j4
zf^FDNRP99{2CxT%$il*548bO%1}LD2hww0w^--ehX*`W*@GPFg8N7m5@fuzaY5NX7
zoJ7R!6N!kA#uKsFlXo4@JxsO|aVK_<wd;aFnH;4?-2XRE{r&&C%oPt3BM>8SV<Ld^
z&Yn*3Q(&sMt+;C^DIcUPlIS-kW#~eM=pyXlb)4jtKMbj!jM`F>9Fvk-DE;pr0_b;O
R>4$E7{>SHkNWz=2_!o%I8mRyP

literal 0
HcmV?d00001

diff --git a/test/integration/e2e_io_recursive.sh b/test/integration/e2e_io_recursive.sh
index 7163281..9b6128b 100755
--- a/test/integration/e2e_io_recursive.sh
+++ b/test/integration/e2e_io_recursive.sh
@@ -134,10 +134,14 @@ echo
 echo "On-disk output file list matches expected"
 
 # Verify in GCS that the DEEP directory is deep and the SHALLOW directory
-# is shallow. Gcloud storage prints directories with a trailing "/" that is
-# stripped using sed in order to match the output format of the `find` utility.
+# is shallow. For each directory, `gcloud storage ls -r` prints both a
+# redundant header line ending in "/:" and a self-referencing entry line
+# ending in "/". The header lines are dropped and the trailing "/" is
+# stripped from the remaining lines to match the output format of the
+# `find` utility.
 readonly GCS_FIND="$(gcloud storage ls -r "${OUTPUTS}" \
                      | grep -v '^ *$' \
+                     | grep -v ':$' \
                      | sed -e 's#/$##')"
 
 for REC in "${EXPECTED_REMOTE_OUTPUT_ENTRIES[@]}"; do
diff --git a/test/integration/script_block_external_network.sh b/test/integration/script_block_external_network.sh
index af1eadf..052ace3 100755
--- a/test/integration/script_block_external_network.sh
+++ b/test/integration/script_block_external_network.sh
@@ -21,7 +21,7 @@ set -o nounset
 
 RC=0
 
-if ! gcloud storage ls --retry-max-attempts=0 gs://genomics-public-data; then
+if ! CLOUDSDK_STORAGE_MAX_RETRIES=0 gcloud storage ls gs://genomics-public-data; then
   1>&2 echo "\`gcloud storage ls\` should not have succeeded"
   RC=1
 fi

From 139ffbe1611bfc9821b2e7934a62de9a6a177df5 Mon Sep 17 00:00:00 2001
From: "Kristen Liu (Ong)" <43296048+kvo3@users.noreply.github.com>
Date: Fri, 21 Aug 2026 13:06:45 -0700
Subject: [PATCH 09/10] Delete test/.DS_Store

---
 test/.DS_Store | Bin 8196 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 test/.DS_Store

diff --git a/test/.DS_Store b/test/.DS_Store
deleted file mode 100644
index ac9251cc4645ef64fa9c5ff14c7e4aff2a002d8c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 8196
zcmeHMTWl3Y7@lui=<IUa)&gDd(4!SBrG-+;9rg5rT#6QHX-l~r_MEfO4cpyvcF!qD
zh!rmnYKVy%O;lpMJc>c1F-DChJ`goN7@}gL@sjwW4?gf@qWI6w9?};2Vu&GOCz<)@
z|7ZT0oo~K>=gc0)7}^WDm5kLe#yDjm^%+!LBZ+pAuS!Z}DJKc?XPHrdB;$A)(%-+T
zcWj6eh!Kbph!Kbph!MCE5TG+#BtFHtFI(d>Mj%GurbIxzACi=XOhs}`%J9`eMR)}u
zDXsuvqPi|BB$JU$MRH6^YM}(_N>aKax?(`OlRX*rOGR=_O6krJT|N-ajOd1faCRC$
z8Fpt#Ng0<h0x<$tBOt=3oE4bMY!-~azh_1q+Yg$XzkyO#K5cphui!KI{+=Okv?nlw
zg5PfD_X@4o^PEhnZf5#yYc#D_cX)1K+OCxk9qgP%UH0|cZq6HR_X=*#4}E)N16R44
z)@x3jSlPU?G1;=F^<-o6#QK)z#$+qiPpW)w-HMI7(np+O*Ly-P0nuxK&6>2NC&#bH
z@rxbudqmm1(t5rttw+^n7=8T%T3W9v(nEKT1h(h;eOBHlnbzm{w76|q&)u1~M0<bM
za|*+*pVlk0rjso=W?;2BPLKVd<qrhbu|U)YeJ>w$gdGQL@)fqUt`~f3mqiYC(dr~O
zzk1R`Lp?ilZsPVO%bHhjxTkaLj-3}~&7PxcRa$j_(e~|(W9=TbbHUIa)6ZIN&UOzE
z5^`=}AF{2yT8S>T*ENT&3-jjJ=*9x$)~UR>UdIOt8T)9#4vxz?YK_}e{awB<N6&Si
zmlGrA!WN^}Sj_oeVeiOWR7$f}Z#1a7$$NcTnjE4@TcPT{+IxhySGu-ntz6HkyB#xQ
zInuONThH|_wZACwmBvll-MkWw>Oj^T9;Ws9(%fcraQ!?#Am-j-`a$=I<(8hR&Dg^E
zpg&~ht?p6cSIMJ`^Kj3|E?hR}?7+)QKx&O{&i9DH+HyG>N1tCa?}1&0K^tScMVr`6
z8!kz$f6Y0sOEt*-2|=w(HA_vU6Ut!A=_K6B_A!f%u;c70_A)!e-em8wkJ)GJD|V56
z&wgM(vR~OB>=OG6<(P&Fa8zOe79xQ~ScX=t#u~Jv3p;Q>y3vdMIDmsNaR?qpkcW@s
zIE6>>7#_!IJdYRfB3{B9IE%M%4sYXAe2y>hCBDT^_!+<868^?NxU8s3m9kj5Q>j;$
zDh<jiWu4L{A6+R$%iu)fDea_RiGGTf=<A(w^jgu9xOdCe8zWjS&J^)cS#?YG+(mWu
zO)YCSrmj#(lCH$F@%#-bJAvpSWy4*b%r%N6W5Gf#v70z;9h)3|rTEe-1bexlnNge6
z68bb^y;Sd5OzL!Eyi}JqBoewpfhN`ED-xPsMq#9ezN;EYMWn1<r)hdQ1(a;tbQh^i
z#JJSk+G!*a9{+);=hz4A6Ly|mAfo=perJEO%S6uEs3u}AK|PVP5luwSb=ZK7*o4j4
zf^FDNRP99{2CxT%$il*548bO%1}LD2hww0w^--ehX*`W*@GPFg8N7m5@fuzaY5NX7
zoJ7R!6N!kA#uKsFlXo4@JxsO|aVK_<wd;aFnH;4?-2XRE{r&&C%oPt3BM>8SV<Ld^
z&Yn*3Q(&sMt+;C^DIcUPlIS-kW#~eM=pyXlb)4jtKMbj!jM`F>9Fvk-DE;pr0_b;O
R>4$E7{>SHkNWz=2_!o%I8mRyP


From 773f4612432f29ca5775866ce193a86c784f4086 Mon Sep 17 00:00:00 2001
From: Kristen Liu <kristenliu@verily.com>
Date: Fri, 21 Aug 2026 13:55:59 -0700
Subject: [PATCH 10/10] updated gitignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index b46da6e..3680446 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,5 @@ build/
 dist/
 dsub_libs/
 dsub.egg-info/
+.ipynb_checkpoints/
+.DS_Store