Replaced all instances of gsutil with gcloud

.gitignore

@@ -5,3 +5,5 @@ build/
 dist/
 dsub_libs/
 dsub.egg-info/
+.ipynb_checkpoints/
+.DS_Store

README.md

@@ -56,7 +56,7 @@ While not used directly by `dsub` for the `google-batch` provider, you are likel
 Cloud SDK](https://cloud.google.com/sdk/).
 
 If you will be using the `local` provider for faster job development,
-you *will* need to install the Google Cloud SDK, which uses `gsutil` to ensure
+you *will* need to install the Google Cloud SDK, which uses `gcloud storage` to ensure
 file operation semantics consistent with the Google `dsub` providers.
 
 1. [Install the Google Cloud SDK](https://cloud.google.com/sdk/)
@@ -182,10 +182,10 @@ The steps for getting started differ slightly as indicated in the steps below:
 
     The dsub logs and output files will be written to a bucket. Create a
     bucket using the [storage browser](https://console.cloud.google.com/storage/browser?project=)
-    or run the command-line utility [gsutil](https://cloud.google.com/storage/docs/gsutil),
+    or run the command-line utility [gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage),
     included in the Cloud SDK.
 
-        gsutil mb gs://my-bucket
+        gcloud storage buckets create gs://my-bucket
 
     Change `my-bucket` to a unique name that follows the
     [bucket-naming conventions](https://cloud.google.com/storage/docs/bucket-naming).
@@ -215,7 +215,7 @@ The steps for getting started differ slightly as indicated in the steps below:
 
 1. View the output file.
 
-        gsutil cat gs://my-bucket/output/out.txt
+        gcloud storage cat gs://my-bucket/output/out.txt
 
 ## Backend providers
 
@@ -351,9 +351,8 @@ by:
 
 To upload the files to Google Cloud Storage, you can use the
 [storage browser](https://console.cloud.google.com/storage/browser?project=) or
-[gsutil](https://cloud.google.com/storage/docs/gsutil). You can also run on data
-that’s public or shared with your service account, an email address that you
-can find in the [Google Cloud Console](https://console.cloud.google.com).
+[gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage). 
+You can also run on data that’s public or shared with your service account, an email address that you can find in the [Google Cloud Console](https://console.cloud.google.com).
 
 #### Files
 
@@ -728,7 +727,9 @@ of the service account will be `sa-name@project-id.iam.gserviceaccount.com`.
 
 2. Grant IAM access on buckets, etc. to the service account.
 
-        gsutil iam ch serviceAccount:sa-name@project-id.iam.gserviceaccount.com:roles/storage.objectAdmin gs://bucket-name
+        gcloud storage buckets add-iam-policy-binding gs://bucket-name
+            --member=serviceAccount:sa-name@project-id.iam.gserviceaccount.com
+            --role=roles/storage.objectAdmin
 
 3. Update your `dsub` command to include `--service-account`
 

docs/code.md

@@ -187,7 +187,7 @@ To run the driver script, first copy `script1.sh` and `script2.sh` to
 cloud storage:
 
 ```
-gsutil cp my-code/script1.sh my-code/script2.sh gs://MY-BUCKET/my-code/
+gcloud storage cp my-code/script1.sh my-code/script2.sh gs://MY-BUCKET/my-code/
 ```
 
 Then launch a dsub job:
@@ -205,7 +205,7 @@ Extending the previous example, you could copy `script1.sh` and `script2.sh`
 to cloud storage with:
 
 ```
-gsutil rsync -r my-code gs://MY-BUCKET/my-code/
+gcloud storage rsync -r my-code gs://MY-BUCKET/my-code/
 ```
 
 and then launch a `dsub` job with:

docs/providers/README.md

@@ -131,8 +131,8 @@ copying output files.
 
 The copying of files is performed in the host environment, not inside the
 Docker container. This means that for copying to/from Google Cloud Storage,
-the host environment requires a copy of
-[gsutil](https://cloud.google.com/storage/docs/gsutil) to be installed.
+the host environment requires the
+[Google Cloud SDK](https://cloud.google.com/sdk/docs/install) to be installed.
 
 #### Container runtime environment
 

dsub/_dsub_version.py

@@ -27,5 +27,5 @@
 """
 
 
-DSUB_VERSION = '0.5.2'
+DSUB_VERSION = '0.5.3.dev0'
 

dsub/lib/param_util.py

@@ -827,23 +827,23 @@ def directory_fmt(directory):
 
   Multiple files copy, works as intended in all cases:
       $ touch a.txt b.txt
-      $ gsutil cp ./*.txt gs://mybucket/text_dest
-      $ gsutil ls gs://mybucket/text_dest/
+      $ gcloud storage cp ./*.txt gs://mybucket/text_dest
+      $ gcloud storage ls gs://mybucket/text_dest/
             0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/a.txt
             0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/b.txt
       TOTAL: 2 objects, 0 bytes (0 B)
 
   Single file copy fails to copy into a directory:
       $ touch 1.bam
-      $ gsutil cp ./*.bam gs://mybucket/bad_dest
-      $ gsutil ls gs://mybucket/bad_dest
+      $ gcloud storage cp ./*.bam gs://mybucket/bad_dest
+      $ gcloud storage ls gs://mybucket/bad_dest
                0  2017-07-19T21:46:16Z  gs://mybucket/bad_dest
       TOTAL: 1 objects, 0 bytes (0 B)
 
   Adding a trailing forward slash fixes this:
       $ touch my.sam
-      $ gsutil cp ./*.sam gs://mybucket/good_folder
-      $ gsutil ls gs://mybucket/good_folder
+      $ gcloud storage cp ./*.sam gs://mybucket/good_folder
+      $ gcloud storage ls gs://mybucket/good_folder
                0  2017-07-19T21:46:16Z  gs://mybucket/good_folder/my.sam
       TOTAL: 1 objects, 0 bytes (0 B)
 

dsub/lib/providers_util.py

@@ -20,7 +20,7 @@
 from .._dsub_version import DSUB_VERSION
 
 _LOCALIZE_COMMAND_MAP = {
-    job_model.P_GCS: 'gsutil -m rsync -r',
+    job_model.P_GCS: 'gcloud storage rsync -r',
     job_model.P_LOCAL: 'rsync -r',
 }
 

dsub/providers/google_batch.py

@@ -145,11 +145,11 @@ def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None
       "${{LOGGING_DIR}}/stderr.txt" \
       "{user_action}"
 
-  gsutil_cp "${{LOGGING_DIR}}/stdout.txt" "${{STDOUT_PATH}}" "text/plain" "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/stdout.txt" "${{STDOUT_PATH}}" "text/plain" "${{USER_PROJECT}}" &
   STDOUT_PID=$!
-  gsutil_cp "${{LOGGING_DIR}}/stderr.txt" "${{STDERR_PATH}}" "text/plain"  "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/stderr.txt" "${{STDERR_PATH}}" "text/plain"  "${{USER_PROJECT}}" &
   STDERR_PID=$!
-  gsutil_cp "${{LOGGING_DIR}}/log.txt" "${{LOGGING_PATH}}" "text/plain" "${{USER_PROJECT}}" &
+  gcloud_cp "${{LOGGING_DIR}}/log.txt" "${{LOGGING_PATH}}" "text/plain" "${{USER_PROJECT}}" &
   LOG_PID=$!
 
   wait "${{STDOUT_PID}}"
@@ -168,7 +168,7 @@ def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None
   touch "${{LOGGING_DIR}}/.stop_logging"
 
   {log_msg_fn}
-  {gsutil_cp_fn}
+  {gcloud_cp_fn}
 
   {log_cp}
 """)
@@ -182,14 +182,14 @@ def copy_log_to_staging(glob_str: str, staging_path: str, filter_str: str = None
   readonly LOGGING_DIR="{logging_dir}"
 
   {log_msg_fn}
-  {gsutil_cp_fn}
+  {gcloud_cp_fn}
 
   # Make sure the logging work directory exists
   mkdir -p "${{LOGGING_DIR}}"
 
   # Prep the log filter script
   echo "${{{log_filter_var}}}" \
-    | python -c '{python_decode_script}' \
+    | python3 -c '{python_decode_script}' \
     > "{log_filter_script_path}"
   chmod a+x "{log_filter_script_path}"
 
@@ -609,7 +609,7 @@ def _create_batch_request(
 
     continuous_logging_cmd = _CONTINUOUS_LOGGING_CMD.format(
         log_msg_fn=google_utils.LOG_MSG_FN,
-        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
+        gcloud_cp_fn=google_utils.GCLOUD_CP_FN,
         log_filter_var=_LOG_FILTER_VAR,
         log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
         python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
@@ -623,7 +623,7 @@ def _create_batch_request(
 
     logging_cmd = _FINAL_LOGGING_CMD.format(
         log_msg_fn=google_utils.LOG_MSG_FN,
-        gsutil_cp_fn=google_utils.GSUTIL_CP_FN,
+        gcloud_cp_fn=google_utils.GCLOUD_CP_FN,
         log_filter_var=_LOG_FILTER_VAR,
         log_filter_script_path=_LOG_FILTER_SCRIPT_PATH,
         python_decode_script=google_utils.PYTHON_DECODE_SCRIPT,
@@ -729,8 +729,8 @@ def _create_batch_request(
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.LOCALIZATION_LOOP,
                 ),
             ],
@@ -778,8 +778,8 @@ def _create_batch_request(
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.DELOCALIZATION_LOOP,
                 ),
             ],

dsub/providers/google_utils.py

@@ -68,9 +68,9 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
 # pylint: enable=g-complex-comprehension
 
 
-# Action steps that interact with GCS need gsutil and Python.
+# Action steps that interact with GCS need gcloud and Python.
 # Use the 'slim' variant of the cloud-sdk image as it is much smaller.
-CLOUD_SDK_IMAGE = 'gcr.io/google.com/cloudsdktool/cloud-sdk:294.0.0-slim'
+CLOUD_SDK_IMAGE = 'gcr.io/google.com/cloudsdktool/cloud-sdk:499.0.0-slim'
 
 # Name of the data disk
 DATA_DISK_NAME = 'datadisk'
@@ -94,44 +94,44 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
   }
 """)
 
-# Define a bash function for "gsutil cp" to be used by the logging,
+# Define a bash function for "gcloud storage cp" to be used by the logging,
 # localization, and delocalization actions.
-GSUTIL_CP_FN = textwrap.dedent("""\
-  function gsutil_cp() {
+GCLOUD_CP_FN = textwrap.dedent("""\
+  function gcloud_cp() {
     local src="${1}"
     local dst="${2}"
     local content_type="${3}"
     local user_project_name="${4}"
 
     local headers=""
     if [[ -n "${content_type}" ]]; then
-      headers="-h Content-Type:${content_type}"
+      headers="--content-type=${content_type}"
     fi
 
     local user_project_flag=""
     if [[ -n "${user_project_name}" ]]; then
-      user_project_flag="-u ${user_project_name}"
+      user_project_flag="--billing-project=${user_project_name}"
     fi
 
     local attempt
     for ((attempt = 0; attempt < 4; attempt++)); do
-      log_info "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
-      if gsutil ${headers} ${user_project_flag} -mq cp "${src}" "${dst}"; then
+      log_info "gcloud storage cp ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
+      if gcloud storage cp ${headers} ${user_project_flag} "${src}" "${dst}"; then
         return
       fi
       if (( attempt < 3 )); then
-        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
-        log_warning "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+        log_warning "Sleeping 10s before the next attempt of failed gcloud command"
+        log_warning "gcloud storage cp ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
         sleep 10s
       fi
     done
 
-    log_error "gsutil ${headers} ${user_project_flag} -mq cp \"${src}\" \"${dst}\""
+    log_error "gcloud storage cp ${headers} ${user_project_flag} \"${src}\" \"${dst}\""
     exit 1
   }
 """)
 
-LOG_CP_FN = GSUTIL_CP_FN + textwrap.dedent("""\
+LOG_CP_FN = GCLOUD_CP_FN + textwrap.dedent("""\
 
   function log_cp() {
     local src="${1}"
@@ -144,43 +144,43 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
       return
     fi
 
-    # Copy the log files to a local temporary location so that our "gsutil cp" is never
+    # Copy the log files to a local temporary location so that our "gcloud storage cp" is never
     # executed on a file that is changing.
 
     local tmp_path="${tmp}/$(basename ${src})"
     cp "${src}" "${tmp_path}"
 
-    gsutil_cp "${tmp_path}" "${dst}" "text/plain" "${user_project_name}"
+    gcloud_cp "${tmp_path}" "${dst}" "text/plain" "${user_project_name}"
   }
 """)
 
-# Define a bash function for "gsutil rsync" to be used by the logging,
+# Define a bash function for "gcloud rsync" to be used by the logging,
 # localization, and delocalization actions.
-GSUTIL_RSYNC_FN = textwrap.dedent("""\
-  function gsutil_rsync() {
+GCLOUD_RSYNC_FN = textwrap.dedent("""\
+  function gcloud_rsync() {
     local src="${1}"
     local dst="${2}"
     local user_project_name="${3}"
 
     local user_project_flag=""
     if [[ -n "${user_project_name}" ]]; then
-      user_project_flag="-u ${user_project_name}"
+      user_project_flag="--billing-project=${user_project_name}"
     fi
 
     local attempt
     for ((attempt = 0; attempt < 4; attempt++)); do
-      log_info "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
-      if gsutil ${user_project_flag} -mq rsync -r "${src}" "${dst}"; then
+      log_info "gcloud storage rsync -r ${user_project_flag} \"${src}\" \"${dst}\""
+      if gcloud storage rsync -r ${user_project_flag} "${src}" "${dst}"; then
         return
       fi
       if (( attempt < 3 )); then
-        log_warning "Sleeping 10s before the next attempt of failed gsutil command"
-        log_warning "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+        log_warning "Sleeping 10s before the next attempt of failed gcloud command"
+        log_warning "gcloud storage rsync -r ${user_project_flag} \"${src}\" \"${dst}\""
         sleep 10s
       fi
     done
 
-    log_error "gsutil ${user_project_flag} -mq rsync -r \"${src}\" \"${dst}\""
+    log_error "gcloud storage rsync -r ${user_project_flag} \"${src}\" \"${dst}\""
     exit 1
   }
 """)
@@ -198,9 +198,9 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
 
     log_info "Localizing ${!INPUT_VAR}"
     if [[ "${!INPUT_RECURSIVE}" -eq "1" ]]; then
-      gsutil_rsync "${!INPUT_SRC}" "${!INPUT_DST}" "${USER_PROJECT}"
+      gcloud_rsync "${!INPUT_SRC}" "${!INPUT_DST}" "${USER_PROJECT}"
     else
-      gsutil_cp "${!INPUT_SRC}" "${!INPUT_DST}" "" "${USER_PROJECT}"
+      gcloud_cp "${!INPUT_SRC}" "${!INPUT_DST}" "" "${USER_PROJECT}"
     fi
   done
 """)
@@ -218,9 +218,9 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
 
     log_info "Delocalizing ${!OUTPUT_VAR}"
     if [[ "${!OUTPUT_RECURSIVE}" -eq "1" ]]; then
-      gsutil_rsync "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "${USER_PROJECT}"
+      gcloud_rsync "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "${USER_PROJECT}"
     else
-      gsutil_cp "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "" "${USER_PROJECT}"
+      gcloud_cp "${!OUTPUT_SRC}" "${!OUTPUT_DST}" "" "${USER_PROJECT}"
     fi
   done
 """)
@@ -274,7 +274,7 @@ def make_runtime_dirs_command(script_dir: str, tmp_dir: str,
   {mk_runtime_dirs}
 
   echo "${{{script_var}}}" \
-    | python -c '{python_decode_script}' \
+    | python3 -c '{python_decode_script}' \
     > "{script_path}"
   chmod a+x "{script_path}"
 

dsub/providers/google_v2_base.py

@@ -529,8 +529,8 @@ def _build_pipeline_request(self, task_view):
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.LOCALIZATION_LOOP,
                 ),
             ],
@@ -566,8 +566,8 @@ def _build_pipeline_request(self, task_view):
                 '-c',
                 google_utils.LOCALIZATION_CMD.format(
                     log_msg_fn=google_utils.LOG_MSG_FN,
-                    recursive_cp_fn=google_utils.GSUTIL_RSYNC_FN,
-                    cp_fn=google_utils.GSUTIL_CP_FN,
+                    recursive_cp_fn=google_utils.GCLOUD_RSYNC_FN,
+                    cp_fn=google_utils.GCLOUD_CP_FN,
                     cp_loop=google_utils.DELOCALIZATION_LOOP,
                 ),
             ],