Update workflow

CHANGELOG.md

@@ -190,6 +190,9 @@
   * Mid time, mem, cpu for most methods, except below.
   * High (or very high) time, mem, cpu for computationally expensive methods like rPCA.
 
+* `run_benchmark`: write the commit the workflow ran from and the launch time into
+  `task_info.yaml`, instead of publishing `_viash.yaml` verbatim (PR #136).
+
 
 ## BUG FIXES
 

scripts/run_benchmark/run_full_denbi.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+# generate a unique id
+RUN_ID="run_$(date +%Y-%m-%d_%H-%M-%S)"
+publish_dir="/vol/scratch/results/task_cyto_batch_integration/${RUN_ID}"
+
+# write the parameters to file
+cat > /tmp/params.yaml << HERE
+input_states: s3://openproblems-data/resources/task_cyto_batch_integration/datasets/**/state.yaml
+rename_keys: 'input_censored_split1:output_censored_split1;input_censored_split2:output_censored_split2;input_unintegrated:output_unintegrated'
+output_state: "state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+tw launch https://github.com/openproblems-bio/task_cyto_batch_integration.git \
+  --revision build/main \
+  --pull-latest \
+  --main-script target/nextflow/workflows/run_benchmark/main.nf \
+  --workspace 53907369739130 \
+  --compute-env 3qstFmP9lNwdzutSNuJq7c \
+  --params-file /tmp/params.yaml \
+  --entry-name auto \
+  --config common/nextflow_helpers/labels_denbi.config \
+  --labels task_cyto_batch_integration,full,denbi

scripts/run_benchmark/run_test_denbi.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+resources_test_s3=s3://openproblems-data/resources_test/task_cyto_batch_integration
+
+# generate a unique id
+RUN_ID="testrun_$(date +%Y-%m-%d_%H-%M-%S)"
+publish_dir="/vol/scratch/results/task_cyto_batch_integration/${RUN_ID}"
+
+# write the parameters to file
+cat > /tmp/params.yaml << HERE
+id: mouse_spleen_flow_cytometry_subset
+input_unintegrated: $resources_test_s3/mouse_spleen_flow_cytometry_subset/unintegrated.h5ad
+input_censored_split1: $resources_test_s3/mouse_spleen_flow_cytometry_subset/censored_split1.h5ad
+input_censored_split2: $resources_test_s3/mouse_spleen_flow_cytometry_subset/censored_split2.h5ad
+output_state: "state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+tw launch https://github.com/openproblems-bio/task_cyto_batch_integration.git \
+  --revision build/main \
+  --pull-latest \
+  --main-script target/nextflow/workflows/run_benchmark/main.nf \
+  --workspace 53907369739130 \
+  --compute-env 3qstFmP9lNwdzutSNuJq7c \
+  --params-file /tmp/params.yaml \
+  --config common/nextflow_helpers/labels_denbi.config \
+  --labels task_cyto_batch_integration,test,denbi

src/workflows/run_benchmark/main.nf

@@ -248,14 +248,20 @@ workflow run_wf {
       metric_configs_file.write(metric_configs_yaml_blob)
 
       // store the task info in a file
-      def viash_file = meta.resources_dir.resolve("_viash.yaml")
+      def task_info = readYaml(meta.resources_dir.resolve("_viash.yaml"))
+      if (workflow.commitId) {
+        task_info.commit = workflow.commitId
+      }
+      task_info.timestamp = workflow.start.toInstant().truncatedTo(java.time.temporal.ChronoUnit.SECONDS).toString()
+      def task_info_file = tempFile("task_info.yaml")
+      task_info_file.write(toYamlBlob(task_info))
 
       // create output state
       def new_state = [
         output_dataset_info: dataset_uns_file,
         output_method_configs: method_configs_file,
         output_metric_configs: metric_configs_file,
-        output_task_info: viash_file,
+        output_task_info: task_info_file,
         _meta: states[0]._meta
       ]