From 46725de2a6a6da73295d86fb5d295224507bd610 Mon Sep 17 00:00:00 2001 From: Jobin Sunny Date: Wed, 5 Aug 2026 12:45:45 -0500 Subject: [PATCH] feat: test updated tsne --- dx_app/dxapp.json | 110 +++++++++++++------ dx_app/resources/stjude/bin/generate_plot.py | 2 + dx_app/src/t-SNE.sh | 8 +- scripts/itsne-normalize-matrix.R | 5 +- 4 files changed, 85 insertions(+), 40 deletions(-) diff --git a/dx_app/dxapp.json b/dx_app/dxapp.json index 9d9fe5e..9abcdd7 100644 --- a/dx_app/dxapp.json +++ b/dx_app/dxapp.json @@ -1,10 +1,12 @@ { - "name": "stjude_tsne", - "title": "St. Jude Cloud Interactive t-SNE", + "name": "stjude_tsne_test", + "title": "St. Jude Cloud Interactive t-SNE (TEST)", "summary": "Produce a t-SNE plot that highlights input sample(s) against a cohort of existing samples.", "dxapi": "1.0.0", - "version": "EXPRESSION_SHORT_VERSION", - "developers": [ "user-athrashe" ], + "version": "0.8.2", + "developers": [ + "user-athrashe" + ], "authorizedUsers": [ "user-athrashe", "org-stjude_cloud_app_runners", @@ -17,7 +19,9 @@ "class": "array:file", "optional": true, "help": "Provide a cohort of samples to use as reference data", - "patterns": [ "*.feature-counts.txt" ] + "patterns": [ + "*.feature-counts.txt" + ] }, { "name": "input_counts", @@ -25,9 +29,11 @@ "class": "array:file", "optional": true, "help": "Provide sample(s) to compare. File must have the following properties: sample_name, library_type, read_length, strandedness, and pairing.", - "patterns": [ "*.feature-counts.txt" ] + "patterns": [ + "*.feature-counts.txt" + ] }, - { + { "name": "tumor_type", "label": "Reference tumor type(s)", "class": "string", @@ -38,7 +44,7 @@ }, { "name": "output_name", - "label": "Output file name", + "label": "Output file name", "class": "string", "default": "samples.html", "help": "Specify a filename for the output HTML plot. Name should be a valid unix file name (e.g. no space or reserved characters).", @@ -48,44 +54,62 @@ "name": "preservatives", "label": "FFPE and/or Fresh/Frozen", "class": "string", - "choices": ["FFPE", "Fresh/Frozen", "all"], + "choices": [ + "FFPE", + "Fresh/Frozen", + "all" + ], "optional": false, "help": "Specify preservative(s) to include in analysis.", "default": "Fresh/Frozen", "group": "Analysis Settings" }, - { + { "name": "all_strandedness", - "label": "Strandedness", - "class": "string", - "choices": ["Stranded-Forward", "Stranded-Reverse", "Unstranded"], + "label": "Strandedness", + "class": "string", + "choices": [ + "Stranded-Forward", + "Stranded-Reverse", + "Unstranded" + ], "optional": true, "help": "Specify whether library was stranded. This option will apply to any input samples without a specification.", "group": "Input sample parameters" }, - { + { "name": "all_library_type", - "label": "Library type", - "class": "string", - "choices": ["PolyA", "Total"], + "label": "Library type", + "class": "string", + "choices": [ + "PolyA", + "Total" + ], "optional": true, "help": "Specify library type. This option will apply to any input samples without a specification.", "group": "Input sample parameters" }, - { + { "name": "all_read_length", - "label": "Read length", - "class": "string", - "choices": ["101", "126", "150"], + "label": "Read length", + "class": "string", + "choices": [ + "101", + "126", + "150" + ], "optional": true, "help": "Specify read length. This option will apply to any input samples without a specification.", "group": "Input sample parameters" }, - { + { "name": "all_pairing", - "label": "Read pairing", - "class": "string", - "choices": ["Paired-end", "Single-end"], + "label": "Read pairing", + "class": "string", + "choices": [ + "Paired-end", + "Single-end" + ], "optional": true, "help": "Specify read pairing. This option will apply to any input samples without a specification.", "group": "Input sample parameters" @@ -95,7 +119,13 @@ "label": "Return intermediate file", "class": "string", "optional": true, - "choices": ["original_matrix", "variance_stabilized", "batch_corrected", "tsne_output", "matrix"], + "choices": [ + "original_matrix", + "variance_stabilized", + "batch_corrected", + "tsne_output", + "matrix" + ], "help": "If desired, return data from an intermediate point in the analysis. 'original_matrix' is the matrix of counts. 'variance_stabilized' is the matrix after running varianceStabilizingTransformation. 'batch_corrected' is the matrix after ComBat batch correction. 'tsne_output' is the matrix after Rtsne has been run. 'matrix' is the file input to the R script for plotting.", "group": "Analysis Settings" }, @@ -122,7 +152,6 @@ "help": "If true, DFCI PDX samples will be included in the t-SNE to facilitate model selection", "group": "Analysis Settings" } - ], "outputSpec": [ { @@ -181,11 +210,24 @@ "distribution": "Ubuntu", "file": "src/t-SNE.sh", "execDepends": [ - { "name": "parallel"}, - { "name": "aria2"}, - { "name": "csvkit", "package_manager": "pip"}, - { "name": "pandas", "package_manager": "pip"}, - { "name": "logzero", "package_manager": "pip"} + { + "name": "parallel" + }, + { + "name": "aria2" + }, + { + "name": "csvkit", + "package_manager": "pip" + }, + { + "name": "pandas", + "package_manager": "pip" + }, + { + "name": "logzero", + "package_manager": "pip" + } ] }, "regionalOptions": { @@ -199,8 +241,8 @@ }, "access": { "network": [ - "*" + "*" ], "allProjects": "VIEW" - } + } } diff --git a/dx_app/resources/stjude/bin/generate_plot.py b/dx_app/resources/stjude/bin/generate_plot.py index 70c2bfe..7dfc8da 100755 --- a/dx_app/resources/stjude/bin/generate_plot.py +++ b/dx_app/resources/stjude/bin/generate_plot.py @@ -58,6 +58,8 @@ def label_samples (row): # Trim precision on x/y coordinates matrix = matrix.round({'t1': 2, 't2': 2}) # Read the metadata file + print("Metadata file") + print(args.metadata_file) metadata = pd.read_json(args.metadata_file) # Convert metadata JSON column 'properties' into a table. diff --git a/dx_app/src/t-SNE.sh b/dx_app/src/t-SNE.sh index 961b003..3e38046 100755 --- a/dx_app/src/t-SNE.sh +++ b/dx_app/src/t-SNE.sh @@ -494,17 +494,17 @@ main() { gene_list_arg="--gene-list ${container_reference_dir}/gene_list.txt" fi - echo "docker run -v $local_data_dir:$container_data_dir -v $local_reference_dir:$container_reference_dir -v $local_output_dir:$container_output_dir ghcr.io/stjudecloud/expression-classification:EXPRESSION_VERSION bash -c \"cd $container_output_dir && itsne-main --debug-rscript -b $container_reference_dir/gene.excludelist.tsv -g $container_reference_dir/gencode.v31.annotation.gtf.gz -c $container_data_dir/covariates.txt -o $container_output_dir/${output_name} ${in_arg} ${infile_arg} $container_data_dir/reference_counts/*.txt --save-data ${tissue_arg} ${gene_list_arg}\"" + echo "docker run -v $local_data_dir:$container_data_dir -v $local_reference_dir:$container_reference_dir -v $local_output_dir:$container_output_dir ghcr.io/stjudecloud/expression-classification:v0.10.2 bash -c \"cd $container_output_dir && itsne-main --debug-rscript -b $container_reference_dir/gene.excludelist.tsv -g $container_reference_dir/gencode.v31.annotation.gtf.gz -c $container_data_dir/covariates.txt -o $container_output_dir/${output_name} ${in_arg} ${infile_arg} $container_data_dir/reference_counts/*.txt --save-data ${tissue_arg} ${gene_list_arg}\"" - docker run -v $local_data_dir:$container_data_dir -v $local_reference_dir:$container_reference_dir -v $local_output_dir:$container_output_dir ghcr.io/stjudecloud/expression-classification:EXPRESSION_VERSION bash -c "cd $container_output_dir && itsne-main --debug-rscript -b $container_reference_dir/gene.excludelist.tsv -g $container_reference_dir/gencode.v31.annotation.gtf.gz -c $container_data_dir/covariates.txt -o $container_output_dir/${output_name} ${in_arg} ${infile_arg} $container_data_dir/reference_counts/*.txt --save-data ${tissue_arg} ${gene_list_arg}" + docker run -v $local_data_dir:$container_data_dir -v $local_reference_dir:$container_reference_dir -v $local_output_dir:$container_output_dir ghcr.io/stjudecloud/expression-classification:v0.10.2 bash -c "cd $container_output_dir && itsne-main --debug-rscript -b $container_reference_dir/gene.excludelist.tsv -g $container_reference_dir/gencode.v31.annotation.gtf.gz -c $container_data_dir/covariates.txt -o $container_output_dir/${output_name} ${in_arg} ${infile_arg} $container_data_dir/reference_counts/*.txt --save-data ${tissue_arg} ${gene_list_arg}" cp combined_metadata.json $local_output_dir cp /stjude/metadata/Subtype_Groupings_for_tSNE.csv $local_output_dir - docker run -v $local_output_dir:$container_output_dir -v /stjude/bin:/stjude/bin ghcr.io/stjudecloud/expression-classification:EXPRESSION_VERSION bash -c "cd $container_output_dir && python /stjude/bin/generate_plot.py --tsne-file $container_output_dir/tsne.txt --metadata-file $container_output_dir/combined_metadata.json --subtype-file $container_output_dir/Subtype_Groupings_for_tSNE.csv $tissue_arg" + docker run -v $local_output_dir:$container_output_dir -v /stjude/bin:/stjude/bin ghcr.io/stjudecloud/expression-classification:v0.10.2 bash -c "cd $container_output_dir && python /stjude/bin/generate_plot.py --tsne-file $container_output_dir/tsne.txt --metadata-file $container_output_dir/combined_metadata.json --subtype-file $container_output_dir/Subtype_Groupings_for_tSNE.csv $tissue_arg" if [ ${#input_counts[@]} -gt 0 ] then - docker run -v $local_output_dir:$container_output_dir -v /stjude/bin:/stjude/bin ghcr.io/stjudecloud/expression-classification:EXPRESSION_VERSION bash -c "cd $container_output_dir && python /stjude/bin/neighbors.py tsne.txt neighbors.tsv" + docker run -v $local_output_dir:$container_output_dir -v /stjude/bin:/stjude/bin ghcr.io/stjudecloud/expression-classification:v0.10.2 bash -c "cd $container_output_dir && python /stjude/bin/neighbors.py tsne.txt neighbors.tsv" mv $local_output_dir/neighbors.tsv $local_output_dir/${output_prefix}.neighbors.tsv neighbors_file=$(dx upload $local_output_dir/${output_prefix}.neighbors.tsv --brief) dx-jobutil-add-output neighbors "$neighbors_file" --class=file diff --git a/scripts/itsne-normalize-matrix.R b/scripts/itsne-normalize-matrix.R index b11ff1b..99d77b1 100644 --- a/scripts/itsne-normalize-matrix.R +++ b/scripts/itsne-normalize-matrix.R @@ -179,7 +179,7 @@ distMat <- dist(t(dataMatrixTop)) set.seed(opt$seed) cat("Running Rtsne\n", file = stderr()) -tsne_out <- Rtsne(distMat, dims = 2, perplexity = opt$`tsne-perplexity`, +tsne_out <- Rtsne(distMat, dims = 3, perplexity = opt$`tsne-perplexity`, theta = opt$`tsne-theta`, max_iter = opt$`tsne-max-iterations`, check_duplicates = F, num_threads = 0 ) cat("Saving Rtsne output\n", file = stderr()) @@ -188,9 +188,10 @@ write.table(data.frame(tsne_out$Y), file="tsne_output.txt", sep="\t",quote=FALSE cat("Plotting...\n", file = stderr()) # Create plot data objects toPlot <- data.frame(tsne_out$Y) -colnames(toPlot) <- c("t1", "t2") +colnames(toPlot) <- c("t1", "t2", "t3") unknownX<-toPlot[nrow(toPlot),1] unknownY<-toPlot[nrow(toPlot),2] +unknownZ<-toPlot[nrow(toPlot),3] toPlot$classes <- diagnosis toPlot$samples <- samples toPlot$color <- colors