Skip to content

Update tsne workflow #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jsunny23 wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Update tsne workflow #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 76 additions & 34 deletions dx_app/dxapp.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
{
"name": "stjude_tsne",
"title": "St. Jude Cloud Interactive t-SNE",
"name": "stjude_tsne_test",
"title": "St. Jude Cloud Interactive t-SNE (TEST)",
"summary": "Produce a t-SNE plot that highlights input sample(s) against a cohort of existing samples.",
"dxapi": "1.0.0",
"version": "EXPRESSION_SHORT_VERSION",
"developers": [ "user-athrashe" ],
"version": "0.8.2",
"developers": [
"user-athrashe"
],
"authorizedUsers": [
"user-athrashe",
"org-stjude_cloud_app_runners",
Expand All @@ -17,17 +19,21 @@
"class": "array:file",
"optional": true,
"help": "Provide a cohort of samples to use as reference data",
"patterns": [ "*.feature-counts.txt" ]
"patterns": [
"*.feature-counts.txt"
]
},
{
"name": "input_counts",
"label": "Input counts to compare",
"class": "array:file",
"optional": true,
"help": "Provide sample(s) to compare. File must have the following properties: sample_name, library_type, read_length, strandedness, and pairing.",
"patterns": [ "*.feature-counts.txt" ]
"patterns": [
"*.feature-counts.txt"
]
},
{
{
"name": "tumor_type",
"label": "Reference tumor type(s)",
"class": "string",
Expand All @@ -38,7 +44,7 @@
},
{
"name": "output_name",
"label": "Output file name",
"label": "Output file name",
"class": "string",
"default": "samples.html",
"help": "Specify a filename for the output HTML plot. Name should be a valid unix file name (e.g. no space or reserved characters).",
Expand All @@ -48,44 +54,62 @@
"name": "preservatives",
"label": "FFPE and/or Fresh/Frozen",
"class": "string",
"choices": ["FFPE", "Fresh/Frozen", "all"],
"choices": [
"FFPE",
"Fresh/Frozen",
"all"
],
"optional": false,
"help": "Specify preservative(s) to include in analysis.",
"default": "Fresh/Frozen",
"group": "Analysis Settings"
},
{
{
"name": "all_strandedness",
"label": "Strandedness",
"class": "string",
"choices": ["Stranded-Forward", "Stranded-Reverse", "Unstranded"],
"label": "Strandedness",
"class": "string",
"choices": [
"Stranded-Forward",
"Stranded-Reverse",
"Unstranded"
],
"optional": true,
"help": "Specify whether library was stranded. This option will apply to any input samples without a specification.",
"group": "Input sample parameters"
},
{
{
"name": "all_library_type",
"label": "Library type",
"class": "string",
"choices": ["PolyA", "Total"],
"label": "Library type",
"class": "string",
"choices": [
"PolyA",
"Total"
],
"optional": true,
"help": "Specify library type. This option will apply to any input samples without a specification.",
"group": "Input sample parameters"
},
{
{
"name": "all_read_length",
"label": "Read length",
"class": "string",
"choices": ["101", "126", "150"],
"label": "Read length",
"class": "string",
"choices": [
"101",
"126",
"150"
],
"optional": true,
"help": "Specify read length. This option will apply to any input samples without a specification.",
"group": "Input sample parameters"
},
{
{
"name": "all_pairing",
"label": "Read pairing",
"class": "string",
"choices": ["Paired-end", "Single-end"],
"label": "Read pairing",
"class": "string",
"choices": [
"Paired-end",
"Single-end"
],
"optional": true,
"help": "Specify read pairing. This option will apply to any input samples without a specification.",
"group": "Input sample parameters"
Expand All @@ -95,7 +119,13 @@
"label": "Return intermediate file",
"class": "string",
"optional": true,
"choices": ["original_matrix", "variance_stabilized", "batch_corrected", "tsne_output", "matrix"],
"choices": [
"original_matrix",
"variance_stabilized",
"batch_corrected",
"tsne_output",
"matrix"
],
"help": "If desired, return data from an intermediate point in the analysis. 'original_matrix' is the matrix of counts. 'variance_stabilized' is the matrix after running varianceStabilizingTransformation. 'batch_corrected' is the matrix after ComBat batch correction. 'tsne_output' is the matrix after Rtsne has been run. 'matrix' is the file input to the R script for plotting.",
"group": "Analysis Settings"
},
Expand All @@ -122,7 +152,6 @@
"help": "If true, DFCI PDX samples will be included in the t-SNE to facilitate model selection",
"group": "Analysis Settings"
}

],
"outputSpec": [
{
Expand Down Expand Up @@ -181,11 +210,24 @@
"distribution": "Ubuntu",
"file": "src/t-SNE.sh",
"execDepends": [
{ "name": "parallel"},
{ "name": "aria2"},
{ "name": "csvkit", "package_manager": "pip"},
{ "name": "pandas", "package_manager": "pip"},
{ "name": "logzero", "package_manager": "pip"}
{
"name": "parallel"
},
{
"name": "aria2"
},
{
"name": "csvkit",
"package_manager": "pip"
},
{
"name": "pandas",
"package_manager": "pip"
},
{
"name": "logzero",
"package_manager": "pip"
}
]
},
"regionalOptions": {
Expand All @@ -199,8 +241,8 @@
},
"access": {
"network": [
"*"
"*"
],
"allProjects": "VIEW"
}
}
}
2 changes: 2 additions & 0 deletions dx_app/resources/stjude/bin/generate_plot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ def label_samples (row):
# Trim precision on x/y coordinates
matrix = matrix.round({'t1': 2, 't2': 2})
# Read the metadata file
print("Metadata file")
print(args.metadata_file)
metadata = pd.read_json(args.metadata_file)

# Convert metadata JSON column 'properties' into a table.
Expand Down
8 changes: 4 additions & 4 deletions dx_app/src/t-SNE.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,17 +494,17 @@ main() {
gene_list_arg="--gene-list ${container_reference_dir}/gene_list.txt"
fi

echo "docker run -v $local_data_dir:$container_data_dir -v $local_reference_dir:$container_reference_dir -v $local_output_dir:$container_output_dir ghcr.io/stjudecloud/expression-classification:EXPRESSION_VERSION bash -c \"cd $container_output_dir && itsne-main --debug-rscript -b $container_reference_dir/gene.excludelist.tsv -g $container_reference_dir/gencode.v31.annotation.gtf.gz -c $container_data_dir/covariates.txt -o $container_output_dir/${output_name} ${in_arg} ${infile_arg} $container_data_dir/reference_counts/*.txt --save-data ${tissue_arg} ${gene_list_arg}\""
echo "docker run -v $local_data_dir:$container_data_dir -v $local_reference_dir:$container_reference_dir -v $local_output_dir:$container_output_dir ghcr.io/stjudecloud/expression-classification:v0.10.2 bash -c \"cd $container_output_dir && itsne-main --debug-rscript -b $container_reference_dir/gene.excludelist.tsv -g $container_reference_dir/gencode.v31.annotation.gtf.gz -c $container_data_dir/covariates.txt -o $container_output_dir/${output_name} ${in_arg} ${infile_arg} $container_data_dir/reference_counts/*.txt --save-data ${tissue_arg} ${gene_list_arg}\""

docker run -v $local_data_dir:$container_data_dir -v $local_reference_dir:$container_reference_dir -v $local_output_dir:$container_output_dir ghcr.io/stjudecloud/expression-classification:EXPRESSION_VERSION bash -c "cd $container_output_dir && itsne-main --debug-rscript -b $container_reference_dir/gene.excludelist.tsv -g $container_reference_dir/gencode.v31.annotation.gtf.gz -c $container_data_dir/covariates.txt -o $container_output_dir/${output_name} ${in_arg} ${infile_arg} $container_data_dir/reference_counts/*.txt --save-data ${tissue_arg} ${gene_list_arg}"
docker run -v $local_data_dir:$container_data_dir -v $local_reference_dir:$container_reference_dir -v $local_output_dir:$container_output_dir ghcr.io/stjudecloud/expression-classification:v0.10.2 bash -c "cd $container_output_dir && itsne-main --debug-rscript -b $container_reference_dir/gene.excludelist.tsv -g $container_reference_dir/gencode.v31.annotation.gtf.gz -c $container_data_dir/covariates.txt -o $container_output_dir/${output_name} ${in_arg} ${infile_arg} $container_data_dir/reference_counts/*.txt --save-data ${tissue_arg} ${gene_list_arg}"

cp combined_metadata.json $local_output_dir
cp /stjude/metadata/Subtype_Groupings_for_tSNE.csv $local_output_dir
docker run -v $local_output_dir:$container_output_dir -v /stjude/bin:/stjude/bin ghcr.io/stjudecloud/expression-classification:EXPRESSION_VERSION bash -c "cd $container_output_dir && python /stjude/bin/generate_plot.py --tsne-file $container_output_dir/tsne.txt --metadata-file $container_output_dir/combined_metadata.json --subtype-file $container_output_dir/Subtype_Groupings_for_tSNE.csv $tissue_arg"
docker run -v $local_output_dir:$container_output_dir -v /stjude/bin:/stjude/bin ghcr.io/stjudecloud/expression-classification:v0.10.2 bash -c "cd $container_output_dir && python /stjude/bin/generate_plot.py --tsne-file $container_output_dir/tsne.txt --metadata-file $container_output_dir/combined_metadata.json --subtype-file $container_output_dir/Subtype_Groupings_for_tSNE.csv $tissue_arg"

if [ ${#input_counts[@]} -gt 0 ]
then
docker run -v $local_output_dir:$container_output_dir -v /stjude/bin:/stjude/bin ghcr.io/stjudecloud/expression-classification:EXPRESSION_VERSION bash -c "cd $container_output_dir && python /stjude/bin/neighbors.py tsne.txt neighbors.tsv"
docker run -v $local_output_dir:$container_output_dir -v /stjude/bin:/stjude/bin ghcr.io/stjudecloud/expression-classification:v0.10.2 bash -c "cd $container_output_dir && python /stjude/bin/neighbors.py tsne.txt neighbors.tsv"
mv $local_output_dir/neighbors.tsv $local_output_dir/${output_prefix}.neighbors.tsv
neighbors_file=$(dx upload $local_output_dir/${output_prefix}.neighbors.tsv --brief)
dx-jobutil-add-output neighbors "$neighbors_file" --class=file
Expand Down
5 changes: 3 additions & 2 deletions scripts/itsne-normalize-matrix.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ distMat <- dist(t(dataMatrixTop))

set.seed(opt$seed)
cat("Running Rtsne\n", file = stderr())
tsne_out <- Rtsne(distMat, dims = 2, perplexity = opt$`tsne-perplexity`,
tsne_out <- Rtsne(distMat, dims = 3, perplexity = opt$`tsne-perplexity`,
theta = opt$`tsne-theta`, max_iter = opt$`tsne-max-iterations`, check_duplicates = F,
num_threads = 0 )
cat("Saving Rtsne output\n", file = stderr())
Expand All @@ -188,9 +188,10 @@ write.table(data.frame(tsne_out$Y), file="tsne_output.txt", sep="\t",quote=FALSE
cat("Plotting...\n", file = stderr())
# Create plot data objects
toPlot <- data.frame(tsne_out$Y)
colnames(toPlot) <- c("t1", "t2")
colnames(toPlot) <- c("t1", "t2", "t3")
unknownX<-toPlot[nrow(toPlot),1]
unknownY<-toPlot[nrow(toPlot),2]
unknownZ<-toPlot[nrow(toPlot),3]
toPlot$classes <- diagnosis
toPlot$samples <- samples
toPlot$color <- colors
Expand Down
Loading