Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ Currently, the pipeline supports four submission modes, each routed to a dedicat
- `metagenomic_assemblies` for assembly submission with `ASSEMBLYSUBMIT` workflow
- `reads` for raw sequencing reads submission with `READSUBMIT` workflow

![seqsubmit workflow diagram](assets/seqsubmit_schema.png)
![seqsubmit workflow diagram](docs/images/seqsubmit_schema.png)

## Requirements

Expand Down
2 changes: 1 addition & 1 deletion assets/multiqc_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ sp:
genome_metadata:
fn_re: ".*genomes_metadata.tsv$"
sample_registration:
fn_re: '.*registered.*\.tsv$'
fn_re: '.*registered\.*\.tsv$'
submission_results_genomes:
fn_re: 'genomes_accessions\.tsv$'
submission_results_assemblies:
Expand Down
24 changes: 15 additions & 9 deletions assets/schema_input_genome.json
Original file line number Diff line number Diff line change
Expand Up @@ -75,43 +75,49 @@
},
"completeness": {
"type": "number",
"description": "MAG/bin completeness value"
"description": "MAG/bin completeness score: the ratio of observed single-copy marker genes to total single-copy marker genes in chosen marker gene set (%). ENA docs: https://ena-docs.readthedocs.io/en/latest/faq/metagenomes.html#how-is-the-quality-of-a-metagenomic-assembly-defined",
"exclusiveMinimum": 0,
"exclusiveMaximum": 100
},
"contamination": {
"type": "number",
"description": "MAG/bin contamination value"
"description": "MAG/bin contamination score: the ratio of observed single-copy marker genes in ≥2 copies to total single-copy marker genes in chosen marker gene set (%). ENA docs: https://ena-docs.readthedocs.io/en/latest/faq/metagenomes.html#how-is-the-quality-of-a-metagenomic-assembly-defined",
"exclusiveMinimum": 0,
"exclusiveMaximum": 100
},
"genome_coverage": {
"type": "number",
"description": "MAG/bin coverage value"
},
"metagenome": {
"type": "string",
"description": "ENA accepted metagenome name"
"description": "ENA accepted metagenome name. It needs to be listed in the taxonomy tree https://www.ebi.ac.uk/ena/browser/view/408169?show=tax-tree (you might need to press \"Tax tree - Show\" in the right most section of the page). Full list can also be found https://github.com/EBI-Metagenomics/genome_uploader/blob/main/genomeuploader/constants.py#L22"
},
"co-assembly": {
"type": "string",
"description": "Yes if data was co-assembled, No otherwise"
"description": "'Yes' if data was co-assembled, 'No' otherwise. Options: 'Yes' or 'No'",
"enum": ["Yes", "No"]
},
"broad_environment": {
"type": "string",
"description": "broad ecological context of a sample"
"description": "Broad ecological context of the sample, for example 'marine biome', 'desert biome'. It is recommended to use subclasses of EnvO 'biome' class (http://purl.obolibrary.org/obo/ENVO_00000428). Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS"
},
"local_environment": {
"type": "string",
"description": "local ecological context"
"description": "Local environmental context of the sample, for example 'tropical dry broadleaf forest biome', 'marine abyssal zone biome'. It is recommended to use EnvO terms which are of smaller spatial grain than your entry for \"broad-scale environmental context\". Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS"
},
"environmental_medium": {
"type": "string",
"description": "material displaced by the sample"
"description": "Material displaced by the sample, or the material in which the sample was embedded before sampling, for example 'mucus', 'lake water'. It is recommended to use subclasses of EnvO 'environmental material' class (http://purl.obolibrary.org/obo/ENVO_00010483). Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS"
},
"RNA_presence": {
"type": "string",
"description": "Yes/No if at least 18 tRNAs and 5S, 16S, 23S rRNA genes detected"
"description": "Presence or absence of the 23S, 16S, and 5S rRNA genes and at least 18 tRNAs. This is used for MISAG/MIMAG assembly quality classification. Options: 'Yes' or 'No'.",
"enum": ["Yes", "No"]
},
"NCBI_lineage": {
"type": "string",
"description": "full NCBI lineage - format: x;y;z"
"description": "full NCBI lineage - format: x;y;z. For example, the lineage for E. coli can be: 'Bacteria;Pseudomonadati;Pseudomonadota;Gammaproteobacteria;Enterobacterales;Enterobacteriaceae;Escherichia' or '2;1224;1236;91347;543;561;562'"
}
},
"required": [
Expand Down
2 changes: 1 addition & 1 deletion conf/test_assembly_complete_metadata.config
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ process {

params {
config_profile_name = 'Test --mode metagenomic_assemblies complete_metadata profile'
config_profile_description = 'Single-case assembly test with complete metadata values provided'
config_profile_description = 'Single-case assembly test with complete metadata in samplesheet. No statistics calculations will be performed; the pipeline will only upload the data. Study for submission was already registered and provided via --submission_study.'

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/assembly_complete_metadata.csv'
Expand Down
2 changes: 1 addition & 1 deletion conf/test_mag_complete_metadata.conf
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ process {

params {
config_profile_name = 'Test --mode mags complete_metadata profile'
config_profile_description = 'Single-case MAG test with complete metadata values provided'
config_profile_description = 'Single-case MAG test with complete metadata in samplesheet. No statistics or taxonomy calculations will be performed; the pipeline will only upload the data. Study for submission was already registered and provided via --submission_study.'

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_complete_metadata.csv'
Expand Down
1 change: 1 addition & 0 deletions conf/test_mag_no_quality.conf
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ params {
cat_db = null
// CheckM2 doesn't allow usage of small test database, so real database will be downloaded during the test run,
// which is not ideal but necessary to test the pipeline's behaviour with missing quality values
// Track upstream issue here: https://github.com/chklovski/CheckM2/issues/140
checkm2_db = null

}
File renamed without changes
2 changes: 1 addition & 1 deletion docs/methods.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ Genome coverage values from `coverm genome` output TSV are parsed and merged int

### Taxonomy assignment

If `NCBI_lineage` is already present in the input samlesheet, the value is retained. If it is missing, the workflow runs taxonomy classification using the `CAT_pack` tool.
If `NCBI_lineage` is already present in the input samplesheet, the value is retained. If it is missing, the workflow runs taxonomy classification using the `CAT_pack` tool.

Before classification, input FASTA files are normalized to a `.fasta` suffix by `RENAME_FASTA_FOR_CATPACK`. Classification is then run in bin mode (`CAT_pack bins`, followed by `CAT_pack add_names`).

Expand Down
10 changes: 5 additions & 5 deletions docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ mag_001,data/mag_001.fasta.gz,SRR24458089,,,SPAdes 3.15.5,MetaBAT2 2.15,default,

| Column | Description |
| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `sample` | A unique identifier for this data entry. Must be globally unique within the input dataset (used as `meta.id` throughout the pipeline). |
| `sample` | A unique identifier for this data entry. Must be globally unique within the input dataset. |
| `fasta` | Path to MAG/bin contigs in FASTA format compressed with `gzip`. All names of the FASTA files must be unique to prevent pipeline errors. |
| `accession` | ENA accession of the run or metagenomic assembly used to generate the MAG/bin. |
| `fastq_1` | Path to the read file in FASTQ format used to generate the source metagenomic assembly. Required if `genome_coverage` is not provided. |
Expand Down Expand Up @@ -95,7 +95,7 @@ assembly_002,data/assembly_002.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9

| Column | Description |
| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
| `sample` | A unique identifier for this data entry. Must be globally unique within the input dataset (used as `meta.id` throughout the pipeline). |
| `sample` | A unique identifier for this data entry. Must be globally unique within the input dataset. |
| `fasta` | Path to assembly contigs in FASTA format compressed with `gzip`. |
| `fastq_1` | Path to the read file in FASTQ format used to generate the metagenomic assembly. Required if `coverage` is not provided. |
| `fastq_2` | Path to the second read file in FASTQ format for paired-end data used to generate the source metagenomic assembly. Leave empty for single-end reads. |
Expand Down Expand Up @@ -246,7 +246,7 @@ Key parameters:
| `--webincli_mode` | Webin-CLI mode for ENA interaction: `submit` uploads data, `validate` performs validation only. |
| `--upload_tpa` | Mark assemblies as third party assemblies when required. |

Test example for `mags` run with docker:
Example `mags` mode upload to TEST server, run command with docker:

```bash
nextflow run nf-core/seqsubmit \
Expand All @@ -260,7 +260,7 @@ nextflow run nf-core/seqsubmit \
--outdir results/validate_mags
```

Test example for `metagenomic_assemblies` run with docker:
Example `metagenomic_assemblies` mode upload to TEST server, run command with docker:

```bash
nextflow run nf-core/seqsubmit \
Expand All @@ -274,7 +274,7 @@ nextflow run nf-core/seqsubmit \
--outdir results/validate_assemblies
```

Test example for `reads` run with docker:
Example `reads` mode upload to TEST server, run command with docker:

```bash
nextflow run nf-core/seqsubmit \
Expand Down
8 changes: 6 additions & 2 deletions modules/local/count_rna/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,19 @@ process COUNT_RNA {
tuple val(meta), path("*rna_decision.tsv"), emit: rna_decision
tuple val("${task.process}"), val('python'), eval('python --version 2>&1 | sed "s/Python //g"'), topic: versions

when:
task.ext.when == null || task.ext.when

script:
def prefix = task.ext.prefix ?: "${meta.id}"
"""
count_rna.py \\
--trna ${trnas_stats} \\
--rrna ${rrna_gff} \\
--name ${meta.id} \\
--name ${prefix} \\
--trna-limit ${min_trna_count} \\
--rrna-limit ${min_rrna_percentage} \\
--output ${meta.id}_rna_decision.tsv
--output ${prefix}_rna_decision.tsv
"""

stub:
Expand Down
4 changes: 0 additions & 4 deletions modules/local/count_rna/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,6 @@ output:
Tab-separated file containing genome ID and decision (Yes/No) on whether
sufficient RNA genes are present
pattern: "*_rna_decision.tsv"
- versions:
type: file
description: File containing software versions
pattern: "versions.yml"

authors:
- "@KateSakharova"
Expand Down
5 changes: 0 additions & 5 deletions modules/local/create_assembly_metadata_csv/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,6 @@ output:
type: file
description: CSV file containing assembly metadata to be used as input for the assembly_uploader
pattern: "*_assembly_metadata.csv"
- versions:
- versions.yml:
type: file
description: File containing software versions
pattern: "versions.yml"

authors:
- "@ochkalova"
Expand Down
5 changes: 0 additions & 5 deletions modules/local/create_genome_metadata_tsv/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,6 @@ output:
type: file
description: Tab-separated TSV file with genome metadata
pattern: "*.tsv"
- versions:
- versions.yml:
type: file
description: File containing software versions
pattern: "versions.yml"

authors:
- "@ochkalova"
Expand Down
12 changes: 12 additions & 0 deletions modules/local/ena_webin_cli_wrapper/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ process ENA_WEBIN_CLI_WRAPPER {
tuple val("${task.process}"), val('ena-webin-cli'), eval('ena-webin-cli -version'), topic: versions
tuple val("${task.process}"), val('mgnify-pipelines-toolkit'), eval('python -c "import importlib.metadata; print(importlib.metadata.version(\'mgnify-pipelines-toolkit\'))"'), topic: versions

when:
task.ext.when == null || task.ext.when

script:
def args = task.ext.args ?: ""
def prefix = task.ext.prefix ?: "${meta.id}"
Expand All @@ -40,4 +43,13 @@ process ENA_WEBIN_CLI_WRAPPER {
${test_flag} \\
${args}
"""

stub:
def prefix = task.ext.prefix ?: "${meta.id}"
if (webincli_mode == "submit") {
"""
touch ${prefix}_accessions.tsv
"""
}
// there is no file for mode=validate
}
5 changes: 0 additions & 5 deletions modules/local/ena_webin_cli_wrapper/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,6 @@ output:
TSV file containing the accession assigned by ENA for the submitted item.
File has two columns: "alias" and "accession".
pattern: "*_accessions.tsv"
- versions:
- "versions.yml":
type: file
description: File containing software versions.
pattern: "versions.yml"

authors:
- "@KateSakharova"
Expand Down
12 changes: 1 addition & 11 deletions modules/local/generate_assembly_manifest/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ process GENERATE_ASSEMBLY_MANIFEST {

output:
tuple val(meta), path("${assembly_study}_upload/*.manifest") , emit: manifest
path "versions.yml", emit: versions
tuple val("${task.process}"), val('assembly_uploader'), eval("assembly_manifest --version | sed 's/assembly_uploader //'"), topic: versions

when:
task.ext.when == null || task.ext.when
Expand All @@ -37,11 +37,6 @@ process GENERATE_ASSEMBLY_MANIFEST {
${test_flag} \\
${is_private_flag} \\
${args}

cat <<-END_VERSIONS > versions.yml
"${task.process}":
assembly_uploader: \$(assembly_manifest --version | sed 's/assembly_uploader //')
END_VERSIONS
"""

stub:
Expand All @@ -50,10 +45,5 @@ process GENERATE_ASSEMBLY_MANIFEST {
"""
mkdir ${assembly_study}_upload/
touch ${assembly_study}_upload/test.manifest

cat <<-END_VERSIONS > versions.yml
"${task.process}":
assembly_uploader: \$(assembly_manifest --version | sed 's/assembly_uploader //')
END_VERSIONS
"""
}
4 changes: 0 additions & 4 deletions modules/local/generate_assembly_manifest/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,6 @@ output:
- manifest:
type: file
description: Manifest file required for submission with webin-cli.
- versions:
type: file
description: File containing software versions.
pattern: "versions.yml"

authors:
- "@fxquah"
2 changes: 1 addition & 1 deletion modules/local/genome_upload/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ process GENOME_UPLOAD {
path "results/{MAG,bin}_upload/ENA_backup.json" , emit: ena_upload_backup_json
path "results/{MAG,bin}_upload/genome_samples.xml" , emit: upload_genome_samples
path "results/{MAG,bin}_upload/registered_{MAGs,bins}*.tsv", emit: upload_registered_mags
tuple val("${task.process}"), val('genome_uploader'), eval("genome_upload --version 2>&1 | sed 's/genome_uploader //g'"), topic: versions, emit: versions_genome_uploader
tuple val("${task.process}"), val('genome_uploader'), eval("genome_upload --version 2>&1 | sed 's/genome_uploader //g'"), topic: versions

when:
task.ext.when == null || task.ext.when
Expand Down
4 changes: 0 additions & 4 deletions modules/local/genome_upload/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,6 @@ output:
type: file
description: XML file required for ENA submission
pattern: "*submission.xml"
- versions:
type: file
description: File containing software versions
pattern: "versions.yml"

authors:
- "@KateSakharova"
Expand Down
5 changes: 0 additions & 5 deletions modules/local/registerstudy/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,6 @@ output:
JSON file containing the submission results with keys:
submitted (newly created accessions) and failed.
pattern: "*_accessions.json"
- versions:
- "versions.yml":
type: file
description: File containing software versions
pattern: "versions.yml"

authors:
- "@timrozday"
Expand Down
42 changes: 25 additions & 17 deletions modules/local/rename_fasta_for_catpack/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,32 @@ process RENAME_FASTA_FOR_CATPACK {
tuple val(meta), path("output/*.fasta"), emit: renamed_fasta
tuple val("${task.process}"), val('bash'), eval('bash --version | head -n1 | sed "s/.*version //; s/ .*//"'), topic: versions

when:
task.ext.when == null || task.ext.when

script:
def is_compressed = fasta.name.endsWith('.gz')
extension = '.fasta'
def base = fasta.name
.replaceAll(/\.gz$/, '')
.replaceAll(/\.(fa|fasta|fna)$/, '')
def output_name = base + extension
def is_compressed = fasta.name.endsWith('.gz')
extension = '.fasta'
def base = fasta.name
.replaceAll(/\.gz$/, '')
.replaceAll(/\.(fa|fasta|fna)$/, '')
def output_name = base + extension

if (is_compressed) {
"""
mkdir -p output
gunzip -c ${fasta} > output/${output_name}
"""
} else {
"""
mkdir -p output
ln -s ../${fasta} output/${output_name}
"""
}
if (is_compressed) {
"""
mkdir -p output
gunzip -c ${fasta} > output/${output_name}
"""
} else {
"""
mkdir -p output
ln -s ../${fasta} output/${output_name}
"""
}

stub:
"""
mkdir -p output
touch output/${meta.id}.fasta
"""
}
2 changes: 1 addition & 1 deletion nextflow.config
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ profiles {
test { includeConfig 'conf/test.config' }
test_full { includeConfig 'conf/test_full.config' }
test_mag_complete_metadata { includeConfig 'conf/test_mag_complete_metadata.conf' }
test_mag_no_study_complete_metadata { includeConfig 'conf/test_mag_no_study_complete_metadata.config' }
test_mag_no_study_complete_metadata { includeConfig 'conf/test_mag_no_study_complete_metadata.config' }
test_mag_no_coverage_single_reads { includeConfig 'conf/test_mag_no_coverage_single_reads.conf' }
test_mag_no_coverage_paired_reads { includeConfig 'conf/test_mag_no_coverage_paired_reads.conf' }
test_mag_no_quality { includeConfig 'conf/test_mag_no_quality.conf' }
Expand Down
Loading
Loading