Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ default_language_version:
repos:
# Generic 'file quality' testing
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
rev: v6.0.0
hooks:
# Machine-friendliness
- id: trailing-whitespace
Expand All @@ -24,21 +24,21 @@ repos:

# Python file formatting
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.9.6
rev: v0.15.9
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- id: ruff-format

# Snakemake file formatting
- repo: https://github.com/snakemake/snakefmt
rev: v0.10.2
rev: v1.0.0
hooks:
- id: snakefmt

# Spelling
- repo: https://github.com/codespell-project/codespell
rev: v2.4.1
rev: v2.4.2
hooks:
- id: codespell
files: .*\.(py|smk|md)$|^Snakefile$
Expand Down
16 changes: 8 additions & 8 deletions tests/integration/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,36 @@ configfile: workflow.source_path("./test_config.yaml")


rule download_netherlands_shapes:
message:
"Download and unzip the Netherlands shapes."
output:
"results/integration_test/resources/user/shapes/NLD.parquet",
log:
"results/integration_test/logs/download_netherlands_shapes.log",
conda:
"../../workflow/envs/shell.yaml"
message:
"Download and unzip the Netherlands shapes."
shell:
"""
curl -sSLo {output:q} "https://zenodo.org/records/16684683/files/NLD.parquet"
"""


rule download_netherlands_protected_areas:
message:
"Download a dummy drop-in dataset for Netherlands protected areas (not based on WDPA)."
output:
"results/integration_test/resources/user/wdpa.gdb.zip",
log:
"results/integration_test/logs/download_netherlands_protected_areas.log",
conda:
"../../workflow/envs/shell.yaml"
message:
"Download a dummy drop-in dataset for Netherlands protected areas (not based on WDPA)."
shell:
"""
curl -sSLo {output:q} "https://zenodo.org/records/16684513/files/wdpa-nl-dropin.gdb.zip"
"""


rule unzip_netherlands_protected_areas:
message:
"Download protected areas data."
input:
script=workflow.source_path("../../workflow/scripts/unzip_like.py"),
zipfile=rules.download_netherlands_protected_areas.output,
Expand All @@ -44,6 +42,8 @@ rule unzip_netherlands_protected_areas:
"results/integration_test/logs/unzip_netherlands_protected_areas.log",
conda:
"../../workflow/envs/shell.yaml"
message:
"Download protected areas data."
shell:
"""
python {input.script:q} {input.zipfile:q} -t "results/integration_test/resources/user/" 2> {log:q}
Expand All @@ -68,10 +68,10 @@ use rule * from module_area_potentials as module_area_potentials_*

# Request something from the module
rule all:
message:
"Run the module for the Netherlands shapes."
default_target: True
input:
"results/integration_test/resources/user/shapes/NLD.parquet",
"results/integration_test/resources/user/wdpa.gdb",
"results/integration_test/results/NLD/area_potential_report.html",
message:
"Run the module for the Netherlands shapes."
4 changes: 2 additions & 2 deletions workflow/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,14 @@ include: "rules/process.smk"


rule all:
message:
"ERROR: Invalid `rule all:` call"
default_target: True
output:
"INVALID",
log:
stderr="logs/all.stderr",
conda:
"envs/shell.yaml"
message:
"ERROR: Invalid `rule all:` call"
shell:
'echo "This workflow must be called as a snakemake module." > {log.stderr}'
84 changes: 42 additions & 42 deletions workflow/rules/automatic.smk
Original file line number Diff line number Diff line change
Expand Up @@ -7,30 +7,30 @@ if config.get("tiny_files", False):
##

rule clip_slope:
message:
"Download slope data covering the bounds of the input shapefile."
params:
cog_url=internal["resources"]["automatic"]["slope"],
input:
vector="resources/user/shapes/{shape}.parquet",
output:
path="resources/automatic/cutout/{shape}/slope.tif",
log:
"logs/{shape}/clip_slope.log",
params:
cog_url=internal["resources"]["automatic"]["slope"],
message:
"Download slope data covering the bounds of the input shapefile."
wrapper:
"v7.2.0/geo/rasterio/clip-geotiff"

rule clip_bathymetry:
message:
"Download bathymetry data covering the bounds of the input shapefile."
params:
cog_url=internal["resources"]["automatic"]["bathymetry"],
input:
vector="resources/user/shapes/{shape}.parquet",
output:
path="resources/automatic/cutout/{shape}/bathymetry.tif",
log:
"logs/{shape}/clip_bathymetry.log",
params:
cog_url=internal["resources"]["automatic"]["bathymetry"],
message:
"Download bathymetry data covering the bounds of the input shapefile."
wrapper:
"v7.2.0/geo/rasterio/clip-geotiff"

Expand All @@ -41,40 +41,38 @@ else:
##

rule download_slope:
message:
"Download global slope data."
params:
url=internal["resources"]["automatic"]["slope"],
output:
path="resources/automatic/global/slope.tif",
log:
"logs/download_slope.log",
conda:
"../envs/shell.yaml"
params:
url=internal["resources"]["automatic"]["slope"],
message:
"Download global slope data."
shell:
"""
curl -sSLo {output:q} {params.url:q}
"""

rule download_bathymetry:
message:
"Download global bathymetry data."
params:
url=internal["resources"]["automatic"]["bathymetry"],
output:
path="resources/automatic/global/bathymetry.tif",
log:
"logs/download_bathymetry.log",
conda:
"../envs/shell.yaml"
params:
url=internal["resources"]["automatic"]["bathymetry"],
message:
"Download global bathymetry data."
shell:
"""
curl -sSLo {output:q} {params.url:q}
"""

rule clip_slope:
message:
"Cut slope data to the bounds of the input shapefile."
input:
script=workflow.source_path("../scripts/clip_raster.py"),
shapes="resources/user/shapes/{shape}.parquet",
Expand All @@ -85,14 +83,14 @@ else:
"logs/{shape}/clip_slope.log",
conda:
"../envs/default.yaml"
message:
"Cut slope data to the bounds of the input shapefile."
shell:
"""
python {input.script:q} {input.slope:q} {input.shapes:q} {output:q} 2> {log:q}
"""

rule clip_bathymetry:
message:
"Cut bathymetry data to the bounds of the input shapefile."
input:
script=workflow.source_path("../scripts/clip_raster.py"),
shapes="resources/user/shapes/{shape}.parquet",
Expand All @@ -103,6 +101,8 @@ else:
"logs/{shape}/clip_bathymetry.log",
conda:
"../envs/default.yaml"
message:
"Cut bathymetry data to the bounds of the input shapefile."
shell:
"""
python {input.script:q} {input.bathymetry:q} {input.shapes:q} {output:q} 2> {log:q}
Expand All @@ -115,27 +115,23 @@ else:


rule download_globcover:
message:
"Download the GlobCover land cover data (~380 MB)."
params:
url=internal["resources"]["automatic"]["globcover"],
output:
"resources/automatic/global/globcover.zip",
log:
"logs/download_globcover.log",
conda:
"../envs/shell.yaml"
params:
url=internal["resources"]["automatic"]["globcover"],
message:
"Download the GlobCover land cover data (~380 MB)."
shell:
"""
curl -sSLo {output:q} {params.url:q}
"""


rule unzip_globcover:
message:
"Unzip the relevant TIF files from the GlobCover zip file."
params:
target_file=internal["resources"]["automatic"]["globcover_landcover_tif"],
input:
script=workflow.source_path("../scripts/unzip_like.py"),
zipfile=rules.download_globcover.output,
Expand All @@ -145,15 +141,17 @@ rule unzip_globcover:
"logs/unzip_globcover.log",
conda:
"../envs/shell.yaml"
params:
target_file=internal["resources"]["automatic"]["globcover_landcover_tif"],
message:
"Unzip the relevant TIF files from the GlobCover zip file."
shell:
"""
python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} 2> {log:q}
"""


rule clip_landcover:
message:
"Cut land cover data to the bounds of the input shapefile."
input:
script=workflow.source_path("../scripts/clip_raster.py"),
shapes="resources/user/shapes/{shape}.parquet",
Expand All @@ -164,6 +162,8 @@ rule clip_landcover:
"logs/{shape}/clip_landcover.log",
conda:
"../envs/default.yaml"
message:
"Cut land cover data to the bounds of the input shapefile."
shell:
"""
python {input.script:q} {input.landcover:q} {input.shapes:q} {output:q} 2> {log:q}
Expand All @@ -176,27 +176,23 @@ rule clip_landcover:


rule download_ghsl:
message:
"Download the GHSL (Global Human Settlement Layer) built-up surface data."
params:
url=internal["resources"]["automatic"]["ghsl"],
output:
"resources/automatic/global/ghsl_built_s.zip",
log:
"logs/download_ghsl.log",
conda:
"../envs/shell.yaml"
params:
url=internal["resources"]["automatic"]["ghsl"],
message:
"Download the GHSL (Global Human Settlement Layer) built-up surface data."
shell:
"""
curl -sSLo {output:q} {params.url:q}
"""


rule unzip_ghsl:
message:
"Unzip the relevant TIF file from the GHSL data."
params:
target_file=internal["resources"]["automatic"]["ghsl_tif"],
input:
script=workflow.source_path("../scripts/unzip_like.py"),
zipfile=rules.download_ghsl.output,
Expand All @@ -206,15 +202,17 @@ rule unzip_ghsl:
"logs/unzip_ghsl.log",
conda:
"../envs/shell.yaml"
params:
target_file=internal["resources"]["automatic"]["ghsl_tif"],
message:
"Unzip the relevant TIF file from the GHSL data."
shell:
"""
python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} 2> {log:q}
"""


rule clip_settlement:
message:
"Cut settlement data to the bounds of the input shapefile."
input:
script=workflow.source_path("../scripts/clip_raster.py"),
shapes="resources/user/shapes/{shape}.parquet",
Expand All @@ -225,6 +223,8 @@ rule clip_settlement:
"logs/{shape}/clip_settlement.log",
conda:
"../envs/default.yaml"
message:
"Cut settlement data to the bounds of the input shapefile."
shell:
"""
python {input.script:q} {input.settlement:q} {input.shapes:q} {output:q} 2> {log:q}
Expand All @@ -237,8 +237,6 @@ rule clip_settlement:


rule rasterise_clip_wdpa:
message:
"Rasterise and cut WDPA data to the bounds of the input shapefile, using the landcover raster as reference for the rasterisation."
input:
script=workflow.source_path("../scripts/clip_and_rasterise_polys.py"),
shapes="resources/user/shapes/{shape}.parquet",
Expand All @@ -250,6 +248,8 @@ rule rasterise_clip_wdpa:
"logs/{shape}/clip_wdpa.log",
conda:
"../envs/default.yaml"
message:
"Rasterise and cut WDPA data to the bounds of the input shapefile, using the landcover raster as reference for the rasterisation."
shell:
"""
python {input.script:q} {input.shapes:q} {input.reference_raster:q} {input.protected_areas:q} {output:q} 2> {log:q}
Expand Down
Loading