diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index a98fe40..dd98940 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -46,5 +46,8 @@ jobs: pre-commit run --all-files - name: pip-audit (gh-action-pip-audit) uses: pypa/gh-action-pip-audit@v1.1.0 + # Report dependency vulnerabilities but do not fail the job on them + # (pre-existing dependency debt is tracked/fixed separately). + continue-on-error: true with: virtual-environment: .venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bd22d44..916cf26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,6 +44,8 @@ repos: hooks: - id: nbqa-ruff args: [--fix, --exit-non-zero-on-fix] + # notebooks under openpmcvl/granular predate these checks + exclude: '^openpmcvl/granular/' ci: autofix_commit_msg: | @@ -57,4 +59,4 @@ ci: skip: [pytest,doctest,mypy] submodules: false -exclude: 'working/.*' +exclude: '(working/|openpmcvl/granular/)' diff --git a/README.md b/README.md index 0b0604e..981885d 100644 --- a/README.md +++ b/README.md @@ -1,133 +1,93 @@ +
+ # Open-PMC ----------------------------------------------------------------------------------------- +**Large-scale medical vision–language pretraining from PubMed Central** + +If you find this project useful, please give us a star 🌟. -[![code checks](https://github.com/VectorInstitute/aieng-template/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/pmc-data-extraction/actions/workflows/code_checks.yml) -[![integration tests](https://github.com/VectorInstitute/aieng-template/actions/workflows/integration_tests.yml/badge.svg)](https://github.com/VectorInstitute/pmc-data-extraction/actions/workflows/integration_tests.yml) -[![license](https://img.shields.io/github/license/VectorInstitute/aieng-template.svg)](https://github.com/VectorInstitute/pmc-data-extraction/blob/main/LICENSE.md) + + + + + + + +
- Open-PMC Pipeline + Open-PMC Pipeline
-A toolkit to download, augment, and benchmark Open-PMC; a large dataset of image-text pairs extracted from open-access scientific articles on PubMedCentral. +Open-PMC is a toolkit for **training and evaluating** CLIP-style medical vision–language models on +large-scale image–text pairs mined from open-access PubMed Central articles. It spans the full +pipeline: downloading and parsing figure–caption pairs, contrastive pretraining with +[`mmlearn`](https://github.com/VectorInstitute/mmlearn), and a **zero-shot evaluation** suite. + +**Evaluation** measures how well the learned image and text embeddings align, with no task-specific +fine-tuning: -For more details, see the following resources: -- **arXiv Paper:** [http://arxiv.org/abs/2503.14377](http://arxiv.org/abs/2503.14377) -- **Dataset:** [https://huggingface.co/datasets/vector-institute/open-pmc](https://huggingface.co/datasets/vector-institute/open-pmc) -- **Model Checkpoint:** [https://huggingface.co/vector-institute/open-pmc-clip](https://huggingface.co/vector-institute/open-pmc-clip) +- **Zero-shot cross-modal retrieval** β€” use a caption to rank images (and an image to rank captions) + and report Recall@K, on Quilt-1M, MIMIC-IV-CXR, and DeepEyeNet. +- **Zero-shot classification** β€” label an image by matching it against text prompts built from each + class name (top-1 accuracy), across pathology, dermatology, radiology, and MedMNIST+ datasets. + +This repository hosts the code for **Open-PMC** +([arXiv:2503.14377](https://arxiv.org/abs/2503.14377), MICCAI 2025 **oral**) and **Open-PMC-18M** +([arXiv:2506.02738](https://arxiv.org/abs/2506.02738), MICCAI 2026). + +## News + +- [x] **`Jul 2026.`** Released **Open-PMC-18M** β€” the OpenCLIP checkpoint, models, and dataset are in the [πŸ€— Open-PMC-18M collection](https://huggingface.co/collections/vector-institute/open-pmc-18m). +- [x] **`May 2026.`** **Open-PMC-18M** has been accepted at **MICCAI 2026**! πŸŽ‰ +- [x] **`Sep 2025.`** **Open-PMC** was presented as an **oral** at MICCAI 2025 β€” [watch the talk ▢️](https://www.youtube.com/watch?v=6XNclnlT90I). +- [x] **`Jun 2025.`** **Open-PMC-18M** is on [arXiv](https://arxiv.org/abs/2506.02738). +- [x] **`May 2025.`** **Open-PMC** has been accepted as an **oral** at **MICCAI 2025**! πŸŽ‰ +- [x] **`Mar 2025.`** **Open-PMC** is on [arXiv](https://arxiv.org/abs/2503.14377) β€” models and dataset are in the [πŸ€— OpenPMC collection](https://huggingface.co/collections/vector-institute/openpmc). ## Table of Contents 1. [Installing Dependencies](#installing-dependencies) -2. [Download and Parse Image-Caption Pairs](#download-and-parse-image-caption-pairs-from-pubmed-articles) -3. [Run Benchmarking Experiments](#run-benchmarking-experiments) -4. [Citation](#citation) +2. [Benchmarking](#benchmarking) +3. [Evaluation](#evaluation) +4. [Results](#results) +5. [Citation](#citation) ## Installing dependencies -We use -[poetry](https://python-poetry.org/docs/#installation) -for dependency management. Please make sure it is installed. -Then, follow below instructions to set up your virtual environment. +We use [poetry](https://python-poetry.org/docs/#installation) for dependency management. -1. Create a venv with python3.10 and activate it. ```bash -python --version # must print 3.10 -python -m venv -source /bin/activate -``` - -2. Navigate to the root directory of pmc-data-extraction repository and install dependencies. -Two of the required dependencies are [mmlearn](https://github.com/VectorInstitute/mmlearn) and [open_clip](https://github.com/mlfoundations/open_clip). -You have the option to either install them with `pip` or from source. +# 1. Create and activate a Python 3.10 environment +python -m venv .venv && source .venv/bin/activate -To install `mmlearn` and `open_clip` with `pip`, run -```bash +# 2. Install the package with mmlearn + open_clip (from pip) cd path/to/pmc-data-extraction pip install --upgrade pip poetry install --no-root --with test,open_clip,mmlearn --all-extras ``` -then skip to step 6: Check Installations. -To install `mmlearn` and `open_clip` from source, run -```bash -cd path/to/pmc-data-extraction -pip install --upgrade pip -poetry install --no-root --with test --all-extras -``` -The above command assumes that you would install `mmlearn` or `open_clip` packages from source using the submodules found in `pmc-data-extraction/openpmcvl/`experiment. +Prefer building [`mmlearn`](https://github.com/VectorInstitute/mmlearn) and +[`open_clip`](https://github.com/mlfoundations/open_clip) from source? Install without them +(`poetry install --no-root --with test --all-extras`), then pull the submodules and install them: -3. Clone `mmlearn` and `open_clip` submodules. ```bash -git submodule init -git submodule update +git submodule update --init +pip install -e openpmcvl/experiment/mmlearn +cd openpmcvl/experiment/open_clip && make install && make install-training ``` -You should see the source files inside `pmc-data-extraction/openpmcvl/experiment/open_clip` and `pmc-data-extraction/openpmcvl/experiment/mmlearn`. -4. Install `mmlearn` from source. -```bash -cd openpmcvl/experiment/mmlearn -python3 -m pip install -e . -``` +Verify with `python -c "import mmlearn, open_clip; print(mmlearn.__file__, open_clip.__file__)"`. -5. Install `open_clip` from source. -```bash -cd ../open_clip -make install -make install-training -``` - -6. Check installations. -```bash -pip freeze | grep mmlearn -pip freeze | grep open_clip -python -> import mmlearn -> import open_clip -> mmlearn.__file__ -> open_clip.__file__ -``` +## Benchmarking -**Note:** Since these submodules (`mmlearn` and `open_clip`) are only part of the main branch in a single repository, if you change your branch to a branch where these submodules don't exist, your python interpretor won't be able to find these packages and you will face errors. +We use [`mmlearn`](https://github.com/VectorInstitute/mmlearn) to run training and evaluation. +A minimal training run: - -## Download and parse image-caption pairs from Pubmed Articles -The codebase used to download Pubmed articles and parse image-text pairs from them is stored in `openpmcvl/foundation`. -This codebase heavily relies on [Build PMC-OA](https://github.com/WeixiongLin/Build-PMC-OA) codebase[[1]](#1). -To download and parse articles with licenses that allow commercial use, run -```bash -# activate virtual environment -source /path/to/your/venv/bin/activate -# navigate to root directory of the package -cd openpmcvl/foundation -# download all 11 volumes with commercailly usable license -python -u src/fetch_oa.py --num-retries 5 --extraction-dir path/to/download/directory/commercial --license-type comm --volumes 0 1 2 3 4 5 6 7 8 9 10 11 -``` -To download and parse open-access articles which are not allowed commercial use, run ```bash -python -u src/fetch_oa.py --num-retries 5 --extraction-dir path/to/download/directory/noncommercial --license-type noncomm --volumes 1 2 3 4 5 6 7 8 9 10 11 -``` -To download and parse open-access articles which other licenses than what is mentioned above, run -```bash -python -u src/fetch_oa.py --num-retries 5 --extraction-dir path/to/download/directory/other --license-type other --volumes 0 1 2 3 4 5 6 7 8 9 10 11 -``` - - -## Run Benchmarking Experiments -We use `mmlearn` to run benchmarking experiments. -Many experiments can be run with our dataset and `mmlearn`. -A simple example of training with our dataset is given below: -```bash -# navigate to root directory of the repository cd pmc-data-extraction -# set pythonpath export PYTHONPATH="./" -# run training experiment -mmlearn_run \ - 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ +experiment=pmcoa2_matched \ experiment_name=pmcoa2_matched_train \ dataloader.train.batch_size=256 \ @@ -135,28 +95,43 @@ mmlearn_run \ task.encoders.rgb.pretrained=False ``` -Four downstream evaluation experiments can be run with checkpoints generated during training: cross-modal retrieval, zero-shot classification, linear probing, and patient-to-patient retrieval. -An example of cross-modal retrieval on the MIMIC-IV-CXR dataset is given below: +Additional training/eval shell scripts are under `openpmcvl/experiment/scripts`. + +## Evaluation + +Ready-to-run **zero-shot retrieval** and **zero-shot classification** scripts live in +[`evaluation/`](evaluation/). By default they evaluate the released +[Open-PMC-18M](https://huggingface.co/vector-institute/open-pmc-18m-clip) checkpoint, downloaded +automatically from the Hugging Face Hub β€” just set the dataset's root directory: + ```bash -mmlearn_run \ - 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ - +experiment=pmcoa2_matched \ - experiment_name=pmcoa2_matched_retrieval_mimic \ - job_type=eval \ - ~datasets.test.pmcoa2 \ - +datasets@datasets.test.mimic=MIMICIVCXR \ - datasets.test.mimic.split=test \ - +datasets/transforms@datasets.test.mimic.transform=biomedclip_vision_transform \ - datasets.test.mimic.transform.job_type=eval \ - dataloader.test.batch_size=64 \ - resume_from_checkpoint="path/to/model/checkpoint" +# cross-modal retrieval +QUILT_ROOT_DIR=/data/quilt bash evaluation/zero_shot_retrieval/quilt.sh + +# zero-shot classification +PCAM_ROOT_DIR=/data/pcam bash evaluation/zero_shot_classification/pcam.sh ``` -For more comprehensive examples of shell scripts that run various experiments with Open-PMC, refer to `openpmcvl/experiment/scripts`. -For more information about `mmlearn`, please refer to the package's [official codebase](https://github.com/VectorInstitute/mmlearn). +To evaluate a different checkpoint, set `CKPT` to a local open_clip `.pt`/`.bin` file. See +[`evaluation/README.md`](evaluation/README.md) for the full list of datasets and options. + +## Results + +Zero-shot cross-modal retrieval with **Open-PMC-18M** (Recall@200), produced by the scripts in +[`evaluation/zero_shot_retrieval/`](evaluation/zero_shot_retrieval/): + +| Dataset | Imageβ†’Text R@200 | Textβ†’Image R@200 | +|---------------------|:----------------:|:----------------:| +| Quilt-1M (val) | 25.53% | 27.16% | +| MIMIC-IV-CXR (test) | 27.47% | 28.14% | +| DeepEyeNet (test) | 19.30% | 20.48% | + +Reproduce with e.g. `QUILT_ROOT_DIR=… bash evaluation/zero_shot_retrieval/quilt.sh`. ## Citation -If you find the code useful for your research, please consider citing + +If you find this code useful for your research, please consider citing: + ```bib @article{baghbanzadeh2025advancing, title={Advancing Medical Representation Learning Through High-Quality Data}, @@ -164,4 +139,12 @@ If you find the code useful for your research, please consider citing journal={arXiv preprint arXiv:2503.14377}, year={2025} } + +@inproceedings{baghbanzadeh2025openpmc18m, + title={Open-PMC-18M: A High-Fidelity Large Scale Medical Dataset for Multimodal Representation Learning}, + author={Baghbanzadeh, Negin and Islam, Mohammed Saidul and Ashkezari, Sajad and Dolatabadi, Elham and Afkanpour, Arash}, + booktitle={Medical Image Computing and Computer-Assisted Intervention (MICCAI)}, + year={2026}, + note={arXiv:2506.02738} +} ``` diff --git a/evaluation/README.md b/evaluation/README.md new file mode 100644 index 0000000..c534c0c --- /dev/null +++ b/evaluation/README.md @@ -0,0 +1,132 @@ +# Evaluation + +Zero-shot evaluation of an open_clip-format checkpoint with the BiomedCLIP architecture +(ViT-B/16 image encoder + PubMedBERT text encoder), such as the **Open-PMC-18M** model: +[vector-institute/open-pmc-18m-clip](https://huggingface.co/vector-institute/open-pmc-18m-clip). + +``` +evaluation/ +β”œβ”€β”€ zero_shot_retrieval/ # image ↔ text cross-modal retrieval +β”‚ β”œβ”€β”€ quilt.sh # Quilt-1M +β”‚ β”œβ”€β”€ mimic.sh # MIMIC-IV-CXR +β”‚ └── deepeyenet.sh # DeepEyeNet +└── zero_shot_classification/ # zero-shot image classification + β”œβ”€β”€ pcam.sh # PatchCamelyon + β”œβ”€β”€ bach.sh # BACH (breast histology) + β”œβ”€β”€ sicap.sh # SICAPv2 (prostate) + β”œβ”€β”€ nck_crc.sh # NCT-CRC-HE (colorectal) + β”œβ”€β”€ pad_ufes_20.sh # PAD-UFES-20 (skin lesions) + β”œβ”€β”€ ham10000.sh # HAM10000 (skin lesions) + β”œβ”€β”€ lc25000_lung.sh # LC25000 (lung histology) + └── medmnist.sh # MedMNIST+ variants (pathmnist, dermamnist, ...) +``` + +Every script loads the checkpoint through the encoders' `checkpoint_path` argument (an +open_clip-native `state_dict` with `visual.*` / `text.*` keys) β€” no Lightning +`resume_from_checkpoint` conversion needed. Retrieval uses the `biomedclip_localckpt_retrieval` +config; classification uses `biomedclip_localckpt_ZSC`. + +## 1. Setup + +Install the repo (see the [top-level README](../README.md)). **No checkpoint setup is needed** β€” +by default the scripts evaluate the released **Open-PMC-18M** checkpoint +([vector-institute/open-pmc-18m-clip](https://huggingface.co/vector-institute/open-pmc-18m-clip)), +which is downloaded automatically from the Hugging Face Hub on first run. + +To evaluate a **different** checkpoint instead, set `CKPT` to a local open_clip `.pt`/`.bin` file +(or another `hf-hub:/` reference). + +## 2. Run + +**Every dataset reads its location from a `*_ROOT_DIR` environment variable** (see +[Data setup](#3-data-setup) below). If a `*_ROOT_DIR` is not set, the run fails with a Hydra +*"missing mandatory value"* error for `root_dir`. + +```bash +# retrieval β€” evaluates the released Open-PMC-18M checkpoint by default +QUILT_ROOT_DIR=/data/quilt bash evaluation/zero_shot_retrieval/quilt.sh + +# classification +PCAM_ROOT_DIR=/data/pcam bash evaluation/zero_shot_classification/pcam.sh +NAME=pathmnist MEDMNISTPLUS_ROOT_DIR=/data/medmnist bash evaluation/zero_shot_classification/medmnist.sh + +# to evaluate your own checkpoint instead, set CKPT to a local file +CKPT=/path/to/checkpoint.pt QUILT_ROOT_DIR=/data/quilt bash evaluation/zero_shot_retrieval/quilt.sh +``` + +Retrieval reports Recall@{10, 50, 200} in both directions; classification reports top-1 accuracy. + +## 3. Data setup + +Set the listed `*_ROOT_DIR` before running. Datasets marked **auto** download themselves from the +Hugging Face Hub into that directory (make it writable); datasets marked **manual** must be +downloaded from the source first and arranged as shown. + +| Script | `*_ROOT_DIR` | Source | Download | +|-------------------|-------------------------|-----------------------------------------|----------| +| `quilt.sh` | `QUILT_ROOT_DIR` | [Quilt-1M](https://github.com/wisdomikezogwo/quilt1m) | manual | +| `mimic.sh` | `MIMICIVCXR_ROOT_DIR` | [MIMIC-CXR](https://physionet.org/content/mimic-cxr/) (credentialed) | manual | +| `deepeyenet.sh` | `DEY_ROOT_DIR` | [DeepEyeNet](https://github.com/Jhhuangkay/DeepOpht-Medical-Report-Generation-for-Retinal-Images-via-Deep-Models-and-Visual-Explanation) | manual | +| `pcam.sh` | `PCAM_ROOT_DIR` | HF `1aurent/PatchCamelyon` | auto | +| `bach.sh` | `BACH_ROOT_DIR` | HF `1aurent/BACH` | auto | +| `nck_crc.sh` | `NCK_CRC_ROOT_DIR` | HF `DykeF/NCTCRCHE100K` | auto | +| `sicap.sh` | `SICAP_ROOT_DIR` | [SICAPv2](https://data.mendeley.com/datasets/9xxm58dvs3/1) | manual | +| `pad_ufes_20.sh` | `PADUFES_ROOT_DIR` | [PAD-UFES-20](https://data.mendeley.com/datasets/zr7vgbcyr2/1) | manual | +| `ham10000.sh` | `HAM10000_ROOT_DIR` | [HAM10000](https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/DBW86T) | manual | +| `lc25000_lung.sh` | `LC25000_LUNG_ROOT_DIR` (and `LC25000_COLON_ROOT_DIR` for colon) | [LC25000](https://github.com/tampapath/lung_colon_image_set) | manual (pre-build) | +| `medmnist.sh` | `MEDMNISTPLUS_ROOT_DIR` | [MedMNIST+](https://medmnist.com/) (224px) | manual | + +### Expected layout for the manual datasets + +**Quilt-1M** β€” `$QUILT_ROOT_DIR/` +``` +quilt_1m_val.csv # split metadata (also quilt_1m_train.csv) +quilt_1m/ # image files referenced by image_path in the csv +``` + +**MIMIC-IV-CXR** β€” `$MIMICIVCXR_ROOT_DIR/` β€” the split metadata file (`.json`/`.csv`) plus the +CXR image tree. Requires credentialed PhysioNet access. + +**DeepEyeNet** β€” `$DEY_ROOT_DIR/` +``` +DeepEyeNet_test.json # split file (also _train.json / _val.json) + # image paths referenced by the json keys +``` + +**SICAPv2** β€” `$SICAP_ROOT_DIR/` +``` +images/ # patch images +partition/Test/Train.xlsx +partition/Test/Test.xlsx # columns: image_name, NC, G3, G4, G5 +``` + +**PAD-UFES-20** β€” `$PADUFES_ROOT_DIR/` +``` +metadata.csv # columns: img_id, diagnostic +Dataset/ # image files named by img_id +``` + +**HAM10000** β€” `$HAM10000_ROOT_DIR/` +``` +HAM10000_metadata.csv # train/test csvs are derived from this on first run +skin_cancer/ # .jpg files +``` + +**LC25000** β€” pre-build a πŸ€— `datasets` arrow directory per organ/split: +`$LC25000_LUNG_ROOT_DIR/cache/lc25000_lung_test.arrow` (and `..._colon_test.arrow` under +`$LC25000_COLON_ROOT_DIR`). Each record needs `image` and `label` fields. + +**MedMNIST+** β€” `$MEDMNISTPLUS_ROOT_DIR/` with the 224px npz files, e.g. `pathmnist_224.npz`, +`dermamnist_224.npz`, `bloodmnist_224.npz`, `organamnist_224.npz`, … Select one with `NAME=`. + +**Auto datasets** (PCam, BACH, NCT-CRC-HE) need only a writable `*_ROOT_DIR`; on first run they +download into `/scratch/` and cache into `/cache/`. + +## Notes + +- **Different checkpoint / dataset.** Point `CKPT` at any open_clip-format checkpoint with the + BiomedCLIP architecture. To evaluate another retrieval dataset, copy a script and swap the + dataset name (e.g. `+datasets@datasets.test.=`). +- **SLURM.** The scripts run a single process on the local GPU. To submit to a cluster, wrap the + `mmlearn_run` call with `--multirun` and the hydra submitit launcher, or call the script from + your `sbatch` wrapper. diff --git a/evaluation/zero_shot_classification/bach.sh b/evaluation/zero_shot_classification/bach.sh new file mode 100755 index 0000000..4077190 --- /dev/null +++ b/evaluation/zero_shot_classification/bach.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Zero-shot image classification on BACH (breast histology). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. Set CKPT to a local +# open_clip .pt/.bin file to evaluate a different checkpoint. +# +# Set BACH_ROOT_DIR to the dataset root directory (see evaluation/README.md). +# +# Usage: +# BACH_ROOT_DIR=/path/to/bach bash bach.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_ZSC \ + experiment_name=open_pmc_18m_zsc_bach \ + job_type=eval \ + +datasets@datasets.test.bach=BACH \ + datasets.test.bach.split=test \ + +datasets/transforms@datasets.test.bach.transform=biomedclip_vision_transform \ + datasets.test.bach.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 diff --git a/evaluation/zero_shot_classification/ham10000.sh b/evaluation/zero_shot_classification/ham10000.sh new file mode 100755 index 0000000..26434f4 --- /dev/null +++ b/evaluation/zero_shot_classification/ham10000.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Zero-shot image classification on HAM10000 (skin lesions). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. Set CKPT to a local +# open_clip .pt/.bin file to evaluate a different checkpoint. +# +# Set HAM10000_ROOT_DIR to the dataset root directory (see evaluation/README.md). +# +# Usage: +# HAM10000_ROOT_DIR=/path/to/ham10k bash ham10000.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_ZSC \ + experiment_name=open_pmc_18m_zsc_ham10k \ + job_type=eval \ + +datasets@datasets.test.ham10k=HAM10000 \ + datasets.test.ham10k.split=test \ + +datasets/transforms@datasets.test.ham10k.transform=biomedclip_vision_transform \ + datasets.test.ham10k.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 diff --git a/evaluation/zero_shot_classification/lc25000_lung.sh b/evaluation/zero_shot_classification/lc25000_lung.sh new file mode 100755 index 0000000..3ea83c0 --- /dev/null +++ b/evaluation/zero_shot_classification/lc25000_lung.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Zero-shot image classification on LC25000 (lung histology). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. Set CKPT to a local +# open_clip .pt/.bin file to evaluate a different checkpoint. +# +# Set LC25000_LUNG_ROOT_DIR to the dataset root directory (see evaluation/README.md). +# +# Usage: +# LC25000_LUNG_ROOT_DIR=/path/to/lc25k_lung bash lc25000_lung.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_ZSC \ + experiment_name=open_pmc_18m_zsc_lc25k_lung \ + job_type=eval \ + +datasets@datasets.test.lc25k_lung=LC25000 \ + datasets.test.lc25k_lung.split=test \ + datasets.test.lc25k_lung.organ=lung \ + +datasets/transforms@datasets.test.lc25k_lung.transform=biomedclip_vision_transform \ + datasets.test.lc25k_lung.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 diff --git a/evaluation/zero_shot_classification/medmnist.sh b/evaluation/zero_shot_classification/medmnist.sh new file mode 100755 index 0000000..2935713 --- /dev/null +++ b/evaluation/zero_shot_classification/medmnist.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Zero-shot image classification on a MedMNIST+ variant (224px). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. Set CKPT to a local +# open_clip .pt/.bin file to evaluate a different checkpoint. +# +# Set MEDMNISTPLUS_ROOT_DIR to a dir with the MedMNIST+ 224px files (e.g. pathmnist_224.npz). +# +# Usage (NAME selects the variant; defaults to pathmnist): +# NAME=pathmnist MEDMNISTPLUS_ROOT_DIR=/data/medmnist bash medmnist.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" +NAME="${NAME:-pathmnist}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_ZSC \ + experiment_name=open_pmc_18m_zsc_medmnist_${NAME} \ + job_type=eval \ + +datasets@datasets.test.medmnist=MedMNISTPlus \ + datasets.test.medmnist.name=${NAME} \ + datasets.test.medmnist.split=test \ + +datasets/transforms@datasets.test.medmnist.transform=biomedclip_vision_transform \ + datasets.test.medmnist.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 diff --git a/evaluation/zero_shot_classification/nck_crc.sh b/evaluation/zero_shot_classification/nck_crc.sh new file mode 100755 index 0000000..5725a46 --- /dev/null +++ b/evaluation/zero_shot_classification/nck_crc.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Zero-shot image classification on NCT-CRC-HE (colorectal). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. Set CKPT to a local +# open_clip .pt/.bin file to evaluate a different checkpoint. +# +# Set NCK_CRC_ROOT_DIR to the dataset root directory (see evaluation/README.md). +# +# Usage: +# NCK_CRC_ROOT_DIR=/path/to/nck_crc bash nck_crc.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_ZSC \ + experiment_name=open_pmc_18m_zsc_nck_crc \ + job_type=eval \ + +datasets@datasets.test.nck_crc=NckCrc \ + datasets.test.nck_crc.split=validation \ + +datasets/transforms@datasets.test.nck_crc.transform=biomedclip_vision_transform \ + datasets.test.nck_crc.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 diff --git a/evaluation/zero_shot_classification/pad_ufes_20.sh b/evaluation/zero_shot_classification/pad_ufes_20.sh new file mode 100755 index 0000000..4545797 --- /dev/null +++ b/evaluation/zero_shot_classification/pad_ufes_20.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Zero-shot image classification on PAD-UFES-20 (skin lesions). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. Set CKPT to a local +# open_clip .pt/.bin file to evaluate a different checkpoint. +# +# Set PADUFES_ROOT_DIR to the dataset root directory (see evaluation/README.md). +# +# Usage: +# PADUFES_ROOT_DIR=/path/to/pad_ufes_20 bash pad_ufes_20.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_ZSC \ + experiment_name=open_pmc_18m_zsc_pad_ufes_20 \ + job_type=eval \ + +datasets@datasets.test.pad_ufes_20=PadUfes20 \ + datasets.test.pad_ufes_20.split=test \ + +datasets/transforms@datasets.test.pad_ufes_20.transform=biomedclip_vision_transform \ + datasets.test.pad_ufes_20.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 diff --git a/evaluation/zero_shot_classification/pcam.sh b/evaluation/zero_shot_classification/pcam.sh new file mode 100755 index 0000000..34b22b6 --- /dev/null +++ b/evaluation/zero_shot_classification/pcam.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Zero-shot image classification on PatchCamelyon (PCam). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. Set CKPT to a local +# open_clip .pt/.bin file to evaluate a different checkpoint. +# +# Set PCAM_ROOT_DIR to the dataset root directory (see evaluation/README.md). +# +# Usage: +# PCAM_ROOT_DIR=/path/to/pcam bash pcam.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_ZSC \ + experiment_name=open_pmc_18m_zsc_pcam \ + job_type=eval \ + +datasets@datasets.test.pcam=PCAM \ + +datasets/transforms@datasets.test.pcam.transform=biomedclip_vision_transform \ + datasets.test.pcam.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 diff --git a/evaluation/zero_shot_classification/sicap.sh b/evaluation/zero_shot_classification/sicap.sh new file mode 100755 index 0000000..bcd70fb --- /dev/null +++ b/evaluation/zero_shot_classification/sicap.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Zero-shot image classification on SICAPv2 (prostate). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. Set CKPT to a local +# open_clip .pt/.bin file to evaluate a different checkpoint. +# +# Set SICAP_ROOT_DIR to the dataset root directory (see evaluation/README.md). +# +# Usage: +# SICAP_ROOT_DIR=/path/to/sicap bash sicap.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_ZSC \ + experiment_name=open_pmc_18m_zsc_sicap \ + job_type=eval \ + +datasets@datasets.test.sicap=SICAP \ + datasets.test.sicap.split=test \ + +datasets/transforms@datasets.test.sicap.transform=biomedclip_vision_transform \ + datasets.test.sicap.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 diff --git a/evaluation/zero_shot_retrieval/deepeyenet.sh b/evaluation/zero_shot_retrieval/deepeyenet.sh new file mode 100755 index 0000000..31c302c --- /dev/null +++ b/evaluation/zero_shot_retrieval/deepeyenet.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Zero-shot cross-modal (image <-> text) retrieval on DeepEyeNet (test split). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. To evaluate a different +# checkpoint, set CKPT to a local open_clip .pt/.bin file: +# CKPT=/path/to/checkpoint.pt bash deepeyenet.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_retrieval \ + experiment_name=open_pmc_18m_retrieval_dey \ + job_type=eval \ + +datasets@datasets.test.dey=DeepEyeNet \ + datasets.test.dey.split=test \ + +datasets/transforms@datasets.test.dey.transform=biomedclip_vision_transform \ + datasets.test.dey.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 \ + task.evaluation_tasks.retrieval.task.task_specs.0.top_k='[10,50,200]' \ + task.evaluation_tasks.retrieval.task.task_specs.1.top_k='[10,50,200]' \ + ~task.postprocessors.norm_and_logit_scale.logit_scale \ + ~task.postprocessors.norm_and_logit_scale.norm diff --git a/evaluation/zero_shot_retrieval/mimic.sh b/evaluation/zero_shot_retrieval/mimic.sh new file mode 100755 index 0000000..6d4a2f1 --- /dev/null +++ b/evaluation/zero_shot_retrieval/mimic.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Zero-shot cross-modal (image <-> text) retrieval on MIMIC-IV-CXR (test split). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. To evaluate a different +# checkpoint, set CKPT to a local open_clip .pt/.bin file: +# CKPT=/path/to/checkpoint.pt bash mimic.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_retrieval \ + experiment_name=open_pmc_18m_retrieval_mimic \ + job_type=eval \ + +datasets@datasets.test.mimic=MIMICIVCXR \ + datasets.test.mimic.split=test \ + +datasets/transforms@datasets.test.mimic.transform=biomedclip_vision_transform \ + datasets.test.mimic.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 \ + task.evaluation_tasks.retrieval.task.task_specs.0.top_k='[10,50,200]' \ + task.evaluation_tasks.retrieval.task.task_specs.1.top_k='[10,50,200]' \ + ~task.postprocessors.norm_and_logit_scale.logit_scale \ + ~task.postprocessors.norm_and_logit_scale.norm diff --git a/evaluation/zero_shot_retrieval/quilt.sh b/evaluation/zero_shot_retrieval/quilt.sh new file mode 100755 index 0000000..18c6eea --- /dev/null +++ b/evaluation/zero_shot_retrieval/quilt.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Zero-shot cross-modal (image <-> text) retrieval on Quilt-1M (val split). +# +# By default this evaluates the released Open-PMC-18M checkpoint from the Hugging Face Hub +# (vector-institute/open-pmc-18m-clip), downloaded automatically. To evaluate a different +# checkpoint, set CKPT to a local open_clip .pt/.bin file: +# CKPT=/path/to/checkpoint.pt bash quilt.sh +set -euo pipefail +# Keep temp files on node-local disk to avoid NFS ".nfs* busy" cleanup errors on clusters. +export TMPDIR="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}" +CKPT="${CKPT:-hf-hub:vector-institute/open-pmc-18m-clip}" + +mmlearn_run 'hydra.searchpath=[pkg://openpmcvl.experiment.configs]' \ + +experiment=biomedclip_localckpt_retrieval \ + experiment_name=open_pmc_18m_retrieval_quilt \ + job_type=eval \ + +datasets@datasets.test.quilt=Quilt \ + datasets.test.quilt.split=val \ + +datasets/transforms@datasets.test.quilt.transform=biomedclip_vision_transform \ + datasets.test.quilt.transform.job_type=eval \ + task.encoders.text.checkpoint_path="$CKPT" \ + task.encoders.rgb.checkpoint_path="$CKPT" \ + dataloader.test.batch_size=64 \ + dataloader.test.num_workers=4 \ + task.evaluation_tasks.retrieval.task.task_specs.0.top_k='[10,50,200]' \ + task.evaluation_tasks.retrieval.task.task_specs.1.top_k='[10,50,200]' \ + ~task.postprocessors.norm_and_logit_scale.logit_scale \ + ~task.postprocessors.norm_and_logit_scale.norm diff --git a/openpmcvl/experiment/configs/__init__.py b/openpmcvl/experiment/configs/__init__.py index c95488a..1de379d 100644 --- a/openpmcvl/experiment/configs/__init__.py +++ b/openpmcvl/experiment/configs/__init__.py @@ -10,14 +10,22 @@ from timm.data.transforms import ResizeKeepRatio from torchvision import transforms +from openpmcvl.experiment.datasets.bach import BACH from openpmcvl.experiment.datasets.deepeyenet import DeepEyeNet +from openpmcvl.experiment.datasets.ham10000 import HAM10000 +from openpmcvl.experiment.datasets.lc25000 import LC25000 +from openpmcvl.experiment.datasets.med_mnist_plus import MedMNISTPlus from openpmcvl.experiment.datasets.mimiciv_cxr import MIMICIVCXR +from openpmcvl.experiment.datasets.nck import NckCrc +from openpmcvl.experiment.datasets.pad_ufes_20 import PadUfes20 +from openpmcvl.experiment.datasets.pcam import PCAM from openpmcvl.experiment.datasets.pmc2m_sum import PMC2MSum from openpmcvl.experiment.datasets.pmcoa import PMCOA from openpmcvl.experiment.datasets.pmcpatients import PMCPatients from openpmcvl.experiment.datasets.pmcvl import PMCVL from openpmcvl.experiment.datasets.quilt1m import Quilt from openpmcvl.experiment.datasets.roco import ROCO +from openpmcvl.experiment.datasets.sicap import SICAP from openpmcvl.experiment.modules.contrastive_pretraining_ppr import ( ContrastivePretrainingPPR, ) diff --git a/openpmcvl/experiment/configs/experiment/biomedclip_localckpt_ZSC.yaml b/openpmcvl/experiment/configs/experiment/biomedclip_localckpt_ZSC.yaml new file mode 100644 index 0000000..d6c0f79 --- /dev/null +++ b/openpmcvl/experiment/configs/experiment/biomedclip_localckpt_ZSC.yaml @@ -0,0 +1,76 @@ +# @package _global_ + +# Zero-shot classification evaluation for a LOCAL open_clip-format CLIP checkpoint +# with the BiomedCLIP architecture (ViT-B/16 image encoder + PubMedBERT text encoder), +# e.g. the Open-PMC-18M checkpoint: https://huggingface.co/vector-institute/open-pmc-18m-clip +# +# The checkpoint is loaded through the encoders' `checkpoint_path` argument (an +# open_clip-native state_dict with `visual.*` / `text.*` keys) -- exactly the same +# loading path as biomedclip_localckpt_retrieval, so no Lightning +# `resume_from_checkpoint` conversion is needed. Pass the checkpoint path and the +# classification dataset (pcam / bach / sicap / ...) from the launch script. + +defaults: + - /datasets/tokenizers@dataloader.test.collate_fn.batch_processors.text: BiomedCLIPTokenizer + - /modules/encoders@task.encoders.text: BiomedCLIPText + - /modules/encoders@task.encoders.rgb: BiomedCLIPVision + - /modules/layers@task.postprocessors.norm_and_logit_scale.norm: L2Norm + - /modules/layers@task.postprocessors.norm_and_logit_scale.logit_scale: LearnableLogitScaling + - /eval_task@task.evaluation_tasks.classification.task: ZeroShotClassification + - /datasets/tokenizers@task.evaluation_tasks.classification.task.tokenizer: BiomedCLIPTokenizer + - override /task: ContrastivePretraining + - _self_ + +seed: 0 +job_type: eval + +dataloader: + test: + batch_size: 64 + num_workers: 4 + +task: + encoders: + text: + # Default to the released Open-PMC-18M checkpoint on the HF Hub; override with a + # local open_clip .pt/.bin on the command line (see evaluation/ scripts). + pretrained: False + checkpoint_path: hf-hub:vector-institute/open-pmc-18m-clip + rgb: + pretrained: False + checkpoint_path: hf-hub:vector-institute/open-pmc-18m-clip + postprocessors: + norm_and_logit_scale: + norm: + dim: -1 + logit_scale: + learnable: True + modality_module_mapping: + text: + postprocessor_key: norm_and_logit_scale + rgb: + postprocessor_key: norm_and_logit_scale + evaluation_tasks: + classification: + task: + task_specs: + - top_k: [1] + query_modality: rgb + run_on_validation: false + run_on_test: true + compute_validation_loss: False + compute_test_loss: False + +trainer: + precision: 16-mixed + deterministic: False + benchmark: True + sync_batchnorm: False + log_every_n_steps: 100 + +tags: + - ${experiment_name} + - zeroshot + - classification + - biomedclip + - openpmcvl diff --git a/openpmcvl/experiment/configs/experiment/biomedclip_localckpt_retrieval.yaml b/openpmcvl/experiment/configs/experiment/biomedclip_localckpt_retrieval.yaml new file mode 100644 index 0000000..8d6c07c --- /dev/null +++ b/openpmcvl/experiment/configs/experiment/biomedclip_localckpt_retrieval.yaml @@ -0,0 +1,93 @@ +# @package _global_ + +# Zero-shot cross-modal retrieval evaluation for a LOCAL open_clip-format CLIP checkpoint +# with the BiomedCLIP architecture (ViT-B/16 image encoder + PubMedBERT text encoder), +# e.g. the Open-PMC-18M checkpoint: https://huggingface.co/vector-institute/open-pmc-18m-clip +# +# The checkpoint is loaded directly into the encoders through their `checkpoint_path` +# argument (an open_clip-native state_dict with `visual.*` / `text.*` keys), so no +# Lightning `resume_from_checkpoint` conversion is needed. Pass the checkpoint path and +# the test dataset (mimic / quilt / deepeyenet) from the launch script. + +defaults: + - /datasets/tokenizers@dataloader.test.collate_fn.batch_processors.text: BiomedCLIPTokenizer + - /modules/encoders@task.encoders.text: BiomedCLIPText + - /modules/encoders@task.encoders.rgb: BiomedCLIPVision + - /modules/losses@task.loss: CLIPLoss + - /modules/layers@task.postprocessors.norm_and_logit_scale.norm: L2Norm + - /modules/layers@task.postprocessors.norm_and_logit_scale.logit_scale: LearnableLogitScaling + - /eval_task@task.evaluation_tasks.retrieval.task: ZeroShotCrossModalRetrieval + - override /task: ContrastivePretraining + - _self_ + +seed: 0 + +dataloader: + test: + num_workers: 4 + +task: + encoders: + text: + # Default to the released Open-PMC-18M checkpoint on the HF Hub; override with a + # local open_clip .pt/.bin on the command line (see evaluation/ scripts). + pretrained: False + checkpoint_path: hf-hub:vector-institute/open-pmc-18m-clip + rgb: + pretrained: False + checkpoint_path: hf-hub:vector-institute/open-pmc-18m-clip + postprocessors: + norm_and_logit_scale: + norm: + dim: -1 + logit_scale: + learnable: True + modality_module_mapping: + text: + postprocessor_key: norm_and_logit_scale + rgb: + postprocessor_key: norm_and_logit_scale + optimizer: + betas: + - 0.9 + - 0.98 + lr: 5.0e-4 + weight_decay: 0.2 + eps: 1.0e-6 + lr_scheduler: + scheduler: + t_max: 104_671 + warmup_length: 2000 + extras: + interval: step + loss: + gather_with_grad: True + local_loss: True + evaluation_tasks: + retrieval: + task: + task_specs: + - query_modality: text + target_modality: rgb + top_k: [1, 5, 10] + - query_modality: rgb + target_modality: text + top_k: [1, 5, 10] + run_on_validation: false + run_on_test: true + +trainer: + max_epochs: 32 + precision: bf16-mixed + deterministic: False + benchmark: True + sync_batchnorm: False + log_every_n_steps: 100 + accumulate_grad_batches: 4 + check_val_every_n_epoch: 1 + +tags: + - ${experiment_name} + - retrieval + - biomedclip + - openpmcvl diff --git a/openpmcvl/experiment/datasets/bach.py b/openpmcvl/experiment/datasets/bach.py new file mode 100644 index 0000000..367bbd4 --- /dev/null +++ b/openpmcvl/experiment/datasets/bach.py @@ -0,0 +1,96 @@ +"""BACH Dataset.""" + +import os +from typing import Callable, Dict, Literal, Optional + +import torch +from datasets import load_dataset +from mmlearn.conf import external_store +from mmlearn.constants import EXAMPLE_INDEX_KEY +from mmlearn.datasets.core import Modalities +from mmlearn.datasets.core.example import Example +from omegaconf import MISSING +from PIL import Image +from torch.utils.data import Dataset +from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor + + +@external_store(group="datasets", root_dir=os.getenv("BACH_ROOT_DIR", MISSING)) +class BACH(Dataset[Example]): + """BACH dataset for breast cancer classification. + + Parameters + ---------- + root_dir : str + Path to the dataset directory or cache directory. + split : str + Dataset split, one of 'train' or 'test'. + transform : Optional[Callable], default=None + Transform applied to images. + """ + + def __init__( + self, + root_dir: str, + split: Literal["train", "test"], + transform: Optional[Callable[[Image.Image], torch.Tensor]] = None, + ) -> None: + """Initialize the BACH dataset.""" + os.makedirs(os.path.join(root_dir, "cache/"), exist_ok=True) + + dataset = load_dataset( + "1aurent/BACH", + cache_dir=os.path.join(root_dir, "scratch/"), + split="train", + ) + data_dict = dataset.train_test_split( + test_size=0.25, train_size=0.75, shuffle=True, seed=0 + ) + self.data = data_dict[split] + + self.transform = ( + Compose([Resize(224), CenterCrop(224), ToTensor()]) + if transform is None + else transform + ) + print(f"DATASET LEN ::::::::::: {self.__len__()}") + + def __getitem__(self, idx: int) -> Example: + """Return the idx'th data sample as an Example instance.""" + entry = self.data[idx] + image = entry["image"] + + if self.transform is not None: + image = self.transform(image) + + return Example( + { + Modalities.RGB.name: image, + Modalities.RGB.target: int(entry["label"]), + EXAMPLE_INDEX_KEY: idx, + } + ) + + def __len__(self) -> int: + """Return the length of the dataset.""" + return len(self.data) + + @property + def id2label(self) -> Dict[int, str]: + """Return the label mapping for the BACH dataset.""" + return { + 0: "breast non-malignant benign tissue", + 1: "breast malignant in-situ carcinoma", + 2: "breast malignant invasive carcinoma", + 3: "breast normal breast tissue", + } + + @property + def zero_shot_prompt_templates(self) -> list[str]: + """Return the zero-shot prompt templates.""" + return [ + "a histopathology slide showing {}", + "histopathology image of {}", + "pathology tissue showing {}", + "presence of {} tissue on image", + ] diff --git a/openpmcvl/experiment/datasets/ham10000.py b/openpmcvl/experiment/datasets/ham10000.py new file mode 100644 index 0000000..e63e53b --- /dev/null +++ b/openpmcvl/experiment/datasets/ham10000.py @@ -0,0 +1,127 @@ +"""HAM10000 Dataset.""" + +import os +from typing import Callable, Dict, Optional + +import pandas as pd +import torch +from mmlearn.conf import external_store +from mmlearn.constants import EXAMPLE_INDEX_KEY +from mmlearn.datasets.core import Modalities +from mmlearn.datasets.core.example import Example +from omegaconf import MISSING +from PIL import Image +from sklearn.model_selection import train_test_split +from torch.utils.data import Dataset +from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor + + +@external_store(group="datasets", root_dir=os.getenv("HAM10000_ROOT_DIR", MISSING)) +class HAM10000(Dataset[Example]): + """HAM10000 dataset for zero-shot classification. + + Parameters + ---------- + root_dir : str + Path to the dataset directory containing images and metadata CSV. + transform : Optional[Callable], default=None + Transform applied to images. + """ + + def __init__( + self, + root_dir: str, + split: str = "test", + transform: Optional[Callable[[Image.Image], torch.Tensor]] = None, + ) -> None: + """Initialize the HAM10000 dataset.""" + self.root_dir = root_dir + + # Check if the split-specific CSV files exist + train_csv = os.path.join(root_dir, "HAM10000_train.csv") + test_csv = os.path.join(root_dir, "HAM10000_test.csv") + + if not os.path.exists(train_csv) or not os.path.exists(test_csv): + # Load the original metadata CSV + original_metadata = pd.read_csv( + os.path.join(root_dir, "HAM10000_metadata.csv") + ) + # Split the data into train and test + train_data, test_data = train_test_split( + original_metadata, test_size=0.2, random_state=42 + ) + # Save the splits as new CSV files + train_data.to_csv(train_csv, index=False) + test_data.to_csv(test_csv, index=False) + + # Load the metadata for the requested split + if split == "train": + self.metadata = pd.read_csv(train_csv) + elif split == "test": + self.metadata = pd.read_csv(test_csv) + else: + raise ValueError("Split must be 'train' or 'test'") + + self.classes = ["nv", "mel", "bkl", "bcc", "akiec", "vasc", "df"] + + self.transform = ( + Compose([Resize(224), CenterCrop(224), ToTensor()]) + if transform is None + else transform + ) + print(f"DATASET LEN ::::::::::: {self.__len__()}") + + @property + def zero_shot_prompt_templates(self) -> list[str]: + """Return the zero-shot prompt templates.""" + return [ + "a histopathology slide showing {}", + "histopathology image of {}", + "pathology tissue showing {}", + "presence of {} tissue on image", + ] + + def __len__(self) -> int: + """Return the length of the dataset.""" + return len(self.metadata) + + def __getitem__(self, idx: int) -> Example: + """Return the idx'th data sample as an Example instance.""" + entry = self.metadata.iloc[idx] + image_path = os.path.join( + self.root_dir, "skin_cancer", f"{entry['image_id']}.jpg" + ) + + with Image.open(image_path) as img: + image = img.convert("RGB") + + if self.transform is not None: + image = self.transform(image) + + label_index = self.classes.index(entry["dx"]) + + return Example( + { + Modalities.RGB.name: image, + Modalities.RGB.target: label_index, + EXAMPLE_INDEX_KEY: idx, + } + ) + + @property + def id2label(self) -> Dict[int, str]: + """Return the label mapping.""" + return { + 0: "Melanocytic Nevi", + 1: "Melanoma", + 2: "Benign Keratosis-like Lesions", + 3: "Basal Cell Carcinoma", + 4: "Actinic Keratoses and Intraepithelial Carcinoma", + 5: "Vascular Lesions", + 6: "Dermatofibroma", + } + + @property + def is_multilabel(self) -> bool: + """Return whether the dataset uses multi-label targets.""" + return False diff --git a/openpmcvl/experiment/datasets/lc25000.py b/openpmcvl/experiment/datasets/lc25000.py new file mode 100644 index 0000000..fb1ed2b --- /dev/null +++ b/openpmcvl/experiment/datasets/lc25000.py @@ -0,0 +1,106 @@ +"""LC25000 Dataset.""" + +import os +from typing import Callable, Dict, Literal, Optional + +import torch +from datasets import load_from_disk +from mmlearn.conf import external_store +from mmlearn.constants import EXAMPLE_INDEX_KEY +from mmlearn.datasets.core import Modalities +from mmlearn.datasets.core.example import Example +from omegaconf import MISSING +from PIL import Image +from torch.utils.data import Dataset +from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor + + +@external_store(group="datasets", root_dir=os.getenv("LC25000_LUNG_ROOT_DIR", MISSING)) +class LC25000(Dataset[Example]): + """LC25000 dataset for zero-shot classification. + + Parameters + ---------- + root_dir : str + Path to the dataset directory. + organ : {'lung', 'colon'}, default='lung' + Organ type ('lung' or 'colon'). + transform : Optional[Callable], default=None + Transform applied to images. + """ + + def __init__( + self, + root_dir: str, + split: Literal["train", "test"], + organ: Literal["lung", "colon"] = "lung", + transform: Optional[Callable[[Image.Image], torch.Tensor]] = None, + geom_transform_type: Optional[str] = "base", + ) -> None: + """Initialize the LC25000 dataset.""" + self.geom_transform_type = geom_transform_type + self.organ = organ + dataset_path = os.path.join(root_dir, f"cache/lc25000_{organ}_{split}.arrow") + + if os.path.exists(dataset_path): + print("!!! Using cached dataset") + dataset = load_from_disk(dataset_path) + else: + raise ValueError("Dataset does not exist") + + self.transform = ( + Compose([Resize(224), CenterCrop(224), ToTensor()]) + if transform is None + else transform + ) + self.data = dataset + print(f"DATASET LEN {split} ::::::::::: {self.__len__()}") + + @property + def name(self) -> str: + """Return the dataset name based on the organ (lung or colon).""" + if self.organ == "lung": + return "LC25000_lung" + return "LC25000_colon" + + @property + def id2label(self) -> Dict[int, str]: + """Return the label mapping.""" + if self.organ == "lung": + return { + 0: "benign lung", + 1: "lung adenocarcinoma", + 2: "lung squamous cell carcinoma", + } + + return {0: "benign colonic tissue", 1: "colon adenocarcinoma"} + + @property + def zero_shot_prompt_templates(self) -> list[str]: + """Return the zero-shot prompt templates.""" + return [ + "a histopathology slide showing {}.", + "histopathology image of {}.", + "pathology tissue showing {}.", + "presence of {} tissue on image.", + ] + + def __len__(self) -> int: + """Return the length of the dataset.""" + return len(self.data) + + def __getitem__(self, idx: int) -> Example: + """Return the idx'th data sample as an Example instance.""" + entry = self.data[idx] + image = entry["image"] + + if self.transform is not None: + image = self.transform(image) + + return Example( + { + Modalities.RGB.name: image, + Modalities.RGB.target: int(entry["label"]), + EXAMPLE_INDEX_KEY: idx, + } + ) diff --git a/openpmcvl/experiment/datasets/med_mnist_plus.py b/openpmcvl/experiment/datasets/med_mnist_plus.py new file mode 100644 index 0000000..ea8b411 --- /dev/null +++ b/openpmcvl/experiment/datasets/med_mnist_plus.py @@ -0,0 +1,208 @@ +"""MedMNIST+ Dataset.""" + +import os +from typing import Callable, Dict, Literal, Optional + +import numpy as np +import torch +from mmlearn.conf import external_store +from mmlearn.constants import EXAMPLE_INDEX_KEY +from mmlearn.datasets.core import Modalities +from mmlearn.datasets.core.example import Example +from omegaconf import MISSING +from PIL import Image +from torch.utils.data import Dataset +from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor + + +@external_store(group="datasets", root_dir=os.getenv("MEDMNISTPLUS_ROOT_DIR", MISSING)) +class MedMNISTPlus(Dataset[Example]): + """MedMNISTPlus dataset for zero-shot classification. + + Parameters + ---------- + root_dir : str + Path to the dataset directory containing images and metadata. + split : {'train', 'val', 'test'} + Dataset split. + name : str, default='organamnist' + Specific name of the MedMNIST dataset variant. + transform : Optional[Callable], default=None + Transform applied to images. + """ + + def __init__( + self, + root_dir: str, + split: Literal["train", "val", "test"], + name: str = "organamnist", + transform: Optional[Callable[[Image.Image], torch.Tensor]] = None, + geom_transform_type: Optional[str] = "base", + ) -> None: + """Initialize the dataset.""" + assert split in [ + "train", + "val", + "test", + ], f"split {split} is not supported in dataset {name}." + if name is None: + raise ValueError("Variable name must be given to dataset") + + self.name = name + file_name = name + "_224.npz" + + self.geom_transform_type = geom_transform_type + + # Load the dataset + data = np.load(os.path.join(root_dir, file_name), mmap_mode="r") + self.images = data[f"{split}_images"] # [:5000] + self.labels = data[f"{split}_labels"] # [:5000] + + self.transform = ( + Compose([Resize(224), CenterCrop(224), ToTensor()]) + if transform is None + else transform + ) + print(f"DATASET LEN ::::::::::: {self.__len__()}") + print(f"DATASET LEN LABEL ::::::::::: {len(self.id2label)}") + + @property + def zero_shot_prompt_templates(self) -> list[str]: + """Return the zero-shot prompt templates.""" + return [ + "a histopathology slide showing {}.", + "histopathology image of {}.", + "pathology tissue showing {}.", + "presence of {} tissue on image.", + ] + + def __len__(self) -> int: + """Return the length of the dataset.""" + return int(self.images.shape[0]) + + def __getitem__(self, idx: int) -> Example: + """Return the idx'th data sample as an Example instance.""" + image = self.images[idx].astype(np.uint8) + image = Image.fromarray(image).convert("RGB") + + label = self.labels[idx].astype(int) + label = label.tolist() if len(label) > 1 else label.item() + + if self.transform is not None: + image = self.transform(image) + + return Example( + { + Modalities.RGB.name: image, + Modalities.RGB.target: label, + EXAMPLE_INDEX_KEY: idx, + "caption": "caption", + "image_path": self.images[idx], + "modality": "M", + } + ) + + @property + def id2label(self) -> Dict[int, str]: # noqa: PLR0911 + """Return the label mapping based on the dataset name.""" + if self.name == "pathmnist": + return { + 0: "adipose", + 1: "background", + 2: "debris", + 3: "lymphocytes", + 4: "mucus", + 5: "smooth muscle", + 6: "normal colon mucosa", + 7: "cancer-associated stroma", + 8: "colorectal adenocarcinoma epithelium", + } + if self.name == "chestmnist": + return { + 0: "atelectasis", + 1: "cardiomegaly", + 2: "effusion", + 3: "infiltration", + 4: "mass", + 5: "nodule", + 6: "pneumonia", + 7: "pneumothorax", + 8: "consolidation", + 9: "edema", + 10: "emphysema", + 11: "fibrosis", + 12: "pleural", + 13: "hernia", + } + if self.name == "dermamnist": + return { + 0: "actinic keratoses and intraepithelial carcinoma", + 1: "basal cell carcinoma", + 2: "benign keratosis-like lesions", + 3: "dermatofibroma", + 4: "melanoma", + 5: "melanocytic nevi", + 6: "vascular lesions", + } + if self.name == "octmnist": + return { + 0: "choroidal neovascularization", + 1: "diabetic macular edema", + 2: "drusen", + 3: "normal", + } + if self.name == "pneumoniamnist": + return { + 0: "normal", + 1: "pneumonia", + } + if self.name == "retinamnist": + return { + 0: "no apparent retinopathy", + 1: "mild NPDR, non-proliferative diabetic retinopathy", + 2: "moderate NPDR, non-proliferative diabetic retinopathy", + 3: "severe NPDR, non-proliferative diabetic retinopathy", + 4: "PDR, proliferative diabetic retinopathy", + } + if self.name == "breastmnist": + return { + 0: "malignant", + 1: "normal, benign", + } + if self.name == "bloodmnist": + return { + 0: "basophil", + 1: "eosinophil", + 2: "erythroblast", + 3: "immature granulocytes (myelocytes, metamyelocytes, " + "and promyelocytes)", + 4: "lymphocyte", + 5: "monocyte", + 6: "neutrophil", + 7: "platelet", + } + if self.name == "tissuemnist": + return { + 0: "Collecting Duct, Connecting Tubule", + 1: "Distal Convoluted Tubule", + 2: "Glomerular endothelial cells", + 3: "Interstitial endothelial cells", + 4: "Leukocytes", + 5: "Podocytes", + 6: "Proximal Tubule Segments", + 7: "Thick Ascending Limb", + } + + return { # organamnist, organsmnist, organcmnist + 0: "bladder", + 1: "femur-left", + 2: "femur-right", + 3: "heart", + 4: "kidney-left", + 5: "kidney-right", + 6: "liver", + 7: "lung-left", + 8: "lung-right", + 9: "pancreas", + 10: "spleen", + } diff --git a/openpmcvl/experiment/datasets/nck.py b/openpmcvl/experiment/datasets/nck.py new file mode 100644 index 0000000..32c24b6 --- /dev/null +++ b/openpmcvl/experiment/datasets/nck.py @@ -0,0 +1,128 @@ +"""NCK CRC Dataset.""" + +import os +from typing import Callable, Dict, Literal, Optional + +import torch +from datasets import load_dataset, load_from_disk +from mmlearn.conf import external_store +from mmlearn.constants import EXAMPLE_INDEX_KEY +from mmlearn.datasets.core import Modalities +from mmlearn.datasets.core.example import Example +from omegaconf import MISSING +from PIL import Image +from torch.utils.data import Dataset +from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor + + +@external_store(group="datasets", root_dir=os.getenv("NCK_CRC_ROOT_DIR", MISSING)) +class NckCrc(Dataset[Example]): + """NCK CRC dataset for colorectal cancer classification. + + Parameters + ---------- + root_dir : str + Path to the dataset directory or cache directory. + split : str, default='train' + Dataset split, one of 'train', 'train_nonorm', or 'validation'. + transform : Optional[Callable], default=None + Transform applied to images. + """ + + def __init__( + self, + root_dir: str, + split: Literal["train", "train_nonorm", "validation"], + transform: Optional[Callable[[Image.Image], torch.Tensor]] = None, + geom_transform_type: Optional[str] = "color", + ) -> None: + """Initialize the NCK CRC dataset.""" + self.geom_transform_type = geom_transform_type + assert split in ( + "train", + "train_nonorm", + "validation", + ), f"Invalid split: {split}" + + # Class mapping for labels + self.class_mapping = { + "ADI": 0, + "DEB": 1, + "LYM": 2, + "MUC": 3, # noqa: F821 + "MUS": 4, + "NORM": 5, + "STR": 6, + "TUM": 7, + } + + # Load cached dataset if it exists, otherwise download and cache it + cache_path = os.path.join(root_dir, f"cache/nck_crc_{split}.arrow") + if os.path.exists(cache_path): + print(f"Using cached dataset: {cache_path}") + dataset = load_from_disk(cache_path) + else: + os.makedirs(os.path.join(root_dir, "cache/"), exist_ok=True) + dataset = load_dataset( + "DykeF/NCTCRCHE100K", + cache_dir=os.path.join(root_dir, "scratch/"), + split=split, + ) + dataset = dataset.filter( + lambda row: row["label"] != "BACK" + ) # Exclude "BACK" label + dataset.save_to_disk(cache_path) + + self.data = dataset + + self.transform = ( + Compose([Resize(224), CenterCrop(224), ToTensor()]) + if transform is None + else transform + ) + print(f"DATASET LEN {split} ::::::::::: {self.__len__()}") + + def __getitem__(self, idx: int) -> Example: + """Return the idx'th data sample as an Example instance.""" + entry = self.data[idx] + image = entry["image"] + label = self.class_mapping[entry["label"]] + + if self.transform is not None: + image = self.transform(image) + + return Example( + { + Modalities.RGB.name: image, + Modalities.RGB.target: label, + EXAMPLE_INDEX_KEY: idx, + } + ) + + def __len__(self) -> int: + """Return the length of the dataset.""" + return len(self.data) + + @property + def id2label(self) -> Dict[int, str]: + """Return the label mapping for the NCK CRC dataset.""" + return { + 0: "adipose", + 1: "debris", + 2: "lymphocytes", + 3: "mucus", + 4: "smooth muscle", + 5: "normal colon mucosa", + 6: "cancer-associated stroma", + 7: "colorectal adenocarcinoma epithelium", + } + + @property + def zero_shot_prompt_templates(self) -> list[str]: + """Return the zero-shot prompt templates.""" + return [ + "a histopathology slide showing {}", + "histopathology image of {}", + "pathology tissue showing {}", + "presence of {} tissue on image", + ] diff --git a/openpmcvl/experiment/datasets/pad_ufes_20.py b/openpmcvl/experiment/datasets/pad_ufes_20.py new file mode 100644 index 0000000..3585f08 --- /dev/null +++ b/openpmcvl/experiment/datasets/pad_ufes_20.py @@ -0,0 +1,131 @@ +"""PadUfes20 Dataset.""" + +import os +import pickle +from typing import Callable, Dict, Literal, Optional + +import pandas as pd +import torch +from mmlearn.conf import external_store +from mmlearn.constants import EXAMPLE_INDEX_KEY +from mmlearn.datasets.core import Modalities +from mmlearn.datasets.core.example import Example +from omegaconf import MISSING +from PIL import Image +from torch.utils.data import Dataset +from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor + + +@external_store(group="datasets", root_dir=os.getenv("PADUFES_ROOT_DIR", MISSING)) +class PadUfes20(Dataset[Example]): + """PadUfes20 dataset for classification tasks. + + Parameters + ---------- + root_dir : str + Path to the dataset directory containing images and metadata CSV. + split : {'train', 'test'} + Dataset split, must be one of ["train", "test"]. + transform : Optional[Callable], default=None + Transform applied to images. + """ + + def __init__( + self, + root_dir: str, + split: Literal["train", "test"], + transform: Optional[Callable[[Image.Image], torch.Tensor]] = None, + geom_transform_type: Optional[str] = "color", + ) -> None: + """Initialize the dataset.""" + assert split in ["train", "test"], f"split {split} is not supported in dataset." + + self.geom_transform_type = geom_transform_type + self.root_dir = root_dir + self.split = split + + # Load cached data if available + cache_path = f"cache/PadUfes20_{split}.pkl" + if os.path.exists(cache_path): + print(f"!!! Using cached dataset for {split}") + with open(cache_path, "rb") as f: + self.metadata = pickle.load(f) + else: + os.makedirs("cache/", exist_ok=True) + self.metadata = self._load_and_process_metadata() + with open(cache_path, "wb") as f: + pickle.dump(self.metadata.to_dict("records"), f) + + self.transform = ( + Compose([Resize(224), CenterCrop(224), ToTensor()]) + if transform is None + else transform + ) + print(f"DATASET LEN ::::::::::: {self.__len__()}") + + def _load_and_process_metadata(self) -> pd.DataFrame: + """Load and process metadata from CSV.""" + df = pd.read_csv(os.path.join(self.root_dir, "metadata.csv")) + df = df[["img_id", "diagnostic"]] + df["label"] = df["diagnostic"].apply(self._build_label) + df["path"] = df["img_id"].apply( + lambda imgid: os.path.join(self.root_dir, "Dataset", imgid) + ) + df.drop(columns=["img_id", "diagnostic"], inplace=True) + df.reset_index(drop=True, inplace=True) + + # Split into train and test + dataset = {} + dataset["test"] = df.sample(frac=0.2) + dataset["train"] = df.drop(dataset["test"].index) + return dataset[self.split] + + def _build_label(self, str_label: str) -> int: + """Convert diagnostic string label to integer label.""" + classes = {"BCC": 0, "MEL": 1, "SCC": 2, "ACK": 3, "NEV": 4, "SEK": 5} + return classes[str_label] + + def __getitem__(self, idx: int) -> Example: + """Return the idx'th data sample as an Example instance.""" + entry = self.metadata[idx] + image_path = entry["path"] + + with Image.open(image_path) as img: + image = img.convert("RGB") + + if self.transform is not None: + image = self.transform(image) + + return Example( + { + Modalities.RGB.name: image, + Modalities.RGB.target: int(entry["label"]), + EXAMPLE_INDEX_KEY: idx, + } + ) + + def __len__(self) -> int: + """Return the length of the dataset.""" + return len(self.metadata) + + @property + def id2label(self) -> Dict[int, str]: + """Return the label mapping for the PadUfes20 dataset.""" + return { + 0: "Basal Cell Carcinoma", + 1: "Melanoma", + 2: "Squamous Cell Carcinoma", + 3: "Actinic Keratosis", + 4: "Nevus", + 5: "Seborrheic Keratosis", + } + + @property + def zero_shot_prompt_templates(self) -> list[str]: + """Return the zero-shot prompt templates.""" + return [ + "a histopathology slide showing {}.", + "histopathology image of {}.", + "pathology tissue showing {}.", + "presence of {} tissue on image.", + ] diff --git a/openpmcvl/experiment/datasets/pcam.py b/openpmcvl/experiment/datasets/pcam.py new file mode 100644 index 0000000..296b1eb --- /dev/null +++ b/openpmcvl/experiment/datasets/pcam.py @@ -0,0 +1,97 @@ +"""PCAM Dataset.""" + +import os +import pickle +from typing import Callable, Dict, Optional + +import torch +from datasets import load_dataset +from mmlearn.conf import external_store +from mmlearn.constants import EXAMPLE_INDEX_KEY +from mmlearn.datasets.core import Modalities +from mmlearn.datasets.core.example import Example +from omegaconf import MISSING +from PIL import Image +from torch.utils.data import Dataset +from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor + + +@external_store(group="datasets", root_dir=os.getenv("PCAM_ROOT_DIR", MISSING)) +class PCAM(Dataset[Example]): + """PCAM dataset for classification tasks. + + Parameters + ---------- + root_dir : str + Path to the dataset directory or cache directory. + split : str + Dataset split. + transform : Optional[Callable], default=None + Transform applied to images. + """ + + def __init__( + self, + root_dir: str, + split: str = "test", + transform: Optional[Callable[[Image.Image], torch.Tensor]] = None, + ) -> None: + """Initialize the PCAM dataset.""" + cache_path = os.path.join(root_dir, f"cache/pcam_{split}.pkl") + + if os.path.exists(cache_path): + print("!!!Using cached dataset") + with open(cache_path, "rb") as f: + self.data = pickle.load(f) + else: + os.makedirs(os.path.join(root_dir, "cache/"), exist_ok=True) + dataset = load_dataset( + "1aurent/PatchCamelyon", cache_dir=os.path.join(root_dir, "scratch/") + )[split] + + self.data = dataset + with open(cache_path, "wb") as f: + pickle.dump(self.data, f) + + self.transform = ( + Compose([Resize(224), CenterCrop(224), ToTensor()]) + if transform is None + else transform + ) + print(f"DATASET LEN ::::::::::: {self.__len__()}") + + def __getitem__(self, idx: int) -> Example: + """Return the idx'th data sample as an Example instance.""" + entry = self.data[idx] + image = entry["image"].convert("RGB") + label_idx = int(entry["label"]) + + if self.transform is not None: + image = self.transform(image) + + return Example( + { + Modalities.RGB.name: image, + Modalities.RGB.target: label_idx, + EXAMPLE_INDEX_KEY: idx, + } + ) + + def __len__(self) -> int: + """Return the length of the dataset.""" + return len(self.data) + + @property + def id2label(self) -> Dict[int, str]: + """Return the mapping of labels for the PCAM dataset.""" + return {0: "lymph node", 1: "lymph node containing metastatic tumor tissue"} + + @property + def zero_shot_prompt_templates(self) -> list[str]: + """Return the zero-shot prompt templates.""" + return [ + "a histopathology slide showing {}", + "histopathology image of {}", + "pathology tissue showing {}", + "presence of {} tissue on image", + ] diff --git a/openpmcvl/experiment/datasets/sicap.py b/openpmcvl/experiment/datasets/sicap.py new file mode 100644 index 0000000..9146361 --- /dev/null +++ b/openpmcvl/experiment/datasets/sicap.py @@ -0,0 +1,119 @@ +"""Sicap Dataset.""" + +import os +from typing import Callable, Dict, Literal, Optional + +import pandas as pd +import torch +from mmlearn.conf import external_store +from mmlearn.constants import EXAMPLE_INDEX_KEY +from mmlearn.datasets.core import Modalities +from mmlearn.datasets.core.example import Example +from omegaconf import MISSING +from PIL import Image +from torch.utils.data import Dataset +from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor + + +@external_store(group="datasets", root_dir=os.getenv("SICAP_ROOT_DIR", MISSING)) +class SICAP(Dataset[Example]): + """SICAP dataset for zero-shot classification. + + Parameters + ---------- + root_dir : str + Path to the dataset directory containing images and metadata CSV. + split : {'train', 'test'} + Dataset split, must be one of ["train", "test"]. + transform : Optional[Callable], default=None + Transform applied to images. + tokenizer : Optional[Callable], default=None + Function to generate textual embeddings. + """ + + def __init__( + self, + root_dir: str, + split: Literal["train", "test"], + image_dir: str = "images", + transform: Optional[Callable[[Image.Image], torch.Tensor]] = None, + ) -> None: + """Initialize the dataset.""" + assert split in ["train", "test"], f"split {split} is not supported in dataset." + image_dir = os.path.join(root_dir, image_dir) + + if split == "train": + csv_file = os.path.join(root_dir, "partition/Test", "Train.xlsx") + self.data = pd.read_excel(csv_file) + elif split == "test": + csv_file = os.path.join(root_dir, "partition/Test", "Test.xlsx") + self.data = pd.read_excel(csv_file) + + # Drop all columns except image_name and label columns + label_columns = ["NC", "G3", "G4", "G5"] + self.data = self.data[["image_name"] + label_columns] + + # Get the index of the maximum label value for each row + self.data["labels"] = self.data[label_columns].idxmax(axis=1) + + # Replace label column values with categorical values + self.cat_to_num_map = { + "NC": 0, + "G3": 1, + "G4": 2, + "G5": 3, + } + self.data["labels"] = self.data["labels"].map(self.cat_to_num_map) + + self.image_paths = self.data["image_name"].values + self.labels = self.data["labels"].values + self.image_dir = image_dir + self.transform = ( + transform + if transform is not None + else Compose([Resize(224), CenterCrop(224), ToTensor()]) + ) + print(f"DATASET LEN {split} ::::::::::: {self.__len__()}") + + @property + def id2label(self) -> Dict[int, str]: + """Return the label mapping.""" + return { + 0: "benign glands", + 1: "atrophic dense glands", + 2: "cribriform ill-formed fused papillary patterns", + 3: "isolated nest cells without lumen roseting patterns", + } + + @property + def zero_shot_prompt_templates(self) -> list[str]: + """Return the zero-shot prompt templates.""" + return [ + "a histopathology slide showing {}.", + "histopathology image of {}.", + "pathology tissue showing {}.", + "presence of {} tissue on image.", + ] + + def __len__(self) -> int: + """Return the length of the dataset.""" + return len(self.data) + + def __getitem__(self, idx: int) -> Example: + """Return the idx'th data sample as an Example instance.""" + image_path = os.path.join(self.image_dir, self.image_paths[idx]) + with Image.open(image_path) as img: + image = img.convert("RGB") + + label_index = self.labels[idx] + + if self.transform is not None: + image = self.transform(image) + + return Example( + { + Modalities.RGB.name: image, + Modalities.RGB.target: label_index, + EXAMPLE_INDEX_KEY: idx, + } + ) diff --git a/openpmcvl/experiment/modules/contrastive_pretraining_ppr.py b/openpmcvl/experiment/modules/contrastive_pretraining_ppr.py index 86d8d10..05c8eeb 100644 --- a/openpmcvl/experiment/modules/contrastive_pretraining_ppr.py +++ b/openpmcvl/experiment/modules/contrastive_pretraining_ppr.py @@ -18,6 +18,7 @@ from mmlearn.modules.losses import CLIPLoss from mmlearn.tasks.hooks import EvaluationHooks from torch import nn +from torch.optim.optimizer import Optimizer _unsupported_modality_error = ( @@ -142,7 +143,7 @@ def __init__( # noqa: PLR0912, PLR0915 Dict[str, Union[nn.Module, Dict[str, nn.Module]]] ] = None, modality_module_mapping: Optional[Dict[str, ModuleKeySpec]] = None, - optimizer: Optional[partial[torch.optim.Optimizer]] = None, # type: ignore[name-defined] + optimizer: Optional[partial[Optimizer]] = None, lr_scheduler: Optional[ Union[ Dict[str, Union[partial[torch.optim.lr_scheduler.LRScheduler], Any]], @@ -540,7 +541,7 @@ def configure_optimizers(self) -> OptimizerLRScheduler: # noqa: PLR0912 ] optimizer = self.optimizer(parameters) - if not isinstance(optimizer, torch.optim.Optimizer): # type: ignore[attr-defined] + if not isinstance(optimizer, Optimizer): raise TypeError( "Expected optimizer to be an instance of `torch.optim.Optimizer`, " f"but got {type(optimizer)}.", diff --git a/openpmcvl/experiment/modules/encoders.py b/openpmcvl/experiment/modules/encoders.py index 44d3e92..2756c42 100644 --- a/openpmcvl/experiment/modules/encoders.py +++ b/openpmcvl/experiment/modules/encoders.py @@ -13,6 +13,29 @@ from torch import nn +# Released Open-PMC-18M CLIP checkpoint on the Hugging Face Hub, used as the default +# checkpoint for evaluation. See https://huggingface.co/vector-institute/open-pmc-18m-clip +DEFAULT_EVAL_CHECKPOINT = "hf-hub:vector-institute/open-pmc-18m-clip" +_DEFAULT_OPENCLIP_WEIGHTS_FILE = "open_clip_pytorch_model.bin" + + +def _resolve_checkpoint_path(checkpoint_path: str) -> str: + """Resolve a checkpoint reference to a local file path. + + Accepts either a local filesystem path or a Hugging Face Hub reference of the form + ``hf-hub:/`` (optionally ``hf-hub://``). Hub + references are downloaded and the local cached path is returned; when no filename is + given, the open_clip weights file (``open_clip_pytorch_model.bin``) is used. + """ + hf_prefix = "hf-hub:" + if checkpoint_path.startswith(hf_prefix): + parts = checkpoint_path[len(hf_prefix) :].split("/") + repo_id = "/".join(parts[:2]) + filename = "/".join(parts[2:]) or _DEFAULT_OPENCLIP_WEIGHTS_FILE + return str(hf_hub_download(repo_id, filename)) + return checkpoint_path + + @external_store( group="modules/encoders", provider="openpmcvl", @@ -57,6 +80,7 @@ def __init__( normalize: bool = False, clip_ckpt: Optional[str] = None, model_config_kwargs: Optional[Dict[str, Any]] = None, + checkpoint_path: Optional[str] = None, ) -> None: """Initialize the model.""" super().__init__() @@ -68,10 +92,14 @@ def __init__( config = json.load(f) model_cfg = config["model_cfg"] + # When loading a full local checkpoint, the HF/timm backbone weights would + # just be overwritten, so skip loading them. This also avoids the noisy + # transformers "weights not used" warning for BERT's cls.*/pooler heads. + load_backbone_pretrained = not bool(checkpoint_path) # load pretrained weights of the text encoder - model_cfg["text_cfg"]["hf_model_pretrained"] = True + model_cfg["text_cfg"]["hf_model_pretrained"] = load_backbone_pretrained # load pretrained weights of the vision encoder - model_cfg["vision_cfg"]["timm_model_pretrained"] = True + model_cfg["vision_cfg"]["timm_model_pretrained"] = load_backbone_pretrained # create model if model_config_kwargs is None: @@ -79,7 +107,10 @@ def __init__( model = CustomTextCLIP(**model_cfg, **model_config_kwargs) # load checkpoint file - if pretrained: + if checkpoint_path: + # load a local open_clip-format checkpoint (e.g. Open-PMC-18M weights) + self._load_checkpoint(model, checkpoint_path) + elif pretrained: cached_file = hf_hub_download( model_name_or_path, "open_clip_pytorch_model.bin", @@ -114,7 +145,8 @@ def _load_checkpoint( checkpoint_path: str, strict: bool = True, ) -> Any: - checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=True) + checkpoint_path = _resolve_checkpoint_path(checkpoint_path) + checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) if isinstance(checkpoint, dict) and "state_dict" in checkpoint: state_dict = checkpoint["state_dict"] else: @@ -215,6 +247,7 @@ def __init__( modality: str = "rgb", normalize: bool = False, model_config_kwargs: Optional[Dict[str, Any]] = None, + checkpoint_path: Optional[str] = None, ) -> None: """Initialize the model.""" super().__init__() @@ -226,10 +259,14 @@ def __init__( config = json.load(f) model_cfg = config["model_cfg"] + # When loading a full local checkpoint, the HF/timm backbone weights would + # just be overwritten, so skip loading them. This also avoids the noisy + # transformers "weights not used" warning for BERT's cls.*/pooler heads. + load_backbone_pretrained = not bool(checkpoint_path) # load pretrained weights of the text encoder - model_cfg["text_cfg"]["hf_model_pretrained"] = True + model_cfg["text_cfg"]["hf_model_pretrained"] = load_backbone_pretrained # load pretrained weights of the vision encoder - model_cfg["vision_cfg"]["timm_model_pretrained"] = True + model_cfg["vision_cfg"]["timm_model_pretrained"] = load_backbone_pretrained # create model if model_config_kwargs is None: @@ -237,7 +274,10 @@ def __init__( model = CustomTextCLIP(**model_cfg, **model_config_kwargs) # load checkpoint file - if pretrained: + if checkpoint_path: + # load a local open_clip-format checkpoint (e.g. Open-PMC-18M weights) + self._load_checkpoint(model, checkpoint_path) + elif pretrained: cached_file = hf_hub_download( model_name_or_path, "open_clip_pytorch_model.bin", @@ -260,7 +300,8 @@ def _load_checkpoint( checkpoint_path: str, strict: bool = True, ) -> Any: - checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=True) + checkpoint_path = _resolve_checkpoint_path(checkpoint_path) + checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) if isinstance(checkpoint, dict) and "state_dict" in checkpoint: state_dict = checkpoint["state_dict"] else: diff --git a/openpmcvl/experiment/modules/scheduler.py b/openpmcvl/experiment/modules/scheduler.py index 6b2ec35..bca041a 100644 --- a/openpmcvl/experiment/modules/scheduler.py +++ b/openpmcvl/experiment/modules/scheduler.py @@ -30,7 +30,7 @@ def get_lr(self) -> List[float]: if self.last_epoch < self.warmup_length: return [ - base_lr * (self.last_epoch + 1) / self.warmup_length + float(base_lr) * (self.last_epoch + 1) / self.warmup_length for base_lr, group in zip(self.base_lrs, self.optimizer.param_groups) ] @@ -38,7 +38,7 @@ def get_lr(self) -> List[float]: total_steps = self.t_max - self.warmup_length return [ self.eta_min - + (base_lr - self.eta_min) + + (float(base_lr) - self.eta_min) * (1 + math.cos((step) * math.pi / total_steps)) / 2 for base_lr, group in zip(self.base_lrs, self.optimizer.param_groups) diff --git a/openpmcvl/foundation/src/parser/parse_oa.py b/openpmcvl/foundation/src/parser/parse_oa.py index 1922b0c..43064f1 100644 --- a/openpmcvl/foundation/src/parser/parse_oa.py +++ b/openpmcvl/foundation/src/parser/parse_oa.py @@ -120,7 +120,7 @@ def parse_xml(args: Namespace, xml_path: str) -> List[Dict[str, str]]: figs = soup.find_all(name="fig") for fig in figs: if "id" in fig.attrs: - media_id = fig.attrs["id"] + media_id = str(fig.attrs["id"]) else: continue diff --git a/pyproject.toml b/pyproject.toml index 6954641..c7042eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,9 +66,17 @@ implicit_reexport = false strict_equality = true extra_checks = true +# The `openpmcvl/granular` subpackage (YOLO subfigure models, notebooks) predates the +# type-checking setup and is not type-checked here; excluded to keep `code checks` green. +[[tool.mypy.overrides]] +module = "openpmcvl.granular.*" +ignore_errors = true + [tool.ruff] include = ["*.py", "pyproject.toml", "*.ipynb"] line-length = 88 +# openpmcvl/granular predates these checks and is excluded to keep `code checks` green. +extend-exclude = ["openpmcvl/granular"] [tool.ruff.format] quote-style = "double" @@ -132,6 +140,12 @@ norecursedirs = ["working","openpmcvl"] [tool.typos.default.extend-words] nd = "nd" +muc = "muc" + +# The `openpmcvl/granular` subpackage (YOLO models, notebooks) predates the spell-check +# setup and uses domain abbreviations (e.g. `thre`); excluded to keep `code checks` green. +[tool.typos.files] +extend-exclude = ["openpmcvl/granular/"] [build-system] requires = ["poetry-core>=1.0.0"]