Add eFP/ePlant gene ID validation, microarray probeset support, and m…

.gitignore

@@ -156,3 +156,12 @@ config/databases/tomato_ils.sql
 
 # Archive test data — large DB dumps, exports, and CSVs; not for version control
 archive/test_data/
+
+# Scratch output from scripts/check_efp_experiment_links.py
+scripts/results/
+
+# Real per-database sample dumps used only as build_combined_master_json.py's
+# input (schema/regex verification against real data) -- not read at runtime
+# by the API or by any test, and large/noisy enough (193 files, ~52k lines)
+# to not belong in the PR diff. Keep locally to rerun the build script.
+api/random_rows_json/

api/models/efp_dynamic.py

@@ -1,6 +1,4 @@
 """
-Reena Obmina | BCB330 Project 2025-2026 | University of Toronto
-
 Dynamic SQLAlchemy model generation for all eFP databases.
 
 At import time, one ORM model class is generated per database entry in

api/models/efp_schemas.py

@@ -1,6 +1,4 @@
 """
-Reena Obmina | BCB330 Project 2025-2026 | University of Toronto
-
 Schema definitions for all eFP databases that expose a sample_data table.
 
 Every database shares the same three-column structure:
@@ -105,7 +103,7 @@ def _schema(species: str, charset: str = "latin1") -> DatabaseSpec:
     ("grape_developmental", "grape"),
     ("guard_cell", "guard cell"),
     ("gynoecium", "gynoecium"),
-    ("heterodera_schachtii", "heterodera"),
+    ("heterodera_schachtii", "arabidopsis"),
     ("hnahal", "hnahal"),
     ("human_body_map_2", "human"),
     ("human_developmental", "human"),
@@ -181,7 +179,7 @@ def _schema(species: str, charset: str = "latin1") -> DatabaseSpec:
     ("root_Schaefer_lab", "root Schaefer lab"),
     ("rpatel", "rpatel"),
     ("seed_db", "seed db"),
-    ("seedcoat", "oat"),
+    ("seedcoat", "arabidopsis seedcoat"),
     ("selaginella", "selaginella"),
     ("shoot_apex", "arabidopsis"),
     ("silique", "arabidopsis"),

api/resources/gene_density.py

@@ -1,6 +1,4 @@
 """
-Reena Obmina | UTEA Project 2026 | University of Toronto
-
 Gene density endpoint for the BAR API.
 
 Returns per-bin gene density across all Arabidopsis thaliana chromosomes for a

api/resources/gene_expression.py

@@ -1,26 +1,12 @@
-"""
-Reena Obmina | BCB330 Project 2025-2026 | University of Toronto
-
-REST endpoint for gene expression queries across all eFP databases.
-
-Routes: GET /gene_expression/expression/<database>/<gene_id>
-
-All gene IDs are validated by species before reaching the query layer.
-Probeset conversion is applied automatically for microarray databases.
-"""
 from flask_restx import Namespace, Resource
 from markupsafe import escape
 
 from api.services.efp_data import query_efp_database_dynamic
 from api.utils.bar_utils import BARUtils
 from api.utils.gene_id_utils import (
-    CROSS_SPECIES_DATABASES,
     DATABASE_SPECIES,
-    PROBESET_DATABASES,
-    convert_gene_to_probeset,
-    is_probeset_id,
-    normalize_gene_id,
-    validate_gene_id,
+    GeneIdUtils,
+    DATABASE_EFP_PROJECT,
 )
 
 gene_expression = Namespace(
@@ -46,38 +32,24 @@
 )
 class GeneExpression(Resource):
     def get(self, database, gene_id):
-        """Retrieve expression values for a gene from a given eFP database.
-        """
+        """Retrieve expression values for a gene from a given eFP database."""
         database = str(escape(database))
         gene_id = str(escape(gene_id))
 
-        # 1. Resolve database species and expected input species.
-        #    Cross-species databases (e.g. phelipanche) accept an Arabidopsis AGI
-        #    even though the database itself belongs to a different species.
         species = DATABASE_SPECIES.get(database)
         if species is None:
             return BARUtils.error_exit(f"Unknown database '{database}'"), 400
-        input_species = CROSS_SPECIES_DATABASES.get(database, species)
 
-        # 2. If the caller already supplied a probeset ID, use it directly
-        if is_probeset_id(gene_id):
+        if BARUtils.is_injection_attempt(gene_id):
+            return BARUtils.error_exit(f"Invalid gene ID for {database}: '{gene_id}'"), 400
+
+        if GeneIdUtils.is_probeset_id(gene_id):
             query_id = gene_id
         else:
-            # 3. Validate gene ID format against the expected input species regex
-            if not validate_gene_id(gene_id, input_species):
-                return BARUtils.error_exit(f"Invalid {input_species} gene ID: '{gene_id}'"), 400
-
-            # 4. Normalise (e.g. strip maize transcript suffix _T##)
-            gene_id = normalize_gene_id(gene_id, species)
-
-            # 5. Microarray / non-direct databases need gene ID -> probeset conversion
-            if database in PROBESET_DATABASES:
-                probeset, err = convert_gene_to_probeset(gene_id, species, database)
-                if err:
-                    return BARUtils.error_exit(err), 404
-                query_id = probeset
-            else:
-                query_id = gene_id
+            if not GeneIdUtils.validate_gene_for_database(gene_id, database):
+                label = DATABASE_EFP_PROJECT.get(database) or species or database
+                return BARUtils.error_exit(f"Invalid gene ID for {label}: '{gene_id}'"), 400
+            query_id = GeneIdUtils.normalize_gene_id(gene_id, species)
 
         result = query_efp_database_dynamic(database, query_id)
 

api/resources/microarray_gene_expression.py

@@ -4,9 +4,9 @@
 from api.models.annotations_lookup import AtAgiLookup
 from api.models.efp_dynamic import SIMPLE_EFP_SAMPLE_MODELS
 from api.utils.bar_utils import BARUtils
+from api.utils.master_data_utils import load_combined_master
 from api.utils.world_efp_utils import WorldeFPUtils
 from sqlalchemy import func
-import json
 
 # Pull the dynamic model so this resource stays in sync with the schema catalog
 EcotypesSampleData = SIMPLE_EFP_SAMPLE_MODELS["arabidopsis_ecotypes"]
@@ -64,163 +64,44 @@ def get(self, species="", gene_id=""):
             return BARUtils.error_exit("There are no data found for the given gene")
 
 
-# Endpoint made by Reena
 @microarray_gene_expression.route("/<string:species>/databases")
 class GetDatabases(Resource):
     @microarray_gene_expression.param("species", _in="path", default="arabidopsis")
     def get(self, species=""):
-        """This endpoint returns available database and view mappings for a given species"""
-        species = escape(species)
+        """This endpoint returns the databases and views available for a given
+        species, and how many of each -- sourced live from combined_master.json
+        (data/efp_info/combined_master.json) instead of a hardcoded mapping, so
+        it can't drift out of sync with the actual database/frontend catalog.
+        """
+        species = str(escape(species)).lower()
+
+        master = load_combined_master()
+        if species not in master["species"]:
+            return BARUtils.error_exit("Invalid species")
 
-        species_databases = {
-            "actinidia": {
-                "Bud_Development": "actinidia_bud_development",
-                "Flower_Fruit_Development": "actinidia_flower_fruit_development",
-                "Postharvest": "actinidia_postharvest",
-                "Vegetative_Growth": "actinidia_vegetative_growth",
-            },
-            "arabidopsis": {
-                "Abiotic_Stress": "atgenexp_stress",
-                "Abiotic_Stress_II": "atgenexp_stress",
-                "Biotic_Stress": "atgenexp_pathogen",
-                "Biotic_Stress_II": "atgenexp_pathogen",
-                "Chemical": "atgenexp_hormone",
-                "DNA_Damage": "dna_damage",
-                "Development_RMA": "atgenexp",
-                "Developmental_Map": "atgenexp_plus",
-                "Developmental_Mutants": "atgenexp_plus",
-                "Embryo": "embryo",
-                "Germination": "germination",
-                "Guard_Cell": "guard_cell",
-                "Gynoecium": "gynoecium",
-                "Hormone": "atgenexp_hormone",
-                "Klepikova_Atlas": "klepikova",
-                "Lateral_Root_Initiation": "lateral_root_initiation",
-                "Light_Series": "light_series",
-                "Natural_Variation": "arabidopsis_ecotypes",
-                "Regeneration": "meristem_db",
-                "Root": "root",
-                "Root_II": "root",
-                "Seed": "seed_db",
-                "Shoot_Apex": "shoot_apex",
-                "Silique": "silique",
-                "Single_Cell": "single_cell",
-                "Tissue_Specific": "atgenexp_plus",
-            },
-            "arabidopsis seedcoat": {"Seed_Coat": "seedcoat"},
-            "arachis": {"Arachis_Atlas": "arachis"},
-            "barley": {"barley_mas": "barley_mas", "barley_rma": "barley_rma"},
-            "brachypodium": {
-                "Brachypodium_Atlas": "brachypodium",
-                "Brachypodium_Grains": "brachypodium_grains",
-                "Brachypodium_Spikes": "brachypodium_Bd21",
-                "Photo_Thermocycle": "brachypodium_photo_thermocycle",
-            },
-            "brassica rapa": {"Embryogenesis": "brassica_rapa"},
-            "cacao ccn": {
-                "Developmental_Atlas": "cacao_developmental_atlas",
-                "Drought_Diurnal_Atlas": "cacao_drought_diurnal_atlas",
-            },
-            "cacao sca": {
-                "Developmental_Atlas": "cacao_developmental_atlas_sca",
-                "Drought_Diurnal_Atlas": "cacao_drought_diurnal_atlas_sca",
-                "Meristem_Atlas": "cacao_meristem_atlas_sca",
-                "Seed_Atlas": "cacao_seed_atlas_sca",
-            },
-            "cacao tc": {"Cacao_Infection": "cacao_infection", "Cacao_Leaf": "cacao_leaf"},
-            "camelina": {"Developmental_Atlas_FPKM": "camelina", "Developmental_Atlas_TPM": "camelina_tpm"},
-            "cannabis": {"Cannabis_Atlas": "cannabis"},
-            "canola": {"Canola_Seed": "canola_seed"},
-            "eutrema": {"Eutrema": "thellungiella_db"},
-            "grape": {"grape_developmental": "grape_developmental"},
-            "kalanchoe": {"Light_Response": "kalanchoe"},
-            "little millet": {"Life_Cycle": "little_millet"},
-            "lupin": {
-                "LCM_Leaf": "lupin_lcm_leaf",
-                "LCM_Pod": "lupin_lcm_pod",
-                "LCM_Stem": "lupin_lcm_stem",
-                "Whole_Plant": "lupin_whole_plant",
-            },
-            "maize": {
-                "Downs_et_al_Atlas": "maize_gdowns",
-                "Early_Seed": "maize_early_seed",
-                "Embryonic_Leaf_Development": "maize_embryonic_leaf_development",
-                "Hoopes_et_al_Atlas": "maize_buell_lab",
-                "Hoopes_et_al_Stress": "maize_buell_lab",
-                "Maize_Kernel": "maize_early_seed",
-                "Maize_Root": "maize_root",
-                "Sekhon_et_al_Atlas": "maize_RMA_linear",
-                "Tassel_and_Ear_Primordia": "maize_ears",
-                "maize_iplant": "maize_iplant",
-                "maize_leaf_gradient": "maize_leaf_gradient",
-                "maize_rice_comparison": "maize_rice_comparison",
-            },
-            "mangosteen": {
-                "Aril_vs_Rind": "mangosteen_aril_vs_rind",
-                "Callus": "mangosteen_callus",
-                "Diseased_vs_Normal": "mangosteen_diseased_vs_normal",
-                "Fruit_Ripening": "mangosteen_fruit_ripening",
-                "Seed_Development": "mangosteen_seed_development",
-                "Seed_Germination": "mangosteen_seed_germination",
-            },
-            "medicago": {
-                "medicago_mas": "medicago_mas",
-                "medicago_rma": "medicago_rma",
-                "medicago_seed": "medicago_seed",
-            },
-            "poplar": {"Poplar": "poplar", "PoplarTreatment": "poplar"},
-            "potato": {"Potato_Developmental": "potato_dev", "Potato_Stress": "potato_stress"},
-            "rice": {
-                "rice_drought_heat_stress": "rice_drought_heat_stress",
-                "rice_leaf_gradient": "rice_leaf_gradient",
-                "rice_maize_comparison": "rice_maize_comparison",
-                "rice_mas": "rice_mas",
-                "rice_rma": "rice_rma",
-                "riceanoxia_mas": "rice_mas",
-                "riceanoxia_rma": "rice_rma",
-                "ricestigma_mas": "rice_mas",
-                "ricestigma_rma": "rice_rma",
-                "ricestress_mas": "rice_mas",
-                "ricestress_rma": "rice_rma",
-            },
-            "soybean": {
-                "soybean": "soybean",
-                "soybean_embryonic_development": "soybean_embryonic_development",
-                "soybean_heart_cotyledon_globular": "soybean_heart_cotyledon_globular",
-                "soybean_senescence": "soybean_senescence",
-                "soybean_severin": "soybean_severin",
-            },
-            "strawberry": {
-                "Developmental_Map_Strawberry_Flower_and_Fruit": "strawberry",
-                "Strawberry_Green_vs_White_Stage": "strawberry",
-            },
-            "tomato": {
-                "ILs_Leaf_Chitwood_et_al": "tomato_ils",
-                "ILs_Root_Tip_Brady_Lab": "tomato_ils2",
-                "M82_S_pennellii_Atlases_Koenig_et_al": "tomato_s_pennellii",
-                "Rose_Lab_Atlas": "tomato",
-                "Rose_Lab_Atlas_Renormalized": "tomato_renormalized",
-                "SEED_Lab_Angers": "tomato_seed",
-                "Shade_Mutants": "tomato_shade_mutants",
-                "Shade_Timecourse_WT": "tomato_shade_timecourse",
-                "Tomato_Meristem": "tomato_meristem",
-            },
-            "triticale": {"triticale": "triticale", "triticale_mas": "triticale_mas"},
-            "wheat": {
-                "Developmental_Atlas": "wheat",
-                "Wheat_Abiotic_Stress": "wheat_abiotic_stress",
-                "Wheat_Embryogenesis": "wheat_embryogenesis",
-                "Wheat_Meiosis": "wheat_meiosis",
-            },
+        databases = {
+            db_name: db_info
+            for db_name, db_info in master["databases"].items()
+            if db_info["species"] == species
         }
 
-        if species not in species_databases:
-            return BARUtils.error_exit("Invalid species")
+        # A database can be exposed under different view display names by
+        # different frontend instances (efp vs eplant); collapse to a single
+        # view_name -> database_name mapping like the endpoint has always returned.
+        views = {}
+        for db_name, db_info in databases.items():
+            for used_by in db_info["used_by"]:
+                view_key = used_by["view"].replace(" ", "_")
+                views[view_key] = db_name
 
-        return BARUtils.success_exit({"species": species, "databases": species_databases[species]})
+        return BARUtils.success_exit({
+            "species": species,
+            "num_databases": len(databases),
+            "num_views": len(views),
+            "databases": views,
+        })
 
 
-# Endpoint made by Reena
 @microarray_gene_expression.route("/<string:species>/<string:view>/samples")
 class GetSamples1(Resource):
     """This endpoint returns control and sample group mappings for a given species and view (or all views)"""
@@ -229,32 +110,42 @@ class GetSamples1(Resource):
     @microarray_gene_expression.param("view", _in="path", default="Abiotic_Stress")
     def get(self, species="", view=""):
         """This endpoint returns control and sample group mappings for a given species and view (or all views)"""
-        species = escape(species.lower())
-        view = escape(view)
-
-        try:
-            with open("data/efp_info/efp_species_view_info_typed.json") as f:
-                all_species_data = json.load(f)
-        except Exception as e:
-            return BARUtils.error_exit(f"Data file missing or invalid: {e}")
+        species = str(escape(species)).lower()
+        view = str(escape(view))
 
-        if species not in all_species_data:
+        master = load_combined_master()
+        if species not in master["species"]:
             return BARUtils.error_exit("Invalid species")
 
-        species_data = all_species_data[species]["data"]
+        species_databases = {
+            db_name: db_info
+            for db_name, db_info in master["databases"].items()
+            if db_info["species"] == species
+        }
+
+        # Collapse every database's views to a single view_name -> {database,
+        # platform, groups} mapping, same view-name normalization as /databases.
+        all_views = {}
+        for db_name, db_info in species_databases.items():
+            for view_info in db_info["views"].values():
+                view_key = view_info["display_name"].replace(" ", "_")
+                all_views[view_key] = {
+                    "database": db_name,
+                    "platform": db_info["platform"],
+                    "groups": view_info["sample_groups"],
+                }
 
-        # if user requests all views
         if view.lower() == "all":
-            return BARUtils.success_exit({"species": species, "views": species_data["views"]})
+            return BARUtils.success_exit({"species": species, "views": all_views})
 
-        # otherwise check single view
-        if view not in species_data["views"]:
+        if view not in all_views:
             return BARUtils.error_exit("Invalid view for this species")
 
-        view_data = species_data["views"][view]
+        view_data = all_views[view]
         return BARUtils.success_exit({
             "species": species,
             "view": view,
-            "data_type": view_data.get("data_type", "Unknown"),
-            "groups": view_data["groups"]
+            "database": view_data["database"],
+            "platform": view_data["platform"],
+            "groups": view_data["groups"],
         })

api/services/efp_bootstrap.py

@@ -1,6 +1,4 @@
 """
-Reena Obmina | BCB330 Project 2025-2026 | University of Toronto
-
 Bootstrap utilities for creating eFP MySQL databases from the shared schema registry.
 
 Used by: