From 39f859952d8189b51ae8d83c51f1c744d2101173 Mon Sep 17 00:00:00 2001 From: sfluegel Date: Tue, 18 Aug 2026 19:48:14 +0200 Subject: [PATCH 1/8] various updates w/ claude --- .gitignore | 3 +- README.md | 35 +- backend/api/chemclass.py | 409 +++++++++++----- backend/app.py | 15 +- backend/chebi_utils.py | 24 - backend/config.template.json | 26 +- backend/ontology.py | 69 +++ backend/requirements.txt | 4 +- react-app/src/About.js | 172 ++++--- .../src/smiles-form/attribution-chart.js | 123 +++++ .../src/smiles-form/classification-form.js | 448 ++++++++---------- .../src/smiles-form/details-page-chemlog.js | 1 - react-app/src/smiles-form/details-page.js | 1 - .../src/smiles-form/ensemble-settings.js | 99 ++++ react-app/src/smiles-form/ontology-utils.js | 305 ++++++------ 15 files changed, 1103 insertions(+), 631 deletions(-) delete mode 100644 backend/chebi_utils.py create mode 100644 backend/ontology.py create mode 100644 react-app/src/smiles-form/attribution-chart.js create mode 100644 react-app/src/smiles-form/ensemble-settings.js diff --git a/.gitignore b/.gitignore index fe27cf6..9559f7a 100644 --- a/.gitignore +++ b/.gitignore @@ -138,4 +138,5 @@ react-app/node_modules/ data/ backend/data/ .vscode/ -backend/config.json \ No newline at end of file +backend/config.json +.playwright-mcp/ diff --git a/README.md b/README.md index bbe5d72..0b41e07 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ Chebifier is a tool for automated classification of chemicals in the [ChEBI](https://www.ebi.ac.uk/chebi/) ontology. This repository only hosts the front end of Chebifier. For the classification itself, see [python-chebifier](https://github.com/ChEB-AI/python-chebifier). ## News +- 2026/08/18: Moved to the calibrated ensembles of python-chebifier (WMV-F1 on ChEBI v252 3-STAR, + score-based inconsistency resolution). Predictions now come with a 0-1 ensemble score and a + per-model attribution, model weights can be tuned per prediction in "Ensemble settings", and + molecules can be entered as InChI as well as SMILES. - 2026/02/16: Added Lopster and new deep learning models. - 2025/11/11: Fixed processing error for GNNs. - 2025/11/05: Added new models (v244, including GAT, 3-STAR models and augmented GNNs), redesigned frontend. @@ -28,11 +32,24 @@ After that, you can install the prediction system and web framework: and change the path for each setting according to your setup. The ensemble can take any models that are implemented in [python-chebifier](https://github.com/ChEB-AI/python-chebifier). See the repository for example configurations. Common arguments for a model are: - * `type`: one of the available [MODEL_TYPES](https://github.com/ChEB-AI/python-chebifier/blob/dev/chebifier/model_registry.py), e.g. `electra`, - * `batch_size`: Number of molecules that are passed to the model at once, - * `classwise_weights_path` (optional): Weights that should be assigned to each class (i.e., trust scores calculated on a validation set with [this script](https://github.com/ChEB-AI/python-chebai/blob/dev/chebai/result/generate_class_properties.py) + * `type`: one of the available [MODEL_TYPES](https://github.com/ChEB-AI/python-chebifier/blob/dev/chebifier/model_registry.py), e.g. `gat`, + * `ckpt_path`: path to the model checkpoint (deep learning models only), + * `batch_size`: number of molecules that are passed to the model at once, + * `calibration_name` (optional): the name the model's calibration files in `ENSEMBLE_DIR` were written under, if it differs from the name shown in the web app, + * `model_weight` (optional, default 1): how much the model's votes count, independently of the class. This is the value the "Ensemble settings" sliders start from. +Besides the models, the configuration points at the calibration of the ensemble: + * `ENSEMBLE_DIR`: directory holding the calibration written by `chebifier build` (`prediction_thresholds.yaml`, `_classwise_f1.txt`, `best_hyperparameters.csv`). Only the files of the models listed in `MODELS` have to be present - a model without a `_classwise_f1.txt` votes with full trust and a neutral threshold, so its influence is set by its model weight alone. + * `ENSEMBLE_CLASSES`: the class list the ensemble was calibrated on, one ChEBI id per line. The class-wise F1 scores are stored positionally, so this list has to match the calibration exactly. + * `CHEBI_GRAPH`: the ChEBI graph pickle the calibration was built against. If the file is missing, it is downloaded from [Hugging Face](https://huggingface.co/datasets/chebai/chebifier). + * `INCONSISTENCY_RESOLUTION`: `score-based` (the default) or `none`. +### How a prediction is explained + +Molecules can be entered as SMILES or InChI strings, mixed freely. An InChI is translated to SMILES +before it reaches the base learners, and the response carries the SMILES each input was read as. + +For each predicted class, the web app reports the ensemble score - a probability in [0, 1], with the class predicted above the ensemble's decision threshold - together with the share of that decision each base learner is responsible for (the shares sum to 1) and the raw 0-1 prediction each model made for the class. ### Setup Frontend @@ -44,14 +61,22 @@ npm run build ### Run in development -You can now start the development server with +You can now start the development server with ``` cd backend flask run ``` -The server should now run at [localhost:5000](localhost:5000) +The server should now run at [localhost:5000](localhost:5000), serving both the API and the built +frontend. Start it from the `backend` directory - the configuration and `data/disjoint_*.csv` are +looked up relative to the working directory. Startup takes a while: the ChEBI graph, the model +checkpoints and the SMILES lookup table are all loaded up front. + +The backend has to run in an environment that has `chebifier` and all of its base learners +installed (`chebai`, `chebai-graph`, `chemlog`, `chemlog-extra`, `c3p`). If that environment is a +virtualenv of the python-chebifier checkout, run flask from it directly, e.g. +`../../python-chebifier/.venv/Scripts/python -m flask --app app run`. ## Citation diff --git a/backend/api/chemclass.py b/backend/api/chemclass.py index dafeb31..f147bc9 100644 --- a/backend/api/chemclass.py +++ b/backend/api/chemclass.py @@ -1,192 +1,369 @@ -import copy -import os -import sys +import math -from flask_restful import Resource, reqparse -from PIL import Image -import base64 -import io -from app import app import matplotlib as mpl -import networkx as nx -from rdkit import Chem -from rdkit.Chem.Draw import rdMolDraw2D import torch -from chebi_utils import CHEBI_FRAGMENT -import hashlib +from app import app +from flask_restful import Resource, reqparse -from chebifier.ensemble.weighted_majority_ensemble import WMVwithF1Ensemble +from api.ensemble import DEFAULT_MODEL_WEIGHT, WeightedWMVF1Ensemble +from chebifier.inconsistency_resolution import ScoreBasedPredictionSmoother +from chebifier.model_registry import MODEL_TYPES +from chebifier.predict import ( + apply_inconsistency_resolution, + collect_base_learner_predictions, + get_base_learner_predictions, +) +from chebi_utils.read_molecule import smiles_or_inchi_to_mol +from chebifier.utils import _smiles_to_mol, get_disjoint_files +from ontology import CHEBI_GRAPH, class_name, most_specific, to_vis_graph +from rdkit import Chem mpl.use("Agg") -if torch.cuda.is_available(): - device = "cuda" -else: - device = "cpu" +MODEL_CONFIG = app.config["MODELS"] + + +def build_models(): + models = {} + for model_name, config in MODEL_CONFIG.items(): + config = dict(config) + model_type = config.pop("type") + config.pop("calibration_name", None) + print(f"Building {model_name} ({model_type})...") + models[model_name] = MODEL_TYPES[model_type]( + model_name, **config, chebi_graph=CHEBI_GRAPH + ) + return models + + +def read_ensemble_classes(): + """The classes the ensemble was calibrated on. -ENSEMBLE = WMVwithF1Ensemble(app.config["MODELS"]) + The class-wise F1 scores of the calibration are stored as one value per class, so the ensemble + can only run on exactly this class list - and in this order. + """ + with open(app.config["ENSEMBLE_CLASSES"], "r", encoding="utf-8") as f: + return [line.strip() for line in f if line.strip()] -def _build_node(ident, node, include_labels=True): - d = dict(id=ident, - color="#EEEEEC" if node.get("artificial") else "#729FCF") - d["title"] = node["lbl"] - if include_labels: - d["label"] = node["lbl"] +class CachedSmoother(ScoreBasedPredictionSmoother): + """Inconsistency resolution for a fixed class list. + + Setting the label names walks the ChEBI hierarchy once per class to build the transitive + subsumption matrix, which takes far too long to redo on every request. + """ + + def set_label_names(self, label_names): + if label_names is not None and label_names == getattr(self, "label_names", None): + return + super().set_label_names(label_names) + + +MODELS = build_models() +DEFAULT_MODEL_WEIGHTS = { + model_name: config.get("model_weight", DEFAULT_MODEL_WEIGHT) + for model_name, config in MODEL_CONFIG.items() +} +ENSEMBLE = WeightedWMVF1Ensemble( + app.config["ENSEMBLE_DIR"], + calibration_names={ + model_name: config.get("calibration_name", model_name) + for model_name, config in MODEL_CONFIG.items() + }, + model_weights=DEFAULT_MODEL_WEIGHTS, +) +ENSEMBLE_CLASSES = read_ensemble_classes() +CLASS_INDEX = {cls: idx for idx, cls in enumerate(ENSEMBLE_CLASSES)} +DECISION_THRESHOLD = ENSEMBLE.decision_threshold +SMOOTHER = ( + CachedSmoother( + chebi_graph=CHEBI_GRAPH, + label_names=ENSEMBLE_CLASSES, + disjoint_files=get_disjoint_files(), + threshold=DECISION_THRESHOLD, + ) + if app.config["INCONSISTENCY_RESOLUTION"] == "score-based" + else None +) +# Classes that came close to being predicted are offered alongside the prediction. "Close" is a +# fraction of the ensemble's own operating point, so it follows the threshold if that ever moves. +NEAR_MISS_FRACTION = 0.5 +NEAR_MISS_THRESHOLD = DECISION_THRESHOLD * NEAR_MISS_FRACTION +NEAR_MISS_LIMIT = 10 + +print( + f"Ensemble ready: {len(MODELS)} models, {len(ENSEMBLE_CLASSES)} classes, " + f"decision threshold {DECISION_THRESHOLD} (near misses from {NEAR_MISS_THRESHOLD})." +) + + +def selected_model_names(selected_models): + """The models to run, in configuration order. Without a selection, the whole ensemble runs.""" + if not selected_models: + return list(MODELS) + return [name for name in MODELS if selected_models.get(name)] + + +def requested_weights(model_weights): + if not model_weights: + return None + weights = {} + for model_name, weight in model_weights.items(): + if model_name in MODELS: + try: + weights[model_name] = max(0.0, float(weight)) + except (TypeError, ValueError): + continue + return weights - return d +def jsonable(value): + """JSON has no NaN, and a base learner that did not cover a class reports exactly that.""" + value = float(value) + return None if math.isnan(value) else value -def nx_to_graph(g: nx.Graph, colors=None): - results = { - "nodes": {n: g.nodes[n] for n in g.nodes}, - "edges": [edge for edge in g.edges() if "label" in g.get_edge_data(edge[0], edge[1])], - } - if colors is not None: - results["node_colors"] = dict() - for node, color in colors.items(): - if node in results["nodes"]: - results["node_colors"][node] = color - return results + +def to_smiles(molecule): + """The SMILES string the models are run on, or None if RDKit cannot read the input. + + Inputs may be SMILES or InChI. A SMILES string is passed on as it was written - canonicalising + it would hand the models a different molecule representation than the user asked about - while + an InChI has to be translated, since the base learners only take SMILES. + """ + if not molecule: + return None + if molecule.startswith("InChI="): + mol = smiles_or_inchi_to_mol(molecule) + return None if mol is None else Chem.MolToSmiles(mol) + return molecule if _smiles_to_mol(molecule) is not None else None + + +def readable_rows(molecules): + """The inputs RDKit can read, as (index, SMILES) pairs. + + Not every base learner reports an unreadable molecule as "no prediction" - C3P, for one, + answers "no" for each of its classes - so the ensemble would report an empty classification + rather than a failure. Sorting them out here also keeps them out of the models entirely. + """ + resolved = [to_smiles(molecule) for molecule in molecules] + return resolved, [ + index for index, smiles in enumerate(resolved) if smiles is not None + ] + + +def near_miss_classes(aggregated, row): + """The highest scoring classes that stayed below the decision threshold. + + Only classes some model actually covered can be near misses - a class no model said anything + about sits at the neutral score, which is not a near miss but an absence of evidence. + """ + scores = aggregated["net_score"][row] + candidates = ( + ~aggregated["class_decisions"][row] + & aggregated["has_valid_predictions"][row] + & (scores > NEAR_MISS_THRESHOLD) + ) + class_indices = torch.nonzero(candidates).flatten() + if class_indices.numel() == 0: + return [] + ranked = class_indices[ + torch.argsort(scores[class_indices], descending=True)[:NEAR_MISS_LIMIT] + ] + return [ENSEMBLE_CLASSES[class_idx] for class_idx in ranked.tolist()] + + +def run_ensemble(smiles_list, selected_models, model_weights): + """Base learner predictions, ensemble aggregation and inconsistency resolution for a batch of + SMILES strings. Returns the per-model predictions alongside the aggregated result.""" + models = {name: MODELS[name] for name in selected_model_names(selected_models)} + if not models: + raise ValueError("No models selected.") + predictions = get_base_learner_predictions(models, smiles_list) + predictions, _ = collect_base_learner_predictions( + predictions, classes=ENSEMBLE_CLASSES + ) + aggregated = ENSEMBLE.with_weights(model_weights).predict( + predictions, attribution=True + ) + if SMOOTHER is not None: + aggregated = apply_inconsistency_resolution( + SMOOTHER, + ENSEMBLE_CLASSES, + aggregated, + decision_threshold=DECISION_THRESHOLD, + ) + aggregated["class_decisions"] = ( + aggregated["net_score"] > DECISION_THRESHOLD + ) & aggregated["has_valid_predictions"] + aggregated["complete_failure"] = torch.all( + ~aggregated["has_valid_predictions"], dim=1 + ) + return predictions, aggregated class ModelInfoAPI(Resource): def get(self): return { - "available_models": [model.model_name for model in ENSEMBLE.models], - "available_models_info_texts": [model.info_text for model in ENSEMBLE.models] + "available_models": list(MODELS), + "available_models_info_texts": [ + model.info_text for model in MODELS.values() + ], + "default_model_weights": DEFAULT_MODEL_WEIGHTS, + "decision_threshold": DECISION_THRESHOLD, + "n_classes": len(ENSEMBLE_CLASSES), } -def verify_disjointness(predicted_classes): - disjoints = [] - with open(os.path.join("data", "disjoint_chebi.csv")) as f: - for line in f: - disjoints.append([f"CHEBI:{i}" for i in line.strip().split(",")]) - with open(os.path.join("data", "disjoint_additional.csv")) as f: - for line in f: - disjoints.append([f"CHEBI:{i}" for i in line.strip().split(",")]) - violations = [] - for sample in predicted_classes: - violations_sample = [] - for disjoint in disjoints: - if all(cls in sample for cls in disjoint): - violations_sample.append(disjoint) - violations.append(violations_sample) - return violations - - class BatchPrediction(Resource): def post(self): """ Accepts a dictionary with the following structure { - "smiles": [ ... list of smiles strings] - "ontology": bool (Optional) + "smiles": [ ... list of SMILES or InChI strings], + "ontology": bool (Optional), + "selectedModels": {model name: bool} (Optional), + "modelWeights": {model name: number} (Optional, overrides the configured weights) } :return: A dictionary with the following structure { - "predicted_parents": [ ... [... parent classes as predicted by the system] or None for each smiles ], - "direct_parents": [ ... [... lowest possible predicted parents] or None for each smiles ] or None - "ontology": Only returned if `ontology` is set. Returns a vis.js conform representation of the ontology containing all predicted classes. + "predicted_parents": [ ... [... parent classes as predicted by the system] or None for each input ], + "direct_parents": [ ... [... lowest predicted parents, each as [ChEBI id, name]] + or None for each input ], + "explanations": [ ... {ChEBI id: {name, score, models: {model: {prediction, + attribution, vote}}}} for every predicted class, or None for each input ], + "smiles": [ ... the SMILES string each input was read as (an InChI is translated) ], + "ontology": Only returned if `ontology` is set. Returns a vis.js conform representation + of the ontology containing all predicted classes. } - If the system is unable to parse any smiles string, the respective entry in each list will be `None`. + If the system is unable to parse an input, the respective entry in each list will be `None`. """ parser = reqparse.RequestParser() parser.add_argument("smiles", type=str, action="append") parser.add_argument("ontology", type=bool, required=False, default=False) - parser.add_argument("selectedModels", type=dict) + parser.add_argument("selectedModels", type=dict, required=False, default=None) + parser.add_argument("modelWeights", type=dict, required=False, default=None) args = parser.parse_args() smiles = args["smiles"] generate_ontology = args["ontology"] - selected_models = args["selectedModels"] if not smiles or len(smiles) == 0: result = { "predicted_parents": [], "direct_parents": [], - "violations": [] + "explanations": [], + "smiles": [], } if generate_ontology: result["ontology"] = [] return result - ensemble_models = ENSEMBLE.models - try: - ENSEMBLE.models = [model for model in ensemble_models if - model.model_name in selected_models and selected_models[model.model_name]] - all_predicted, intermediate_results = ENSEMBLE.predict_smiles_list(smiles, return_intermediate_results=True) - finally: - ENSEMBLE.models = ensemble_models # restore original model list - - graphs_per_smiles = [CHEBI_FRAGMENT.subgraph(predicted) if predicted is not None else None for predicted in all_predicted] + resolved_smiles, rows = readable_rows(smiles) + # without a single model there is nothing to predict with, so every input "fails" + if not selected_model_names(args["selectedModels"]): + rows = [] + if rows: + predictions, aggregated = run_ensemble( + [resolved_smiles[index] for index in rows], + args["selectedModels"], + requested_weights(args["modelWeights"]), + ) + model_names = list(predictions) + attribution = aggregated["attribution"] + positive = aggregated["positive_mask"] + negative = aggregated["negative_mask"] + row_of = {smiles_idx: row for row, smiles_idx in enumerate(rows)} - direct_parents = [] - for smiles_idx in range(len(all_predicted)): - graph = graphs_per_smiles[smiles_idx] - if graph is None: + predicted_parents, direct_parents, ontologies, explanations = [], [], [], [] + for smiles_idx in range(len(smiles)): + row = row_of.get(smiles_idx) + if row is None or aggregated["complete_failure"][row]: + predicted_parents.append(None) direct_parents.append(None) + ontologies.append(None) + explanations.append(None) continue - direct_parents_for_smiles = [] - for cls in all_predicted[smiles_idx]: - cls_idx = intermediate_results["predicted_classes"][cls] - if any(parent in graph.nodes for parent in graph.successors(cls)): - continue - calculations = dict() - for model_idx, model in enumerate([model for model in ensemble_models if - model.model_name in selected_models and selected_models[model.model_name]]): - pos_prediction = intermediate_results["positive_mask"][smiles_idx, cls_idx, model_idx] - neg_prediction = intermediate_results["negative_mask"][smiles_idx, cls_idx, model_idx] - # skip models that made no prediction - if pos_prediction or neg_prediction: - confidence = intermediate_results["confidence"][smiles_idx, cls_idx, model_idx] - trust = intermediate_results["classwise_weights"][0 if pos_prediction else 1][cls_idx, model_idx].item() / model.model_weight - calculations[model.model_name] = { - "prediction": pos_prediction.item(), - "confidence": confidence.item(), - # select either trust for positive or negative predictions - "trust": trust, - "model_weight": model.model_weight, - "model_score": (-1 if neg_prediction else 1) * confidence.item() * trust * model.model_weight, + decisions = aggregated["class_decisions"][row] + predicted = [ + ENSEMBLE_CLASSES[class_idx] + for class_idx in torch.nonzero(decisions).flatten().tolist() + ] + near_misses = near_miss_classes(aggregated, row) + predicted_parents.append(predicted) + ontologies.append( + to_vis_graph(predicted, near_misses) if generate_ontology else None + ) + + direct_parents.append([[cls, class_name(cls)] for cls in most_specific(predicted)]) + + explanations_for_smiles = {} + for cls in predicted + near_misses: + class_idx = CLASS_INDEX[cls] + models = {} + for model_idx, model_name in enumerate(model_names): + # which way the model voted: its prediction against its own threshold. Models + # that did not cover the class cast no vote and hold no share of the decision, + # so they are left out entirely. + vote = int(positive[row, class_idx, model_idx]) - int( + negative[row, class_idx, model_idx] + ) + if vote: + models[model_name] = { + "prediction": jsonable( + predictions[model_name][row, class_idx] + ), + "attribution": jsonable( + attribution[row, class_idx, model_idx] + ), + "vote": vote, } - net_score = intermediate_results["net_score"][smiles_idx, cls_idx].item() - direct_parents_for_smiles.append((cls, graph.nodes[cls]["name"], calculations, net_score)) - direct_parents.append(direct_parents_for_smiles) + explanations_for_smiles[cls] = { + "name": class_name(cls), + "score": jsonable(aggregated["net_score"][row, class_idx]), + "models": models, + "near_miss": cls in near_misses, + } + explanations.append(explanations_for_smiles) result = { - "predicted_parents": all_predicted, + "predicted_parents": predicted_parents, "direct_parents": direct_parents, + "explanations": explanations, + # what the input was read as - an InChI is translated to SMILES for the models, and + # the frontend needs the same string to draw the molecule and ask for details + "smiles": [ + resolved_smiles[index] if row_of.get(index) is not None else None + for index in range(len(smiles)) + ], } if generate_ontology: - result["ontology"] = [nx_to_graph(g) if g is not None else None for g in graphs_per_smiles], - + result["ontology"] = ontologies return result + class PredictionDetailApiHandler(Resource): def post(self): parser = reqparse.RequestParser() - parser.add_argument("type", type=str, required=False, default="type") # can be used to specify different types of requests in the future + # can be used to specify different types of requests in the future + parser.add_argument("type", type=str, required=False, default="type") parser.add_argument("smiles", type=str) - parser.add_argument("selectedModels", type=dict) + parser.add_argument("selectedModels", type=dict, required=False, default=None) args = parser.parse_args() - # note, the post req from frontend needs to match the strings here (e.g. 'type and 'message') - request_type = args["type"] - smiles = args["smiles"] - selected_models = args["selectedModels"] + smiles = to_smiles(args["smiles"]) explain_infos = {"models": dict()} - do_models = [model for model in ENSEMBLE.models if model.model_name in selected_models and selected_models[model.model_name]] - - for model in do_models: + if smiles is None: + return explain_infos + for model_name in selected_model_names(args["selectedModels"]): + model = MODELS[model_name] explain_infos_model = model.explain_smiles(smiles) if explain_infos_model is not None: explain_infos_model["model_type"] = model.__class__.__name__ explain_infos_model["model_info"] = model.info_text - explain_infos["models"][model.model_name] = explain_infos_model + explain_infos["models"][model_name] = explain_infos_model return explain_infos diff --git a/backend/app.py b/backend/app.py index e8caf08..9226d93 100644 --- a/backend/app.py +++ b/backend/app.py @@ -1,4 +1,4 @@ -from flask import Flask, send_from_directory +from flask import Flask, request, send_from_directory from flask_restful import Api, Resource, reqparse from flask_cors import CORS # comment this on deployment import torch.multiprocessing as mp @@ -19,6 +19,19 @@ def serve(path): return send_from_directory(app.static_folder, 'index.html') +@app.errorhandler(404) +def serve_client_route(error): + """Hand unknown paths to the frontend, which routes them itself. + + Without this, opening or reloading a page other than "/" (e.g. /about) hits Flask rather than + the router and 404s. Static files are served before this runs, so only paths that exist as + client-side routes reach it - and API paths keep their 404. + """ + if request.path.startswith("/api/"): + return {"message": "Not found"}, 404 + return send_from_directory(app.static_folder, 'index.html') + + def load_endpoints(): from api.chemclass import PredictionDetailApiHandler, BatchPrediction, ModelInfoAPI api.add_resource(PredictionDetailApiHandler, '/api/details') diff --git a/backend/chebi_utils.py b/backend/chebi_utils.py deleted file mode 100644 index 717ede5..0000000 --- a/backend/chebi_utils.py +++ /dev/null @@ -1,24 +0,0 @@ -import queue - -from app import app -import json -import networkx as nx -from chebifier.utils import load_chebi_graph, build_chebi_graph - - -CHEBI_FRAGMENT = build_chebi_graph(244) - - -def get_transitive_predictions(predicted_classes): - # get all parents of predicted classes - all_predicted = [cls for clss in predicted_classes for cls in clss] - q = queue.Queue() - for cls in all_predicted: - q.put(cls) - while not q.empty(): - cls = q.get() - for parent in CHEBI_FRAGMENT.predecessors(cls): - if parent not in all_predicted: - all_predicted.append(parent) - q.put(parent) - return all_predicted diff --git a/backend/config.template.json b/backend/config.template.json index d392da2..d9b02bf 100644 --- a/backend/config.template.json +++ b/backend/config.template.json @@ -1,12 +1,18 @@ { - "TESTING":true, - "CHEBI_JSON": "path-to-chebi-export.json", - "MODELS": { - "My model": { - "type": "chebifier type (e.g. electra)", - "ckpt_path": "path/to/checkpoint.ckpt", - "batch_size": 32, - "classwise_weights_path": "path/to/trust.json" - } + "TESTING": true, + "CHEBI_VERSION": 252, + "CHEBI_GRAPH": "path/to/chebi_graph_v252.pkl (optional, downloaded from Hugging Face if missing)", + "ENSEMBLE_DIR": "directory holding the ensemble calibration (prediction_thresholds.yaml, _classwise_f1.txt, best_hyperparameters.csv)", + "ENSEMBLE_CLASSES": "file listing the classes the ensemble was calibrated on, one per line", + "INCONSISTENCY_RESOLUTION": "score-based", + "MODELS": { + "My model": { + "type": "chebifier model type (e.g. gat, chemlog, lopster_clingo, c3p, chebi_lookup)", + "ckpt_path": "path/to/checkpoint.ckpt", + "batch_size": 16, + "calibration_name": "name the calibration files in ENSEMBLE_DIR were written under (defaults to the key of this entry)", + "model_weight": 1, + "description": "shown in the model info panel of the web app" } -} \ No newline at end of file + } +} diff --git a/backend/ontology.py b/backend/ontology.py new file mode 100644 index 0000000..015ab41 --- /dev/null +++ b/backend/ontology.py @@ -0,0 +1,69 @@ +"""The ChEBI graph the API serves its class names and hierarchy from. + +This module used to be called `chebi_utils`, which now shadows the installed `chebi_utils` +package that chebifier itself imports - hence the rename. +""" + +import os + +from app import app +from chebi_utils.obo_extractor import get_hierarchy_subgraph +from chebifier.utils import load_chebi_graph + +_local_graph = app.config.get("CHEBI_GRAPH") +if _local_graph and not os.path.exists(_local_graph): + print(f"ChEBI graph {_local_graph} not found, downloading it from Hugging Face...") + _local_graph = None + +# the full graph carries non-subsumption relations (has role, conjugate acid/base, ...) and is +# what the predictors expect; the hierarchy is the is-a subgraph of it +CHEBI_GRAPH = load_chebi_graph(_local_graph) +CHEBI_HIERARCHY = get_hierarchy_subgraph(CHEBI_GRAPH) + +# ChEBI's "molecular entity", the class every other class here descends from. The graph starts +# below it, so it is not a node of its own and can never be predicted - which is the point: it +# holds for every molecule, and saying so carries no information. +MOLECULAR_ENTITY = "23367" +MOLECULAR_ENTITY_NAME = "molecular entity" + + +def class_name(chebi_id: str) -> str: + node = CHEBI_GRAPH.nodes.get(chebi_id) + if node is None or not node.get("name"): + return f"CHEBI:{chebi_id}" + return node["name"] + + +def most_specific(predicted_classes: list[str]) -> list[str]: + """The predicted classes that have no predicted subclass, i.e. the lowest classes the + prediction reaches in the hierarchy.""" + predicted = [cls for cls in predicted_classes if cls in CHEBI_HIERARCHY] + subgraph = CHEBI_HIERARCHY.subgraph(predicted) + # is-a edges point from child to parent, so the predecessors of a class are its subclasses + return [cls for cls in predicted if not any(True for _ in subgraph.predecessors(cls))] + + +def _in_hierarchy(classes) -> list[str]: + return [cls for cls in classes if cls in CHEBI_HIERARCHY and cls != MOLECULAR_ENTITY] + + +def to_vis_graph(predicted_classes: list[str], near_misses=()) -> dict: + """The predicted classes as a vis.js graph: their is-a hierarchy below the top class. + + Edges point from a class to its superclass, as they do in ChEBI. Every class without a + superclass in the graph is hung under `MOLECULAR_ENTITY`, which gives the hierarchy a single + root to grow from. + + Near misses - classes that came close to being predicted but stayed below the threshold - are + marked as such, so the frontend can leave them out until they are asked for. + """ + predicted = _in_hierarchy(predicted_classes) + near = [cls for cls in _in_hierarchy(near_misses) if cls not in set(predicted)] + classes = predicted + near + subgraph = CHEBI_HIERARCHY.subgraph(classes) + edges = [list(edge) for edge in subgraph.edges] + edges += [[cls, MOLECULAR_ENTITY] for cls in classes if subgraph.out_degree(cls) == 0] + nodes = {cls: {"name": class_name(cls)} for cls in predicted} + nodes.update({cls: {"name": class_name(cls), "near_miss": True} for cls in near}) + nodes[MOLECULAR_ENTITY] = {"name": MOLECULAR_ENTITY_NAME, "root": True} + return {"nodes": nodes, "edges": edges} diff --git a/backend/requirements.txt b/backend/requirements.txt index c6e266d..01850fd 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,6 +11,8 @@ gavel>=0.1.7 chemlog>=1.0.6 chebai>=1.2.0 chebai-graph +chebi-utils>=0.3 chemlog-extra +c3p chebifier>=1.2.1 -multiprocess \ No newline at end of file +multiprocess diff --git a/react-app/src/About.js b/react-app/src/About.js index 980d10a..c5d8f30 100644 --- a/react-app/src/About.js +++ b/react-app/src/About.js @@ -1,3 +1,4 @@ +import Divider from '@mui/material/Divider'; import Paper from '@mui/material/Paper'; import Typography from '@mui/material/Typography'; import Link from '@mui/material/Link'; @@ -6,95 +7,148 @@ import axios from "axios"; import * as React from "react"; import {SlChemistry} from "react-icons/sl"; +/** One titled block of the page. Each section is its own card, so where one topic ends and the + * next begins is visible before a word is read. */ +const Section = ({title, children}) => ( + + {title && ( + <> + + {title} + + + + )} + {children} + +); + const About = () => { const [availableModels, setAvailableModels] = React.useState([]); const [availableModelsInfoTexts, setAvailableModelsInfoTexts] = React.useState([]); + const [numClasses, setNumClasses] = React.useState(null); // Load once on mount so About content is fetched when the site loads React.useEffect(() => { axios.get('/api/modelinfo').then(response => { setAvailableModels(response.data.available_models || []); setAvailableModelsInfoTexts(response.data.available_models_info_texts || []); + setNumClasses(response.data.n_classes || null); }).catch(() => { // silently ignore, page content still renders }); }, []); - const modelList = availableModels.map((model, index) => ( - - {model} - - - )); - return (
- - About - - Chebifier is a tool for automated classification of chemicals in the ChEBI ontology. - Currently, it can predict 1,700+ ChEBI classes. - - - To run a prediction, enter a SMILES string (or multiple ones, line-separated) or upload a file. Then, - hit the predict button (running the model might take a few seconds). - You can get more information about a result by clicking on it. - + +
+ About Chebifier + + Chebifier is a tool for automated classification of chemicals in + the ChEBI ontology. It currently + predicts {numClasses ? numClasses.toLocaleString('en-US') : '2,000+'} ChEBI classes. + + + To run a prediction, enter a SMILES or InChI string (or several ones, one per line) + or upload a file, then hit the predict button - running the models takes a few + seconds. Click a result to see the molecule, the predicted part of the ontology and + what each model contributed. + +
- News - - 11/2025: Added new models (Graph Attention Networks and augmented Graph Neural Networks). - Improved the Ensemble weighting mechanism. Redesigned the user interface. - - - 08/2025: Added the ensemble. Added ChemLog, C3P and Graph Convolutional Networks. - +
+ + Chebifier combines machine learning models, rule-based methods and a ChEBI lookup. + For every class, each model that covers it casts a vote. This vote gets weighted by how + reliable it proved to be for that class on validation data and + the model weight you can tune in the ensemble settings. + + + The resulting predictions are checked for consistency with the ChEBI ontology and + corrected if necessary. The final predictions are then sorted by their confidence score and displayed to the user. + + + Clicking a class in the ontology graph of a result shows how much of the decision + each model is responsible for. Details about the ensemble and its implementation can + be found here. + +
- The Ensemble - - Chebifier uses an ensemble of machine learning models and rule-based methods to classify molecules into ChEBI classes. - A weighting mechanism and inconsistency resolution are applied to ensure you get the best our models can over. - Details about the ensemble and the implementation can be found here. - +
+ + At the moment, the following prediction models are supported by Chebifier. You can + either use them together in the ensemble or select a single model. + + {availableModels.map((model, index) => ( + + + {model} + + + + ))} +
- Models - - At the moment, the following prediction models are supported by Chebifier. - You can either use them in the ensemble or select a specific model. - +
+ {[ + ['08/2026', 'Re-calibrated ensemble. Added new, better deep learning models trained on ChEBI ' + + 'version 252. This increased the coverage by ~500 classes. Improved user interface, added InChI support and model attributions. Model weights can now be set manually' + + ''], + ['02/2026', 'Added Lopster and new deep learning models.'], + ['11/2025', 'Added new models (Graph Attention Networks and augmented Graph Neural ' + + 'Networks). Improved the ensemble weighting mechanism. Redesigned the user interface.'], + ['08/2025', 'Added the ensemble. Added ChemLog, C3P and Graph Convolutional Networks.'], + ].map(([date, text], index, entries) => ( + + + {date} + + {text} + + ))} +
- {/* Available models inside a Paper for emphasis */} - {/**/} - {modelList} - - Main Publication for Chebifier - - Glauer, Martin, et al.: Chebifier: Automating Semantic Classification in ChEBI to Accelerate - Data-driven Discovery; Digital Discovery 3.5 (2024), Link - - +
+ + Glauer, Martin, et al.: Chebifier: Automating Semantic Classification in ChEBI to + Accelerate Data-driven Discovery; Digital Discovery 3.5 (2024), Link + +
+
- ); }; diff --git a/react-app/src/smiles-form/attribution-chart.js b/react-app/src/smiles-form/attribution-chart.js new file mode 100644 index 0000000..ef6a08f --- /dev/null +++ b/react-app/src/smiles-form/attribution-chart.js @@ -0,0 +1,123 @@ +import * as React from 'react'; +import ArrowDropDownIcon from '@mui/icons-material/ArrowDropDown'; +import ArrowDropUpIcon from '@mui/icons-material/ArrowDropUp'; +import Box from '@mui/material/Box'; +import Tooltip from '@mui/material/Tooltip'; +import Typography from '@mui/material/Typography'; + +// Two poles that read as opposite and stay separable under colour-vision deficiency +// (validated: worst CVD deltaE 23.8, contrast >= 3:1 on a light surface). The arrow beside each +// bar repeats the direction, so the colour never carries it alone. +const VOTE_STYLE = { + 1: {color: '#2a78d6', label: 'supports', Icon: ArrowDropUpIcon}, + '-1': {color: '#d03b3b', label: 'opposes', Icon: ArrowDropDownIcon}, +}; + +const formatPrediction = (value) => + typeof value === 'number' ? value.toFixed(3) : '–'; + +const formatShare = (share) => { + if (typeof share !== 'number') return '–'; + if (share > 0 && share < 0.001) return '<0.1%'; + return `${(share * 100).toFixed(1)}%`; +}; + +function ScoreBar({score, threshold}) { + const value = typeof score === 'number' ? Math.min(Math.max(score, 0), 1) : 0; + return ( + + + + {typeof score === 'number' ? score.toFixed(3) : '–'} + + + ensemble score (predicted above {threshold}) + + + + threshold ? '#2a78d6' : '#d03b3b', + borderRadius: '4px', + }}/> + + + + 0 + 1 + + + ); +} + +/** + * Per-model attributions for one predicted class: the share of the ensemble decision each base + * learner is responsible for (the shares sum to 1), together with the raw 0-1 prediction the + * model made for this class. Models that did not cover the class are left out - they cast no vote + * and hold no share. + */ +export default function AttributionChart({calculations, netScore, threshold = 0.5}) { + const entries = Object.entries(calculations || {}).filter(([, values]) => values?.vote); + const maxShare = Math.max(...entries.map(([, v]) => v?.attribution || 0), 0.001); + + return ( + + + + {entries.length === 0 ? ( + + No model made a prediction for this class. It was predicted because it follows from the + predictions for classes below it in the ChEBI hierarchy. + + ) : ( + + Model + Prediction + Share + + {entries.map(([model, values]) => { + const vote = VOTE_STYLE[String(values.vote)]; + const share = values?.attribution || 0; + const VoteIcon = vote.Icon; + return ( + + {model} + + {formatPrediction(values?.prediction)} + + + + + + 0 ? 1 : 0)}%`, + backgroundColor: vote.color, + borderRadius: '4px', + }}/> + + + + + {formatShare(share)} + + + ); + })} + + )} + + ); +} diff --git a/react-app/src/smiles-form/classification-form.js b/react-app/src/smiles-form/classification-form.js index 7fb0197..4c790f2 100644 --- a/react-app/src/smiles-form/classification-form.js +++ b/react-app/src/smiles-form/classification-form.js @@ -20,17 +20,24 @@ import Tooltip from '@mui/material/Tooltip'; import {randomId} from '../lib/random-id'; import DetailsPage from "./details-page"; -import {plot_ontology, MoleculeStructure} from "./ontology-utils"; -import {Molecules} from "./ontology-utils"; +import {OntologyGraph, MoleculeStructure} from "./ontology-utils"; import {CircularProgress} from "@mui/material"; import Select from '@mui/material/Select'; import MenuItem from '@mui/material/MenuItem'; -import Collapse from '@mui/material/Collapse'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; +import AttributionChart from "./attribution-chart"; +import EnsembleSettings from "./ensemble-settings"; + +// Everything is white, so the pieces of a prediction are told apart by an outline rather than by +// their fill: the card carries a shadow, the panels on it carry a blue border. +export const ACCENT = '#2a78d6'; + + +const panelSx = { + p: 2, + borderRadius: 2, + backgroundColor: '#ffffff', + border: `1px solid ${ACCENT}`, +}; export default function ClassificationGrid() { const [rows, setRows] = React.useState([]); @@ -40,22 +47,23 @@ export default function ClassificationGrid() { const [availableModelsInfoTexts, setAvailableModelsInfoTexts] = React.useState([]); const [selectedModel, setSelectedModel] = React.useState('Ensemble'); const [modelsLoaded, setModelsLoaded] = React.useState(false); + // model weights: how much say each model has in the ensemble vote (tunable in "Ensemble settings") + const [defaultModelWeights, setDefaultModelWeights] = React.useState({}); + const [modelWeights, setModelWeights] = React.useState({}); + const [decisionThreshold, setDecisionThreshold] = React.useState(0.5); const [inputText, setInputText] = React.useState(""); const [predictionsLoading, setPredictionsLoading] = React.useState(false); const [hasPredicted, setHasPredicted] = React.useState(false); const [expandedRowId, setExpandedRowId] = React.useState(null); - // Track which model option was active when predictions were triggered - const [lastPredictedModel, setLastPredictedModel] = React.useState('Ensemble'); // If user uploads before models are loaded, queue SMILES and auto-run when ready const [queuedSmiles, setQueuedSmiles] = React.useState(null); - // map of `${rowId}-${classIdx}` -> boolean for per-chip "Why this class?" panel - const [openWhyMap, setOpenWhyMap] = React.useState({}); - const toggleWhy = (rowId, classIdx) => () => { - const key = `${rowId}-${classIdx}`; - setOpenWhyMap(prev => ({...prev, [key]: !prev[key]})); - }; - const isWhyOpen = (rowId, classIdx) => !!openWhyMap[`${rowId}-${classIdx}`]; + // map of rowId -> the ChEBI id picked in that row's ontology graph + const [selectedClassByRow, setSelectedClassByRow] = React.useState({}); + // map of rowId -> whether that row's graph also shows the classes that just missed the threshold + const [nearMissesByRow, setNearMissesByRow] = React.useState({}); + const selectClass = (rowId) => (chebiId) => + setSelectedClassByRow(prev => ({...prev, [rowId]: chebiId})); // Ref to hidden file input for uploading SMILES const fileInputRef = React.useRef(null); @@ -77,154 +85,139 @@ export default function ClassificationGrid() { } }; - const selectedModels = buildSelectedModels(); - - - if (availableModels.length === 0) { + React.useEffect(() => { axios.get('/api/modelinfo').then(response => { + const weights = response.data.default_model_weights || {}; setAvailableModels(response.data.available_models); setAvailableModelsInfoTexts(response.data.available_models_info_texts); + setDefaultModelWeights(weights); + setModelWeights({...weights}); + if (typeof response.data.decision_threshold === 'number') { + setDecisionThreshold(response.data.decision_threshold); + } setModelsLoaded(true); - }); - } + }, []); + + // Single entry point for classification: every trigger (button, Ctrl+Enter, upload, queued + // upload, re-run after a weight change) goes through here so they cannot drift apart. + const runPrediction = (smiles) => { + if (!smiles || smiles.length === 0) return; + setHasPredicted(true); + setPredictionsLoading(true); + setExpandedRowId(null); + setDetailsByRow({}); + setSelectedClassByRow({}); + setNearMissesByRow({}); + return axios({ + url: '/api/classify', + method: 'post', + data: { + smiles: smiles, + ontology: true, + selectedModels: buildSelectedModels(), + modelWeights: modelWeights + } + }).then(response => { + setRows((old) => old.map((row, i) => ({ + ...row, + direct_parents: response.data.direct_parents[i], + predicted_parents: response.data.predicted_parents[i], + ontology: response.data.ontology[i], + explanations: (response.data.explanations || [])[i], + // what the backend read the input as - an InChI comes back translated to SMILES, which is + // what the structure drawing and the per-model insights need + resolved_smiles: (response.data.smiles || [])[i], + }))); + }).finally(() => setPredictionsLoading(false)); + }; + + const predictFromInput = () => { + const smiles = inputText.trim().replace(/\r/g, '').split('\n').map(s => s.trim()).filter(Boolean); + if (smiles.length === 0) return; + addRows(smiles); + runPrediction(smiles); + }; // If user uploaded SMILES before models were ready, auto-run once models are loaded React.useEffect(() => { if (modelsLoaded && queuedSmiles && !predictionsLoading) { - setLastPredictedModel(selectedModel); - setHasPredicted(true); - setPredictionsLoading(true); - axios({ - url: '/api/classify', - method: 'post', - data: { - smiles: queuedSmiles, - ontology: true, - selectedModels: selectedModels - } - }).then(response => { - setRows((old) => old.map((row, i) => ({ - ...row, - direct_parents: response.data.direct_parents[i], - predicted_parents: response.data.predicted_parents[i], - ontology: response.data.ontology[0][i], - }))); - }).finally(() => { - setPredictionsLoading(false); - setQueuedSmiles(null); - }); + const smiles = queuedSmiles; + setQueuedSmiles(null); + runPrediction(smiles); } - }, [modelsLoaded, queuedSmiles, predictionsLoading, selectedModels]); - - const renderClasses = (params) => { - const data = params.value; - const row = params.row || {}; - const isExpanded = row.id === expandedRowId; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [modelsLoaded, queuedSmiles]); - if (data === null) { - return Could not process input! - } + /** Whether a row has classes that came close to the threshold without reaching it. */ + const hasNearMisses = (row) => + Object.values(row.explanations || {}).some((explanation) => explanation.near_miss); - if (data.length === 0 && !predictionsLoading) { - return No classes predicted. + /** Summary of a collapsed row: the most specific classes the ensemble predicted. */ + const renderClassSummary = (row) => { + const data = row.direct_parents; + if (data === null) return Could not process input!; + if (!data || (data.length === 0 && !predictionsLoading)) { + return No classes predicted.; } - - // Collapsed view: chips flowing inline - const collapsedView = ( + return ( {data.map((x, idx) => ( - - - + ))} ); + }; - // Expanded view: one class per row with a Why button and collapsible table - const expandedView = ( - - {data.map((x, idx) => { - const whyOpen = isWhyOpen(row.id, idx); - return ( - - - - {lastPredictedModel === 'Ensemble' && ( - - )} - - - - - The ensemble decides by a weighted voting of models. - Each models receives a score (the product of confidence, trust and model weight). - Scores of models that make positive predictions are added, scores of models that make negative predictions are substracted. - If the sum is positive, the ensemble predicts this class. - - - - - Model - Prediction - Confidence - Trust - Model weight - Model score - - - - {Object.entries(x[2] || {}).map(([k, v]) => ( - 0 ? '#a8f099' : '#f0a699'}}> - {k} - {String(v?.prediction)} - {typeof v?.confidence === 'number' ? v.confidence.toFixed(3) : String(v?.confidence)} - {typeof v?.trust === 'number' ? v.trust.toFixed(3) : String(v?.trust)} - {typeof v?.model_weight === 'number' ? v.model_weight.toFixed(3) : String(v?.model_weight)} - {typeof v?.model_score === 'number' ? v.model_score.toFixed(3) : String(v?.model_score)} - - ))} - - Ensemble - true - - - - {typeof x[3] === 'number' ? x[3].toFixed(3) : String(x[3])} - - -
- - - - ); - })} - - ); + /** The class selected in the ontology graph, and why the ensemble predicted it. */ + const renderSelectedClass = (row) => { + if (row.direct_parents === null) return Could not process input!; + const selected = selectedClassByRow[row.id]; + const explanation = selected && (row.explanations || {})[selected]; + if (!selected) { + return ( + + Click on a node in the ontology graph. + + ); + } + if (!explanation) { + // the top class is drawn but never predicted - it holds for every molecule + return ( + + Every molecule is a molecular entity, so the ensemble does not predict this class. + + ); + } return ( - - - {collapsedView} - - - {expandedView} - + + + + {explanation.near_miss && ( + + not predicted - the score stayed below the threshold + + )} + + ); }; @@ -253,7 +246,10 @@ export default function ClassificationGrid() { const thisRow = rows.find((row) => row.id === id); if (!thisRow) return; setDetailsLoading(id); - axios.post('/api/details', {smiles: thisRow.smiles, selectedModels: buildSelectedModels()}).then(response => { + axios.post('/api/details', { + smiles: thisRow.resolved_smiles || thisRow.smiles, + selectedModels: buildSelectedModels() + }).then(response => { const detailObj = { models_info: response.data.models, chebi: response.data.classification, @@ -285,25 +281,7 @@ export default function ClassificationGrid() { addRows(smiles); // Auto-run prediction if models are loaded and we're not already loading if (modelsLoaded && !predictionsLoading) { - setLastPredictedModel(selectedModel); - setHasPredicted(true); - setPredictionsLoading(true); - axios({ - url: '/api/classify', - method: 'post', - data: { - smiles: smiles, - ontology: true, - selectedModels: selectedModels - } - }).then(response => { - setRows((old) => old.map((row, i) => ({ - ...row, - direct_parents: response.data.direct_parents[i], - predicted_parents: response.data.predicted_parents[i], - ontology: response.data.ontology[0][i], - }))); - }).finally(() => setPredictionsLoading(false)); + runPrediction(smiles); } else { // Queue the SMILES to auto-run once models are loaded / ready setQueuedSmiles(smiles); @@ -318,7 +296,7 @@ export default function ClassificationGrid() { event.preventDefault(); const fileData = JSON.stringify(rows.map((r) => ({ "smiles": r["smiles"], - "direct_parents": r["direct_parents"].map(element => [element[0], element[1]]), + "direct_parents": (r["direct_parents"] || []).map(element => [element[0], element[1]]), "predicted_parents": r["predicted_parents"], })).filter((d) => d.direct_parents?.length >= 0)); const blob = new Blob([fileData], {type: "text/plain"}); @@ -348,13 +326,14 @@ export default function ClassificationGrid() { setInputText(e.target.value)} @@ -392,29 +371,7 @@ export default function ClassificationGrid() { if (e.key === 'Enter' && e.ctrlKey) { e.preventDefault(); if (!modelsLoaded || predictionsLoading) return; - const smiles = inputText.trim().replace(/\r/g, '').split('\n').map(s => s.trim()).filter(Boolean); - if (smiles.length === 0) return; - // initialize rows and run classification - addRows(smiles); - setLastPredictedModel(selectedModel); - setHasPredicted(true); - setPredictionsLoading(true); - axios({ - url: '/api/classify', - method: 'post', - data: { - smiles: smiles, - ontology: true, - selectedModels: selectedModels - } - }).then(response => { - setRows((old) => old.map((row, i) => ({ - ...row, - direct_parents: response.data.direct_parents[i], - predicted_parents: response.data.predicted_parents[i], - ontology: response.data.ontology[0][i], - }))); - }).finally(() => setPredictionsLoading(false)); + predictFromInput(); } }} fullWidth @@ -467,6 +424,20 @@ export default function ClassificationGrid() { ))} + {/* model weights only have an effect when the models actually vote against + each other, i.e. when the ensemble is selected */} + {selectedModel === 'Ensemble' && ( + 0} + onChange={(model, value) => setModelWeights(prev => ({...prev, [model]: value}))} + onReset={() => setModelWeights({...defaultModelWeights})} + onRerun={() => runPrediction(rows.map(row => row.smiles))} + /> + )} {/* Hidden file input for SMILES upload */} - + + )} + + - - Molecular graph - + + Predicted class + {renderSelectedClass(row)} - - Ontology graph - {plot_ontology(row.ontology, true, false)} + + Molecular graph + - + Model-specific insights {detailsByRow[row.id] ? ( ); })} - )} - + )} diff --git a/react-app/src/smiles-form/details-page-chemlog.js b/react-app/src/smiles-form/details-page-chemlog.js index 696f1c2..1098a92 100644 --- a/react-app/src/smiles-form/details-page-chemlog.js +++ b/react-app/src/smiles-form/details-page-chemlog.js @@ -25,7 +25,6 @@ import TextField from '@mui/material/TextField'; import Typography from '@mui/material/Typography'; import {styled} from '@mui/material/styles'; -import {plot_ontology} from "./ontology-utils"; import Alert from "@mui/material/Alert"; diff --git a/react-app/src/smiles-form/details-page.js b/react-app/src/smiles-form/details-page.js index e2ffa26..1f359a7 100644 --- a/react-app/src/smiles-form/details-page.js +++ b/react-app/src/smiles-form/details-page.js @@ -7,7 +7,6 @@ import Box from '@mui/material/Box'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import Typography from '@mui/material/Typography'; -import {plot_ontology} from "./ontology-utils"; import {DetailsElectra} from "./details-electra"; import {DetailsBlockwise} from "./details-page-chemlog"; diff --git a/react-app/src/smiles-form/ensemble-settings.js b/react-app/src/smiles-form/ensemble-settings.js new file mode 100644 index 0000000..af88469 --- /dev/null +++ b/react-app/src/smiles-form/ensemble-settings.js @@ -0,0 +1,99 @@ +import * as React from 'react'; +import Badge from '@mui/material/Badge'; +import Box from '@mui/material/Box'; +import Button from '@mui/material/Button'; +import IconButton from '@mui/material/IconButton'; +import Popover from '@mui/material/Popover'; +import RestartAltIcon from '@mui/icons-material/RestartAlt'; +import Slider from '@mui/material/Slider'; +import Tooltip from '@mui/material/Tooltip'; +import TuneIcon from '@mui/icons-material/Tune'; +import Typography from '@mui/material/Typography'; + +/** + * Popover for tuning how much say each base learner has in the ensemble vote. The weight scales a + * model's votes for every class, on top of the confidence and the class-wise reliability the + * ensemble weights them by. A weight of 0 silences a model without removing it. + */ +export default function EnsembleSettings({ + models, + weights, + defaultWeights, + onChange, + onReset, + onRerun, + canRerun, + disabled, + }) { + const [anchorEl, setAnchorEl] = React.useState(null); + if (!models || models.length === 0) return null; + + const weightOf = (model) => Number(weights[model] ?? defaultWeights[model] ?? 1); + const changed = models.filter((model) => weightOf(model) !== Number(defaultWeights[model] ?? 1)); + + return ( + <> + + + setAnchorEl(e.currentTarget)} + disabled={disabled} + aria-label="Ensemble settings" + sx={{height: 36, width: 36}} + > + + + + + + + setAnchorEl(null)} + anchorOrigin={{vertical: 'bottom', horizontal: 'left'}} + slotProps={{paper: {sx: {p: 2, width: 420, maxWidth: '90vw'}}}} + > + Model weights + + {models.map((model) => ( + + {model} + onChange(model, value)} + valueLabelDisplay="auto" + aria-label={`Model weight for ${model}`} + /> + + {weightOf(model)} + + + ))} + + + + {canRerun && ( + + )} + + + + ); +} diff --git a/react-app/src/smiles-form/ontology-utils.js b/react-app/src/smiles-form/ontology-utils.js index 509ff3a..2c89fd4 100644 --- a/react-app/src/smiles-form/ontology-utils.js +++ b/react-app/src/smiles-form/ontology-utils.js @@ -2,184 +2,157 @@ import {useEffect, useRef} from "react"; import * as React from 'react'; import {Network} from "vis-network"; -import ArrowDownwardIcon from '@mui/icons-material/ArrowDownward'; -import ArrowUpwardIcon from '@mui/icons-material/ArrowUpward'; import Box from '@mui/material/Box'; -import Checkbox from '@mui/material/Checkbox'; -import FormGroup from '@mui/material/FormGroup'; -import FormControlLabel from '@mui/material/FormControlLabel'; -import Grid from '@mui/material/Grid'; -import Link from '@mui/material/Link'; -import List from '@mui/material/List'; -import ListItem from '@mui/material/ListItem'; -import ListItemText from '@mui/material/ListItemText'; -import ListSubheader from '@mui/material/ListSubheader'; +import Button from '@mui/material/Button'; +import CenterFocusStrongIcon from '@mui/icons-material/CenterFocusStrong'; +import Tooltip from '@mui/material/Tooltip'; import Typography from '@mui/material/Typography'; import { loadRDKit } from '../lib/rdkit-loader'; import Alert from "@mui/material/Alert"; +const MIN_SCALE = 0.62; +// grey, and dashed, so a near miss is never mistaken for a prediction +const NEAR_MISS_COLOR = {background: '#f0efec', border: '#a8a69c', highlight: {background: '#e4e2dc', border: '#8a887e'}}; +const ROOT_COLOR = {background: '#eceae3', border: '#c3bfb2', highlight: {background: '#e2dfd5', border: '#a9a496'}}; +const CLASS_COLOR = {background: '#dbe8fa', border: '#2a78d6', highlight: {background: '#bcd6f6', border: '#1b5ea9'}}; + +const GRAPH_OPTIONS = { + layout: { + hierarchical: { + enabled: true, + // the top class sits at the top and the hierarchy grows downwards from it + direction: 'UD', + sortMethod: 'directed', + levelSeparation: 62, + nodeSpacing: 150, + treeSpacing: 110, + parentCentralization: true, + }, + }, + physics: false, + nodes: { + shape: 'box', + margin: 6, + widthConstraint: {maximum: 140}, + borderWidth: 1, + font: {size: 13, face: 'inherit', color: '#1a1a1a', multi: false}, + shadow: false, + }, + edges: { + arrows: {to: {enabled: true, scaleFactor: 0.6}}, + color: {color: '#9fb3c8', highlight: '#2a78d6'}, + smooth: {enabled: true, type: 'cubicBezier', forceDirection: 'vertical', roundness: 0.55}, + width: 1, + }, + interaction: { + dragNodes: false, + dragView: true, + zoomView: true, + hover: true, + selectConnectedEdges: false, + tooltipDelay: 300, + }, +}; -function buildNode(id, node, node_color=false, includeLabel=true){ - const d = {id:id} - d["title"] = node["name"] - if(!node.artificial){ - d["name"] = d["title"] - } else { - d["color"] = "#c4c4c0" - } - if (node_color !== false) { - d["color"] = node_color; - } - return d -} - -function buildEdge(id, edge){ - return {from: edge[0], to:edge[1], arrows:{to:true}} -} - -function renderClassListElement(s, node){ - const node_id = String(s) - return ({node["title"] || node_id}) -} - -function subheader(list){ - return list.map((e) => ( - ( - {e} - ) - )) - } - -function renderOverview(node, graph){ - - if(graph == null || node == null){ - - return Select a class - - } - const nodeDict = Object.fromEntries(graph.nodes.map(x => [x["id"], x])); - const superclasses = graph.edges.filter((e) => (e["from"] === node)).map((e) => renderClassListElement(e["to"], nodeDict[e["to"]])) - const subclasses = graph.edges.filter((e) => (e["to"] === node)).map((e) => renderClassListElement(e["from"], nodeDict[e["from"]])) - - return ( - - - This class - - - {renderClassListElement(node, nodeDict[node])} - - - Superclasses - - {subheader(superclasses)} - - Subclasses - - {subheader(subclasses)} - - ) -} - -export function VisNetwork(data) { - +/** + * The predicted part of the ChEBI hierarchy, drawn top-down from the class every molecule belongs + * to. Selecting a node reports it to the parent, which shows why that class was predicted. + */ +export function OntologyGraph({graph, selected, onSelect, showNearMisses = false, height = '620px'}) { const visJsRef = useRef(null); - const [selectedNode, setSelectedNode] = React.useState(null); - const [graph, setGraph] = React.useState(null); - - const [hierarchical, setHierarchical] = React.useState(true); - + const networkRef = useRef(null); + // kept in refs so that re-rendering the parent does not rebuild the network + const onSelectRef = useRef(onSelect); + onSelectRef.current = onSelect; + const selectedRef = useRef(selected); + selectedRef.current = selected; + + const fitView = React.useCallback(() => { + const network = networkRef.current; + if (!network) return; + network.fit(); + // fit() alone shrinks a deep hierarchy until the labels are unreadable; below this scale + // the graph is pannable instead + if (network.getScale() < MIN_SCALE) network.moveTo({scale: MIN_SCALE}); + }, []); useEffect(() => { - var layout = null; - var physics = false; - if(hierarchical){ - layout={ - hierarchical: { - enabled: true, - direction: "RL", - sortMethod: "directed", - levelSeparation: 150, - } + if (!graph || !visJsRef.current) return undefined; + const rootId = Object.keys(graph.nodes).find((id) => graph.nodes[id].root); + const visible = new Set( + Object.keys(graph.nodes).filter((id) => showNearMisses || !graph.nodes[id].near_miss) + ); + const nodes = [...visible].map((id) => { + const node = graph.nodes[id]; + const visNode = { + id, + label: node.name || id, + title: node.root + ? 'Every molecule is a molecular entity' + : node.near_miss ? `${node.name} (not predicted)` : node.name, + color: node.root ? ROOT_COLOR : node.near_miss ? NEAR_MISS_COLOR : CLASS_COLOR, + }; + if (node.near_miss) { + // vis merges these into its defaults, so they are only set where they apply - + // handing it `undefined` replaces the default object and breaks rendering + visNode.font = {color: '#5c5b56'}; + visNode.shapeProperties = {borderDashes: [4, 3]}; } - } else { - layout={ - hierarchical: false, - improvedLayout: false - } - physics = true; - } - - const interaction = { - dragNodes:true, - dragView: true, - hideEdgesOnDrag: false, - hideEdgesOnZoom: false, - hideNodesOnDrag: false, - hover: true, - hoverConnectedEdges: true, - keyboard: { - enabled: false, - speed: {x: 10, y: 10, zoom: 0.02}, - bindToWindow: true, - autoFocus: true, - }, - multiselect: false, - navigationButtons: false, - selectable: true, - selectConnectedEdges: true, - tooltipDelay: 300, - zoomSpeed: 1, - zoomView: true + return visNode; + }); + + const hierarchy = graph.edges.filter(([child, parent]) => visible.has(child) && visible.has(parent)); + // a class whose only superclass is hidden would float free, so it falls back to the root + const hasSuperclass = new Set(hierarchy.map(([child]) => child)); + const orphans = [...visible].filter((id) => id !== rootId && !hasSuperclass.has(id)); + // is-a edges point from a class to its superclass, while the layout grows downwards from + // the superclass - so the edges are handed to vis the other way round + const edges = [...hierarchy, ...orphans.map((id) => [id, rootId])] + .map(([child, parent]) => ({from: parent, to: child})); + + const network = new Network(visJsRef.current, {nodes, edges}, {...GRAPH_OPTIONS, height}); + networkRef.current = network; + network.on('selectNode', (params) => onSelectRef.current(params.nodes[0] || null)); + network.on('deselectNode', () => onSelectRef.current(null)); + fitView(); + if (selectedRef.current && visible.has(selectedRef.current)) { + network.selectNodes([selectedRef.current]); } + return () => { + network.destroy(); + networkRef.current = null; + }; + }, [graph, height, showNearMisses, fitView]); - if(data.graph != null){ - const g = { - nodes: Object.keys(data.graph.nodes).map(k => buildNode(k, data.graph.nodes[k], - ("node_colors" in data.graph && k in data.graph.node_colors) ? data.graph.node_colors[k] : false)), - edges: Object.keys(data.graph.edges).map(k => buildEdge(k, data.graph.edges[k])) - } - const network = - visJsRef.current && - new Network(visJsRef.current, g, { - physics: {enabled: physics}, - layout: layout, - interaction: interaction, - width: data.width || "100%", - height: data.height || "100%", - clickToUse: true - }); - network.fit(); - network.on("selectNode", function (params) { - setSelectedNode(params.nodes[0] || null); - }); - setGraph(g) - } - - }, [visJsRef, data, hierarchical]); - return - -
- - - Graph settings - - } onChange={() => setHierarchical(!hierarchical)} label="Hierarchical (disabling this may take a while)" /> - -
- Node info - {renderOverview(selectedNode, graph)} - - -}; - -export function plot_ontology(graph) { - - if(graph){ - return - } else { - return ; - } + // follow a selection that was cleared or set from outside the graph + useEffect(() => { + if (!networkRef.current) return; + networkRef.current.selectNodes(selected ? [selected] : []); + }, [selected]); + + if (!graph) return null; + + return ( + + +
+ + + + + + Click a class to see why it was predicted. Scroll to zoom, drag to pan. + + + ); } export function MoleculeStructure(data) { From e25082af8d6b9596ae200e68329d598ab7249053 Mon Sep 17 00:00:00 2001 From: sfluegel Date: Tue, 18 Aug 2026 20:55:20 +0200 Subject: [PATCH 2/8] ui tweaks, rdkit.js load fix, precision/recall settings --- README.md | 5 + backend/api/chemclass.py | 86 +++++++++++++--- backend/config.template.json | 1 + react-app/public/index.html | 1 - react-app/src/About.js | 16 ++- .../src/smiles-form/classification-form.js | 62 ++++++++++-- .../src/smiles-form/details-page-chemlog.js | 33 +++---- .../src/smiles-form/ensemble-settings.js | 97 ++++++++++++++++++- react-app/src/smiles-form/ontology-utils.js | 4 + 9 files changed, 260 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 0b41e07..aa896a5 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,11 @@ Besides the models, the configuration points at the calibration of the ensemble: * `ENSEMBLE_CLASSES`: the class list the ensemble was calibrated on, one ChEBI id per line. The class-wise F1 scores are stored positionally, so this list has to match the calibration exactly. * `CHEBI_GRAPH`: the ChEBI graph pickle the calibration was built against. If the file is missing, it is downloaded from [Hugging Face](https://huggingface.co/datasets/chebai/chebifier). * `INCONSISTENCY_RESOLUTION`: `score-based` (the default) or `none`. + * `PR_CURVE` (optional): a CSV of measured operating points (`threshold`, `full_micro_precision`, + `full_micro_recall`), as written by the evaluation grid. It backs the precision/recall sliders in + the ensemble settings: the user asks for more precision or more recall, and the app picks the + threshold that delivered it on the test set. Without the file the operating point stays fixed at + the one the ensemble reports. ### How a prediction is explained diff --git a/backend/api/chemclass.py b/backend/api/chemclass.py index f147bc9..40ece7a 100644 --- a/backend/api/chemclass.py +++ b/backend/api/chemclass.py @@ -1,4 +1,7 @@ +import copy +import csv import math +import os import matplotlib as mpl import torch @@ -85,15 +88,56 @@ def set_label_names(self, label_names): if app.config["INCONSISTENCY_RESOLUTION"] == "score-based" else None ) +def read_operating_points(): + """The measured precision/recall of the ensemble at a range of decision thresholds. + + This is what lets a user ask for "more precision" instead of naming a threshold: each row is an + operating point the ensemble was actually evaluated at, so the number the slider shows was + measured rather than promised. Without the file the operating point stays fixed. + """ + path = app.config.get("PR_CURVE") + if not path or not os.path.exists(path): + print(f"No precision/recall curve at {path}, the decision threshold stays fixed.") + return [] + with open(path, "r", encoding="utf-8") as f: + rows = [ + { + "threshold": float(row["threshold"]), + "precision": float(row["full_micro_precision"]), + "recall": float(row["full_micro_recall"]), + } + for row in csv.DictReader(f) + ] + rows.sort(key=lambda row: row["threshold"]) + print(f"Loaded {len(rows)} operating points from {path}.") + return rows + + +OPERATING_POINTS = read_operating_points() + + +def requested_threshold(threshold): + """A threshold from the request, clamped to the range the ensemble was evaluated over.""" + if threshold is None or not OPERATING_POINTS: + return DECISION_THRESHOLD + try: + threshold = float(threshold) + except (TypeError, ValueError): + return DECISION_THRESHOLD + return min( + max(threshold, OPERATING_POINTS[0]["threshold"]), + OPERATING_POINTS[-1]["threshold"], + ) + + # Classes that came close to being predicted are offered alongside the prediction. "Close" is a # fraction of the ensemble's own operating point, so it follows the threshold if that ever moves. NEAR_MISS_FRACTION = 0.5 -NEAR_MISS_THRESHOLD = DECISION_THRESHOLD * NEAR_MISS_FRACTION NEAR_MISS_LIMIT = 10 print( f"Ensemble ready: {len(MODELS)} models, {len(ENSEMBLE_CLASSES)} classes, " - f"decision threshold {DECISION_THRESHOLD} (near misses from {NEAR_MISS_THRESHOLD})." + f"decision threshold {DECISION_THRESHOLD}." ) @@ -151,7 +195,7 @@ def readable_rows(molecules): ] -def near_miss_classes(aggregated, row): +def near_miss_classes(aggregated, row, threshold): """The highest scoring classes that stayed below the decision threshold. Only classes some model actually covered can be near misses - a class no model said anything @@ -161,7 +205,7 @@ def near_miss_classes(aggregated, row): candidates = ( ~aggregated["class_decisions"][row] & aggregated["has_valid_predictions"][row] - & (scores > NEAR_MISS_THRESHOLD) + & (scores > threshold * NEAR_MISS_FRACTION) ) class_indices = torch.nonzero(candidates).flatten() if class_indices.numel() == 0: @@ -172,7 +216,7 @@ def near_miss_classes(aggregated, row): return [ENSEMBLE_CLASSES[class_idx] for class_idx in ranked.tolist()] -def run_ensemble(smiles_list, selected_models, model_weights): +def run_ensemble(smiles_list, selected_models, model_weights, threshold, resolve=True): """Base learner predictions, ensemble aggregation and inconsistency resolution for a batch of SMILES strings. Returns the per-model predictions alongside the aggregated result.""" models = {name: MODELS[name] for name in selected_model_names(selected_models)} @@ -185,15 +229,21 @@ def run_ensemble(smiles_list, selected_models, model_weights): aggregated = ENSEMBLE.with_weights(model_weights).predict( predictions, attribution=True ) - if SMOOTHER is not None: + if SMOOTHER is not None and resolve: + smoother = SMOOTHER + if threshold != SMOOTHER.threshold: + # the smoother compares scores against the operating point, so it has to move with it - + # on a copy, since requests can overlap + smoother = copy.copy(SMOOTHER) + smoother.threshold = threshold aggregated = apply_inconsistency_resolution( - SMOOTHER, + smoother, ENSEMBLE_CLASSES, aggregated, - decision_threshold=DECISION_THRESHOLD, + decision_threshold=threshold, ) aggregated["class_decisions"] = ( - aggregated["net_score"] > DECISION_THRESHOLD + aggregated["net_score"] > threshold ) & aggregated["has_valid_predictions"] aggregated["complete_failure"] = torch.all( ~aggregated["has_valid_predictions"], dim=1 @@ -211,6 +261,7 @@ def get(self): ], "default_model_weights": DEFAULT_MODEL_WEIGHTS, "decision_threshold": DECISION_THRESHOLD, + "operating_points": OPERATING_POINTS, "n_classes": len(ENSEMBLE_CLASSES), } @@ -223,7 +274,10 @@ def post(self): "smiles": [ ... list of SMILES or InChI strings], "ontology": bool (Optional), "selectedModels": {model name: bool} (Optional), - "modelWeights": {model name: number} (Optional, overrides the configured weights) + "modelWeights": {model name: number} (Optional, overrides the configured weights), + "decisionThreshold": number (Optional, overrides the ensemble's operating point), + "resolveInconsistencies": bool (Optional, default true - resolve predictions that + contradict the ChEBI hierarchy or its disjointness axioms) } :return: A dictionary with the following structure @@ -245,12 +299,18 @@ def post(self): parser.add_argument("ontology", type=bool, required=False, default=False) parser.add_argument("selectedModels", type=dict, required=False, default=None) parser.add_argument("modelWeights", type=dict, required=False, default=None) + parser.add_argument("decisionThreshold", type=float, required=False, default=None) + parser.add_argument( + "resolveInconsistencies", type=bool, required=False, default=True + ) args = parser.parse_args() smiles = args["smiles"] generate_ontology = args["ontology"] + threshold = requested_threshold(args["decisionThreshold"]) if not smiles or len(smiles) == 0: result = { + "decision_threshold": threshold, "predicted_parents": [], "direct_parents": [], "explanations": [], @@ -269,6 +329,8 @@ def post(self): [resolved_smiles[index] for index in rows], args["selectedModels"], requested_weights(args["modelWeights"]), + threshold, + args["resolveInconsistencies"], ) model_names = list(predictions) attribution = aggregated["attribution"] @@ -290,7 +352,7 @@ def post(self): ENSEMBLE_CLASSES[class_idx] for class_idx in torch.nonzero(decisions).flatten().tolist() ] - near_misses = near_miss_classes(aggregated, row) + near_misses = near_miss_classes(aggregated, row, threshold) predicted_parents.append(predicted) ontologies.append( to_vis_graph(predicted, near_misses) if generate_ontology else None @@ -328,6 +390,8 @@ def post(self): explanations.append(explanations_for_smiles) result = { + # the operating point the decisions were taken at, which the request may have moved + "decision_threshold": threshold, "predicted_parents": predicted_parents, "direct_parents": direct_parents, "explanations": explanations, diff --git a/backend/config.template.json b/backend/config.template.json index d9b02bf..eaab79b 100644 --- a/backend/config.template.json +++ b/backend/config.template.json @@ -4,6 +4,7 @@ "CHEBI_GRAPH": "path/to/chebi_graph_v252.pkl (optional, downloaded from Hugging Face if missing)", "ENSEMBLE_DIR": "directory holding the ensemble calibration (prediction_thresholds.yaml, _classwise_f1.txt, best_hyperparameters.csv)", "ENSEMBLE_CLASSES": "file listing the classes the ensemble was calibrated on, one per line", + "PR_CURVE": "precision/recall curve of the ensemble over the decision threshold (optional; enables the precision/recall sliders)", "INCONSISTENCY_RESOLUTION": "score-based", "MODELS": { "My model": { diff --git a/react-app/public/index.html b/react-app/public/index.html index b557530..f7d9daa 100644 --- a/react-app/public/index.html +++ b/react-app/public/index.html @@ -26,7 +26,6 @@ --> Chebifier - diff --git a/react-app/src/About.js b/react-app/src/About.js index c5d8f30..7660028 100644 --- a/react-app/src/About.js +++ b/react-app/src/About.js @@ -93,8 +93,11 @@ const About = () => { Clicking a class in the ontology graph of a result shows how much of the decision - each model is responsible for. Details about the ensemble and its implementation can - be found here. + each model is responsible for. The ensemble settings also let you trade precision + against recall - an experimental feature, whose percentages come from the ChEBI test + set and will be optimistic for unusual molecules and rare classes. Details about the + ensemble and its implementation can be + found here. @@ -119,6 +122,15 @@ const About = () => { ))} +
+ + Chebifier does not collect or store the molecules you submit. A SMILES or InChI + string you enter is used to compute the prediction you asked for and nothing else: + it is never written to disk, never kept after the request, and never passed on to + anyone else. + +
+
{[ ['08/2026', 'Re-calibrated ensemble. Added new, better deep learning models trained on ChEBI ' + diff --git a/react-app/src/smiles-form/classification-form.js b/react-app/src/smiles-form/classification-form.js index 4c790f2..c210fc4 100644 --- a/react-app/src/smiles-form/classification-form.js +++ b/react-app/src/smiles-form/classification-form.js @@ -51,6 +51,10 @@ export default function ClassificationGrid() { const [defaultModelWeights, setDefaultModelWeights] = React.useState({}); const [modelWeights, setModelWeights] = React.useState({}); const [decisionThreshold, setDecisionThreshold] = React.useState(0.5); + const [defaultThreshold, setDefaultThreshold] = React.useState(0.5); + const [operatingPoints, setOperatingPoints] = React.useState([]); + // predictions that contradict the ChEBI hierarchy are corrected against each other by default + const [resolveInconsistencies, setResolveInconsistencies] = React.useState(true); const [inputText, setInputText] = React.useState(""); const [predictionsLoading, setPredictionsLoading] = React.useState(false); @@ -94,7 +98,9 @@ export default function ClassificationGrid() { setModelWeights({...weights}); if (typeof response.data.decision_threshold === 'number') { setDecisionThreshold(response.data.decision_threshold); + setDefaultThreshold(response.data.decision_threshold); } + setOperatingPoints(response.data.operating_points || []); setModelsLoaded(true); }); }, []); @@ -116,7 +122,9 @@ export default function ClassificationGrid() { smiles: smiles, ontology: true, selectedModels: buildSelectedModels(), - modelWeights: modelWeights + modelWeights: modelWeights, + decisionThreshold: decisionThreshold, + resolveInconsistencies: resolveInconsistencies } }).then(response => { setRows((old) => old.map((row, i) => ({ @@ -128,6 +136,7 @@ export default function ClassificationGrid() { // what the backend read the input as - an InChI comes back translated to SMILES, which is // what the structure drawing and the per-model insights need resolved_smiles: (response.data.smiles || [])[i], + threshold: response.data.decision_threshold, }))); }).finally(() => setPredictionsLoading(false)); }; @@ -216,7 +225,7 @@ export default function ClassificationGrid() { ); @@ -434,7 +443,17 @@ export default function ClassificationGrid() { disabled={!modelsLoaded || predictionsLoading} canRerun={rows.length > 0} onChange={(model, value) => setModelWeights(prev => ({...prev, [model]: value}))} - onReset={() => setModelWeights({...defaultModelWeights})} + onReset={() => { + setModelWeights({...defaultModelWeights}); + setDecisionThreshold(defaultThreshold); + setResolveInconsistencies(true); + }} + operatingPoints={operatingPoints} + threshold={decisionThreshold} + defaultThreshold={defaultThreshold} + onThresholdChange={setDecisionThreshold} + resolveInconsistencies={resolveInconsistencies} + onResolveChange={setResolveInconsistencies} onRerun={() => runPrediction(rows.map(row => row.smiles))} /> )} @@ -482,6 +501,11 @@ export default function ClassificationGrid() { Predict + + Chebifier does not collect or store the molecules you submit. They are + processed only to compute the prediction and are never written to disk or + passed on to anyone else. + @@ -490,13 +514,21 @@ export default function ClassificationGrid() { {hasPredicted && ( {rows.length > 0 && ( - + {rows.map((row) => { const canExpand = (row.direct_parents) && !predictionsLoading; return ( - {row.smiles} + + {row.smiles} + {expandedRowId !== row.id && ( - - {renderClassSummary(row)} + + {/* nothing to draw for an input the backend could not read */} + {row.resolved_smiles && ( + + + + )} + + {renderClassSummary(row)} + )} {expandedRowId === row.id && ( diff --git a/react-app/src/smiles-form/details-page-chemlog.js b/react-app/src/smiles-form/details-page-chemlog.js index 1098a92..d21beff 100644 --- a/react-app/src/smiles-form/details-page-chemlog.js +++ b/react-app/src/smiles-form/details-page-chemlog.js @@ -26,27 +26,13 @@ import Typography from '@mui/material/Typography'; import {styled} from '@mui/material/styles'; import Alert from "@mui/material/Alert"; +import { loadRDKit } from '../lib/rdkit-loader'; const GLOBAL_MOL_PARAMS = { width: 300, height: 300, } -window - .initRDKitModule() - .then(function (RDKit) { - console.log("RDKit version: " + RDKit.version()); - window.RDKit = RDKit; - /** - * The RDKit module is now loaded. - * You can use it anywhere. - */ - }) - .catch(() => { - // handle loading errors here... - }); - - const NetworkElement = (data) => { const visJsRef = useRef(null); useEffect(() => { @@ -173,10 +159,21 @@ export function HighlightsBlocks(data) { export function DetailsBlockwise(data) { const handleClose = data.handleClose; data = data.model_data; - var smiles = data.smiles + const smiles = data.smiles; + // RDKit is fetched once for the whole app, so it may not be there on the first render + const [rdkit, setRdkit] = React.useState(window.RDKit || null); + React.useEffect(() => { + let mounted = true; + loadRDKit().then((module) => { + if (mounted) setRdkit(module); + }).catch(() => {}); + return () => { + mounted = false; + }; + }, []); var mol = null; - if (!(smiles === null || smiles === undefined)) { - mol = window.RDKit.get_mol(smiles); + if (rdkit && !(smiles === null || smiles === undefined)) { + mol = rdkit.get_mol(smiles); } //var svg_mol = mol.get_svg_with_highlights(JSON.stringify(GLOBAL_MOL_PARAMS)); //svg_mol = svg_mol.substring(svg_mol.indexOf(" Number(weights[model] ?? defaultWeights[model] ?? 1); const changed = models.filter((model) => weightOf(model) !== Number(defaultWeights[model] ?? 1)); + // The operating points are ordered by threshold, so precision rises and recall falls along the + // list. Both sliders address the same index, which is what makes one give way to the other. + const points = operatingPoints || []; + const last = points.length - 1; + const nearestIndex = (value) => { + let best = 0; + points.forEach((point, index) => { + if (Math.abs(point.threshold - value) < Math.abs(points[best].threshold - value)) best = index; + }); + return best; + }; + const index = points.length ? nearestIndex(threshold) : 0; + const point = points[index]; + const percent = (value) => `${(value * 100).toFixed(1)}%`; + const thresholdChanged = points.length > 0 && index !== nearestIndex(defaultThreshold); + const settingsChanged = changed.length > 0 || thresholdChanged || !resolveInconsistencies; + + const operatingPointSection = points.length === 0 ? null : ( + <> + Precision vs. recall + + Experimental. The percentages are what the ensemble reached on the ChEBI test set - on + molecules unlike those, and on rare classes, it will be less reliable than they suggest. + + + Precision + onThresholdChange(points[value].threshold)} + valueLabelDisplay="auto" + valueLabelFormat={(value) => percent(points[value].precision)} + aria-label="Precision" + /> + + {percent(point.precision)} + + Recall + onThresholdChange(points[last - value].threshold)} + valueLabelDisplay="auto" + valueLabelFormat={(value) => percent(points[last - value].recall)} + aria-label="Recall" + /> + + {percent(point.recall)} + + + + + ); + return ( <> - + - + @@ -53,8 +123,25 @@ export default function EnsembleSettings({ anchorEl={anchorEl} onClose={() => setAnchorEl(null)} anchorOrigin={{vertical: 'bottom', horizontal: 'left'}} - slotProps={{paper: {sx: {p: 2, width: 420, maxWidth: '90vw'}}}} + slotProps={{paper: {sx: {p: 2, width: 440, maxWidth: '90vw'}}}} > + onResolveChange(e.target.checked)} + /> + } + label={ + + Resolve inconsistencies + + } + sx={{ml: 0, mb: 1.5}} + /> + + {operatingPointSection} Model weights {models.map((model) => ( @@ -77,7 +164,7 @@ export default function EnsembleSettings({ ))} - {canRerun && ( @@ -89,7 +176,7 @@ export default function EnsembleSettings({ onRerun(); }} > - Re-run with these weights + Re-run )} diff --git a/react-app/src/smiles-form/ontology-utils.js b/react-app/src/smiles-form/ontology-utils.js index 2c89fd4..e15831d 100644 --- a/react-app/src/smiles-form/ontology-utils.js +++ b/react-app/src/smiles-form/ontology-utils.js @@ -162,6 +162,10 @@ export function MoleculeStructure(data) { React.useEffect(() => { let mounted = true; let mol = null; + // an InChI input is drawn only once the backend has translated it, so a failed first attempt + // has to be forgotten when the string changes - otherwise the error outlives the fix + setSvg(null); + setError(null); async function run() { try { From da28c9fd0590774022ea3f8af5a06246e59e01c2 Mon Sep 17 00:00:00 2001 From: sfluegel Date: Tue, 18 Aug 2026 20:55:50 +0200 Subject: [PATCH 3/8] add backend ensemble (extends WMV-F1) --- backend/api/ensemble.py | 108 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 backend/api/ensemble.py diff --git a/backend/api/ensemble.py b/backend/api/ensemble.py new file mode 100644 index 0000000..0b74cae --- /dev/null +++ b/backend/api/ensemble.py @@ -0,0 +1,108 @@ +"""Ensemble used by the web app: chebifier's WMV-F1 ensemble plus per-model weights. + +Chebifier weights a vote by `confidence * trust`, where trust is the model's class-wise F1 on the +validation set. The `model_weight` that used to scale a model's votes independently of the class is +still read from the configuration by `BasePredictor`, but the ensemble no longer applies it. The +web app needs it back, both to give the symbolic classifiers the say they had before and because +the "Ensemble settings" panel lets a user retune the weights per request. +""" + +import copy +from pathlib import Path + +import torch +from chebifier.ensemble.weighted_majority_ensemble import WMVwithF1Ensemble +from chebifier.inconsistency_resolution import NEUTRAL + +DEFAULT_MODEL_WEIGHT = 1 + + +class WeightedWMVF1Ensemble(WMVwithF1Ensemble): + """WMV-F1 whose votes are additionally scaled by a per-model weight. + + Models the ensemble was not calibrated on (the ChEBI lookup has no validation predictions) + vote with a neutral threshold and full trust, so their influence is set by their weight alone. + + `calibration_names` maps the names the app shows to the names the calibration files in + `ensemble_dir` were written under, so models can be renamed in the configuration without + touching the calibration. + """ + + def __init__( + self, + ensemble_dir: str, + calibration_names: dict[str, str] | None = None, + model_weights: dict[str, float] | None = None, + **kwargs, + ): + super().__init__(ensemble_dir, **kwargs) + self.calibration_names = dict(calibration_names or {}) + self.model_weights = dict(model_weights or {}) + self._f1_cache: dict[str, torch.Tensor | None] = {} + + def with_weights(self, model_weights: dict[str, float] | None): + """A view of this ensemble that votes with the given weights. + + The clone shares the loaded calibration, so this is cheap enough to do per request - and + unlike mutating the shared instance it stays correct when requests overlap. + """ + if not model_weights: + return self + clone = copy.copy(self) + clone.model_weights = {**self.model_weights, **model_weights} + return clone + + def calibration_name(self, model_name: str) -> str: + return self.calibration_names.get(model_name, model_name) + + def model_weight(self, model_name: str) -> float: + return float(self.model_weights.get(model_name, DEFAULT_MODEL_WEIGHT)) + + def _load_prediction_thresholds(self) -> dict[str, float]: + """Thresholds keyed by the names the app uses, with a neutral default for models that + were not part of the calibration.""" + calibrated = super()._load_prediction_thresholds() + thresholds = { + model_name: calibrated.get(self.calibration_name(model_name), NEUTRAL) + for model_name in self.calibration_names + } + # models that are neither renamed nor uncalibrated keep their own entry + return {**calibrated, **thresholds} + + def _classwise_f1(self, model_name: str, num_classes: int) -> torch.Tensor: + if model_name not in self._f1_cache: + path = ( + Path(self.ensemble_dir) + / f"{self.calibration_name(model_name)}_classwise_f1.txt" + ) + if path.exists(): + with open(path, "r", encoding="utf-8") as f: + f1 = torch.tensor([float(x) for x in f.read().splitlines()]) + if f1.shape[0] != num_classes: + raise ValueError( + f"Class-wise F1 scores for {model_name} cover {f1.shape[0]} classes, but " + f"the ensemble runs on {num_classes}. The class list has to be the one the " + f"ensemble was calibrated on." + ) + else: + print( + f"No class-wise F1 scores for {model_name} in {self.ensemble_dir}, " + f"voting with full trust." + ) + f1 = None + self._f1_cache[model_name] = f1 + f1 = self._f1_cache[model_name] + return torch.ones(num_classes) if f1 is None else f1 + + def calculate_trust(self, predictions: dict[str, torch.Tensor]) -> torch.Tensor: + weighting_strength, weighting_exponent = self._load_hyperparameters() + num_molecules, num_classes = next(iter(predictions.values())).shape + trust = torch.ones( + (num_molecules, num_classes, len(predictions)), dtype=torch.float32 + ) + for model_idx, model_name in enumerate(predictions): + classwise_f1 = self._classwise_f1(model_name, num_classes) + trust[:, :, model_idx] = ( + weighting_strength * classwise_f1 + (1 - weighting_strength) + ) ** weighting_exponent * self.model_weight(model_name) + return trust From 4816d7906ab5e41b9e2b3ce84d5ee621a6e2c324 Mon Sep 17 00:00:00 2001 From: sfluegel Date: Tue, 18 Aug 2026 21:02:13 +0200 Subject: [PATCH 4/8] add complain button --- react-app/src/navbar.js | 2 +- .../src/smiles-form/classification-form.js | 81 ++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/react-app/src/navbar.js b/react-app/src/navbar.js index 83920c0..c7c3d52 100644 --- a/react-app/src/navbar.js +++ b/react-app/src/navbar.js @@ -21,7 +21,7 @@ const Navbar = () => { About
  • - Report an Issue + Report an Issue
    • diff --git a/react-app/src/smiles-form/classification-form.js b/react-app/src/smiles-form/classification-form.js index c210fc4..6c7db11 100644 --- a/react-app/src/smiles-form/classification-form.js +++ b/react-app/src/smiles-form/classification-form.js @@ -8,6 +8,7 @@ import Button from '@mui/material/Button'; import Typography from '@mui/material/Typography'; import TextField from '@mui/material/TextField'; import Chip from '@mui/material/Chip'; +import FlagOutlinedIcon from '@mui/icons-material/FlagOutlined'; import LightbulbIcon from '@mui/icons-material/Lightbulb'; import StartIcon from '@mui/icons-material/Start'; import UploadFileIcon from '@mui/icons-material/UploadFile'; @@ -31,6 +32,9 @@ import EnsembleSettings from "./ensemble-settings"; // their fill: the card carries a shadow, the panels on it carry a blue border. export const ACCENT = '#2a78d6'; +// Feedback on a prediction goes to the issue tracker, through the "wrong prediction" issue form. +const FEEDBACK_REPO = 'https://github.com/ChEB-AI/chebifier-web'; + const panelSx = { p: 2, @@ -109,6 +113,11 @@ export default function ClassificationGrid() { // upload, re-run after a weight change) goes through here so they cannot drift apart. const runPrediction = (smiles) => { if (!smiles || smiles.length === 0) return; + const settings = { + models: selectedModel === 'Ensemble' ? 'Ensemble (all models)' : `single model: ${selectedModel}`, + weights: {...modelWeights}, + resolve: resolveInconsistencies, + }; setHasPredicted(true); setPredictionsLoading(true); setExpandedRowId(null); @@ -137,6 +146,7 @@ export default function ClassificationGrid() { // what the structure drawing and the per-model insights need resolved_smiles: (response.data.smiles || [])[i], threshold: response.data.decision_threshold, + settings: {...settings, threshold: response.data.decision_threshold}, }))); }).finally(() => setPredictionsLoading(false)); }; @@ -158,6 +168,60 @@ export default function ClassificationGrid() { // eslint-disable-next-line react-hooks/exhaustive-deps }, [modelsLoaded, queuedSmiles]); + /** + * A link to the "wrong prediction" issue form, with the molecule and the prediction filled in. + * Feedback goes to the issue tracker rather than to us: nothing about a prediction is kept on + * the server, so the report has to carry its own context. + */ + const feedbackUrl = (row) => { + const selected = selectedClassByRow[row.id]; + const explanation = selected && (row.explanations || {})[selected]; + const settings = row.settings || {}; + const changedWeights = Object.entries(settings.weights || {}) + .filter(([model, weight]) => Number(weight) !== Number(defaultModelWeights[model] ?? 1)) + .map(([model, weight]) => `${model}=${weight}`); + + const lines = [`Molecule (as entered): ${row.smiles}`]; + if (row.resolved_smiles && row.resolved_smiles !== row.smiles) { + lines.push(`Molecule (as classified): ${row.resolved_smiles}`); + } + if (explanation) { + lines.push( + '', + `Selected class: ${explanation.name} (CHEBI:${selected})`, + `Ensemble score: ${explanation.score?.toFixed(3)} (predicted above ${settings.threshold ?? decisionThreshold})`, + explanation.near_miss ? 'This class was NOT predicted - it stayed below the threshold.' : '', + 'Model contributions:', + ...Object.entries(explanation.models || {}).map(([model, values]) => + ` ${model}: prediction ${values.prediction?.toFixed(3)}, ` + + `${values.vote > 0 ? 'supports' : 'opposes'}, ` + + `${((values.attribution || 0) * 100).toFixed(1)}% of the decision`), + ); + } + lines.push( + '', + 'Settings:', + ` Models: ${settings.models || 'Ensemble (all models)'}`, + ` Decision threshold: ${settings.threshold ?? decisionThreshold}`, + ` Inconsistency resolution: ${settings.resolve === false ? 'off' : 'on'}`, + ` Model weights: ${changedWeights.length ? changedWeights.join(', ') : 'default'}`, + '', + `All predicted classes: ${(row.predicted_parents || []).map(cls => `CHEBI:${cls}`).join(', ')}`, + ); + + const params = new URLSearchParams({ + template: 'wrong-prediction.yml', + title: `[Prediction] ${explanation ? `${explanation.name} for ` : ''}${row.smiles}`.slice(0, 120), + molecule: row.smiles, + // an over-long URL is rejected by the browser rather than truncated, so cap the dump + prediction: lines.filter(line => line !== '').join('\n').slice(0, 4000), + }); + if (explanation) { + params.set('classes', `CHEBI:${selected} (${explanation.name})`); + } + return `${FEEDBACK_REPO}/issues/new?${params.toString()}`; + }; + /** Whether a row has classes that came close to the threshold without reaching it. */ const hasNearMisses = (row) => Object.values(row.explanations || {}).some((explanation) => explanation.near_miss); @@ -604,7 +668,22 @@ export default function ClassificationGrid() { /> - Predicted class + + Predicted class + + + + {renderSelectedClass(row)} From a73f5f4bd6e553f843bc50466b93e6ba305f90bf Mon Sep 17 00:00:00 2001 From: sfluegel Date: Tue, 18 Aug 2026 21:02:57 +0200 Subject: [PATCH 5/8] add wrong prediction template --- .github/ISSUE_TEMPLATE/wrong-prediction.yml | 55 +++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/wrong-prediction.yml diff --git a/.github/ISSUE_TEMPLATE/wrong-prediction.yml b/.github/ISSUE_TEMPLATE/wrong-prediction.yml new file mode 100644 index 0000000..83efd9d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/wrong-prediction.yml @@ -0,0 +1,55 @@ +name: Wrong prediction +description: Report a ChEBI class Chebifier got wrong - one it predicted that does not apply, or one it missed +title: "[Prediction] " +body: + - type: markdown + attributes: + value: | + Thanks for helping to improve Chebifier. If you arrived here from the "Report a wrong + prediction" button in the web app, the molecule and the prediction details are already + filled in below - please tell us what should have happened instead, and why. + - type: input + id: molecule + attributes: + label: Molecule + description: The SMILES or InChI string that was classified. + placeholder: CC(=O)Oc1ccccc1C(=O)O + validations: + required: true + - type: dropdown + id: kind + attributes: + label: What went wrong? + options: + - A class was predicted that does not apply + - A class that applies was not predicted + - The classification is right but the explanation is not + - Something else + validations: + required: true + - type: textarea + id: classes + attributes: + label: Classes concerned + description: Which ChEBI classes are affected? Please give the ChEBI ids where you can. + placeholder: | + CHEBI:22315 (alkaloid) should not have been predicted. + validations: + required: true + - type: textarea + id: rationale + attributes: + label: Why is this wrong? + description: > + What is the correct classification, and what makes it correct? A definition, a reference or + the structural feature that decides it all help us a great deal. + validations: + required: true + - type: textarea + id: prediction + attributes: + label: Prediction details + description: > + Filled in by the web app - the ensemble score, what each model contributed, and the + settings the prediction was made with. Leave it as it is unless it is empty. + render: text From 19ca41b80d78156d1ad38e216da2f46f1e594445 Mon Sep 17 00:00:00 2001 From: sfluegel Date: Tue, 18 Aug 2026 21:25:45 +0200 Subject: [PATCH 6/8] add usage stats --- README.md | 7 ++ backend/api/chemclass.py | 10 +++ backend/app.py | 3 +- backend/config.template.json | 1 + backend/stats.py | 86 +++++++++++++++++++ react-app/src/About.js | 36 ++++++-- .../src/smiles-form/classification-form.js | 5 -- 7 files changed, 133 insertions(+), 15 deletions(-) create mode 100644 backend/stats.py diff --git a/README.md b/README.md index aa896a5..386f60d 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,13 @@ Besides the models, the configuration points at the calibration of the ensemble: * `ENSEMBLE_CLASSES`: the class list the ensemble was calibrated on, one ChEBI id per line. The class-wise F1 scores are stored positionally, so this list has to match the calibration exactly. * `CHEBI_GRAPH`: the ChEBI graph pickle the calibration was built against. If the file is missing, it is downloaded from [Hugging Face](https://huggingface.co/datasets/chebai/chebifier). * `INCONSISTENCY_RESOLUTION`: `score-based` (the default) or `none`. + * `STATS_DB` (optional): SQLite file the number of classified molecules is counted in, shown in the + app as "x molecules classified since ...". Only a per-day count is stored, never the molecules. + Under uWSGI the request is answered by one of several worker processes, so the count cannot live + in memory - the increment is a single upserting statement in a transaction, which concurrent + workers cannot lose the way a read-modify-write on a plain file would. Put the file on local + disk (SQLite locking is unreliable over NFS) and on a path that survives a deploy. Delete the + file to reset the counter. * `PR_CURVE` (optional): a CSV of measured operating points (`threshold`, `full_micro_precision`, `full_micro_recall`), as written by the evaluation grid. It backs the precision/recall sliders in the ensemble settings: the user asks for more precision or more recall, and the app picks the diff --git a/backend/api/chemclass.py b/backend/api/chemclass.py index 40ece7a..e0cd5af 100644 --- a/backend/api/chemclass.py +++ b/backend/api/chemclass.py @@ -18,6 +18,7 @@ ) from chebi_utils.read_molecule import smiles_or_inchi_to_mol from chebifier.utils import _smiles_to_mol, get_disjoint_files +import stats from ontology import CHEBI_GRAPH, class_name, most_specific, to_vis_graph from rdkit import Chem @@ -266,6 +267,12 @@ def get(self): } +class StatsAPI(Resource): + + def get(self): + return stats.summary() + + class BatchPrediction(Resource): def post(self): """ @@ -389,6 +396,9 @@ def post(self): } explanations.append(explanations_for_smiles) + # an input that could not be read never reached the models, so it is not a prediction + stats.record(sum(1 for parents in predicted_parents if parents is not None)) + result = { # the operating point the decisions were taken at, which the request may have moved "decision_threshold": threshold, diff --git a/backend/app.py b/backend/app.py index 9226d93..e7d8305 100644 --- a/backend/app.py +++ b/backend/app.py @@ -33,10 +33,11 @@ def serve_client_route(error): def load_endpoints(): - from api.chemclass import PredictionDetailApiHandler, BatchPrediction, ModelInfoAPI + from api.chemclass import PredictionDetailApiHandler, BatchPrediction, ModelInfoAPI, StatsAPI api.add_resource(PredictionDetailApiHandler, '/api/details') api.add_resource(BatchPrediction, '/api/classify') api.add_resource(ModelInfoAPI, '/api/modelinfo') + api.add_resource(StatsAPI, '/api/stats') with app.app_context(): mp.set_start_method("spawn") diff --git a/backend/config.template.json b/backend/config.template.json index eaab79b..1902077 100644 --- a/backend/config.template.json +++ b/backend/config.template.json @@ -4,6 +4,7 @@ "CHEBI_GRAPH": "path/to/chebi_graph_v252.pkl (optional, downloaded from Hugging Face if missing)", "ENSEMBLE_DIR": "directory holding the ensemble calibration (prediction_thresholds.yaml, _classwise_f1.txt, best_hyperparameters.csv)", "ENSEMBLE_CLASSES": "file listing the classes the ensemble was calibrated on, one per line", + "STATS_DB": "file the count of classified molecules is kept in (optional)", "PR_CURVE": "precision/recall curve of the ensemble over the decision threshold (optional; enables the precision/recall sliders)", "INCONSISTENCY_RESOLUTION": "score-based", "MODELS": { diff --git a/backend/stats.py b/backend/stats.py new file mode 100644 index 0000000..dec4b93 --- /dev/null +++ b/backend/stats.py @@ -0,0 +1,86 @@ +"""How many molecules Chebifier has classified. + +The web app is served by uWSGI, which answers requests from several worker processes, so the count +cannot live in Python state - each worker would keep its own. It lives in a SQLite file instead: +the increment is a single statement inside a transaction, so concurrent workers cannot lose a +count the way a read-modify-write on a plain file would, and the database survives a crash +mid-write. + +Only the number of molecules and the day it happened are stored - never the molecules themselves, +which the app promises not to keep. +""" + +import os +import sqlite3 +from contextlib import closing +from datetime import datetime, timezone + +from app import app + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS predictions ( + day TEXT PRIMARY KEY, + molecules INTEGER NOT NULL +) +""" + +_path = app.config.get("STATS_DB") + + +def _connect(): + connection = sqlite3.connect(_path, timeout=10) + # readers never block the writer, and the writer never blocks on a reader + connection.execute("PRAGMA journal_mode=WAL") + connection.execute(_SCHEMA) + return connection + + +def _prepare(): + if not _path: + print("No STATS_DB configured, predictions are not counted.") + return False + os.makedirs(os.path.dirname(os.path.abspath(_path)), exist_ok=True) + with closing(_connect()): + pass + print(f"Counting predictions in {_path}.") + return True + + +try: + ENABLED = _prepare() +except sqlite3.Error as error: + print(f"Could not open the prediction counter ({error}), predictions are not counted.") + ENABLED = False + + +def record(molecules: int) -> None: + """Add to the count for today. Never raises - a prediction is worth more than its tally.""" + if not ENABLED or molecules <= 0: + return + day = datetime.now(timezone.utc).strftime("%Y-%m-%d") + try: + # `with connection` commits the transaction but leaves the handle open, so the close has + # to be asked for separately - otherwise every request leaks one + with closing(_connect()) as connection, connection: + connection.execute( + "INSERT INTO predictions (day, molecules) VALUES (?, ?) " + "ON CONFLICT(day) DO UPDATE SET molecules = molecules + excluded.molecules", + (day, molecules), + ) + except sqlite3.Error as error: + print(f"Could not count {molecules} predictions: {error}") + + +def summary() -> dict: + """The total and the day counting started, or an empty summary if nothing was counted yet.""" + if not ENABLED: + return {"molecules": None, "since": None} + try: + with closing(_connect()) as connection: + total, since = connection.execute( + "SELECT SUM(molecules), MIN(day) FROM predictions" + ).fetchone() + except sqlite3.Error as error: + print(f"Could not read the prediction counter: {error}") + return {"molecules": None, "since": None} + return {"molecules": total, "since": since} diff --git a/react-app/src/About.js b/react-app/src/About.js index 7660028..9b5aee5 100644 --- a/react-app/src/About.js +++ b/react-app/src/About.js @@ -36,10 +36,19 @@ const Section = ({title, children}) => ( ); +/** "2026-08-18" as "18 August 2026", without dragging in a date library. */ +const formatDate = (day) => { + const date = new Date(`${day}T00:00:00Z`); + return Number.isNaN(date.getTime()) + ? day + : date.toLocaleDateString('en-GB', {day: 'numeric', month: 'long', year: 'numeric', timeZone: 'UTC'}); +}; + const About = () => { const [availableModels, setAvailableModels] = React.useState([]); const [availableModelsInfoTexts, setAvailableModelsInfoTexts] = React.useState([]); const [numClasses, setNumClasses] = React.useState(null); + const [stats, setStats] = React.useState(null); // Load once on mount so About content is fetched when the site loads React.useEffect(() => { @@ -50,6 +59,7 @@ const About = () => { }).catch(() => { // silently ignore, page content still renders }); + axios.get('/api/stats').then(response => setStats(response.data)).catch(() => {}); }, []); return ( @@ -122,15 +132,6 @@ const About = () => { ))} -
    - - Chebifier does not collect or store the molecules you submit. A SMILES or InChI - string you enter is used to compute the prediction you asked for and nothing else: - it is never written to disk, never kept after the request, and never passed on to - anyone else. - -
    -
    {[ ['08/2026', 'Re-calibrated ensemble. Added new, better deep learning models trained on ChEBI ' + @@ -157,6 +158,23 @@ const About = () => { href="https://doi.org/10.1039/D3DD00238A">Link
    + +
    + {stats?.molecules > 0 && ( + + Chebifier has classified {stats.molecules.toLocaleString('en-US')} molecules + {stats.since ? ` since ${formatDate(stats.since)}` : ''}. That count is all we + keep: the number of molecules per day, and nothing about the molecules + themselves. + + )} + + Chebifier does not collect or store the molecules you submit. A SMILES or InChI + string you enter is used to compute the prediction you asked for and nothing else: + it is never written to disk, never kept after the request, and never passed on to + anyone else. + +
    diff --git a/react-app/src/smiles-form/classification-form.js b/react-app/src/smiles-form/classification-form.js index 6c7db11..aa006ef 100644 --- a/react-app/src/smiles-form/classification-form.js +++ b/react-app/src/smiles-form/classification-form.js @@ -565,11 +565,6 @@ export default function ClassificationGrid() { Predict - - Chebifier does not collect or store the molecules you submit. They are - processed only to compute the prediction and are never written to disk or - passed on to anyone else. - From 0866608d9d1bd198527725e0f77d8153a7ffe041 Mon Sep 17 00:00:00 2001 From: sfluegel Date: Wed, 19 Aug 2026 09:27:53 +0200 Subject: [PATCH 7/8] minor changes to ui and texts --- README.md | 5 +-- react-app/src/About.js | 35 +++++++++---------- react-app/src/Navbar.css | 15 ++++++++ react-app/src/navbar.js | 7 ++++ .../src/smiles-form/classification-form.js | 27 ++++---------- 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 386f60d..ec3f6f4 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,7 @@ Chebifier is a tool for automated classification of chemicals in the [ChEBI](https://www.ebi.ac.uk/chebi/) ontology. This repository only hosts the front end of Chebifier. For the classification itself, see [python-chebifier](https://github.com/ChEB-AI/python-chebifier). ## News -- 2026/08/18: Moved to the calibrated ensembles of python-chebifier (WMV-F1 on ChEBI v252 3-STAR, - score-based inconsistency resolution). Predictions now come with a 0-1 ensemble score and a - per-model attribution, model weights can be tuned per prediction in "Ensemble settings", and - molecules can be entered as InChI as well as SMILES. +- 2026/08/18: Recalibrated ensemble (with ~500 new classes), added new deep learning models (v252) and model attributions. Now supports InChI input, user feedback, and extended ensemble settings. - 2026/02/16: Added Lopster and new deep learning models. - 2025/11/11: Fixed processing error for GNNs. - 2025/11/05: Added new models (v244, including GAT, 3-STAR models and augmented GNNs), redesigned frontend. diff --git a/react-app/src/About.js b/react-app/src/About.js index 9b5aee5..916fc4e 100644 --- a/react-app/src/About.js +++ b/react-app/src/About.js @@ -72,7 +72,8 @@ const About = () => { display: 'flex', flexDirection: 'column', alignItems: 'center', - py: 2, + pt: 2, + pb: 4, }}>
    @@ -80,13 +81,17 @@ const About = () => { Chebifier is a tool for automated classification of chemicals in the ChEBI ontology. It currently - predicts {numClasses ? numClasses.toLocaleString('en-US') : '2,000+'} ChEBI classes. + predicts {numClasses ? numClasses.toLocaleString('en-US') : '2,200+'} ChEBI classes. - To run a prediction, enter a SMILES or InChI string (or several ones, one per line) - or upload a file, then hit the predict button - running the models takes a few - seconds. Click a result to see the molecule, the predicted part of the ontology and - what each model contributed. + To run a prediction, enter SMILES or InChI strings (one per line) + or upload a file. Running the models might take a few + seconds. Click on a result for more details about the prediction. + + + Chebifier is developed as part of the StrOntEx project. + For more information on Chebifier, checkout the GitHub repository and + our latest publication (Flügel et al, 2026: Chebifier 2).
    @@ -95,7 +100,7 @@ const About = () => { Chebifier combines machine learning models, rule-based methods and a ChEBI lookup. For every class, each model that covers it casts a vote. This vote gets weighted by how reliable it proved to be for that class on validation data and - the model weight you can tune in the ensemble settings. + the model weight you can modify in the ensemble settings. The resulting predictions are checked for consistency with the ChEBI ontology and @@ -104,10 +109,9 @@ const About = () => { Clicking a class in the ontology graph of a result shows how much of the decision each model is responsible for. The ensemble settings also let you trade precision - against recall - an experimental feature, whose percentages come from the ChEBI test - set and will be optimistic for unusual molecules and rare classes. Details about the - ensemble and its implementation can be - found here. + against recall. More precision means that the ensemble is more conservative in its predictions. + More recall equates to a more daring ensemble. Note that these values are based on the ChEBI test + set and will be optimistic for unusual molecules and rare classes. @@ -163,16 +167,11 @@ const About = () => { {stats?.molecules > 0 && ( Chebifier has classified {stats.molecules.toLocaleString('en-US')} molecules - {stats.since ? ` since ${formatDate(stats.since)}` : ''}. That count is all we - keep: the number of molecules per day, and nothing about the molecules - themselves. + {stats.since ? ` since ${formatDate(stats.since)}` : ''}. )} - Chebifier does not collect or store the molecules you submit. A SMILES or InChI - string you enter is used to compute the prediction you asked for and nothing else: - it is never written to disk, never kept after the request, and never passed on to - anyone else. + Appart from the overall count, Chebifier does not store any personal information about you.     diff --git a/react-app/src/Navbar.css b/react-app/src/Navbar.css index d6dbcf8..265abe2 100644 --- a/react-app/src/Navbar.css +++ b/react-app/src/Navbar.css @@ -5,6 +5,8 @@ background-color: #282c34; color: #fff; padding: 1rem; + gap: 1.5rem; + flex-wrap: wrap; } .navbar-left .logo { @@ -33,6 +35,19 @@ .navbar-right { display: flex; align-items: center; + max-width: 40%; + margin-left: auto; +} + +.navbar-right .citation { + font-size: 0.8rem; + line-height: 1.3; + color: #d5d8de; + text-align: right; +} + +.navbar-right .citation a { + color: #9ec5ff; } .navbar-right .cart-icon, diff --git a/react-app/src/navbar.js b/react-app/src/navbar.js index c7c3d52..9b9bda7 100644 --- a/react-app/src/navbar.js +++ b/react-app/src/navbar.js @@ -25,6 +25,13 @@ const Navbar = () => {
    +
    + + If you like Chebifier, please cite: Glauer, Martin, et al. "Chebifier: Automating Semantic + Classification in ChEBI to Accelerate Data-driven Discovery."{" "} + Digital Discovery, 2024, 3, 896. + +
    diff --git a/react-app/src/smiles-form/classification-form.js b/react-app/src/smiles-form/classification-form.js index aa006ef..f2a055f 100644 --- a/react-app/src/smiles-form/classification-form.js +++ b/react-app/src/smiles-form/classification-form.js @@ -190,7 +190,7 @@ export default function ClassificationGrid() { '', `Selected class: ${explanation.name} (CHEBI:${selected})`, `Ensemble score: ${explanation.score?.toFixed(3)} (predicted above ${settings.threshold ?? decisionThreshold})`, - explanation.near_miss ? 'This class was NOT predicted - it stayed below the threshold.' : '', + explanation.near_miss ? 'This class was NOT predicted' : '', 'Model contributions:', ...Object.entries(explanation.models || {}).map(([model, values]) => ` ${model}: prediction ${values.prediction?.toFixed(3)}, ` + @@ -282,7 +282,7 @@ export default function ClassificationGrid() { /> {explanation.near_miss && ( - not predicted - the score stayed below the threshold + not predicted )}     @@ -401,25 +401,10 @@ export default function ClassificationGrid() { minHeight: '100vh', backgroundColor: '#ffffff', display: 'flex', - flexDirection: 'column' + flexDirection: 'column', + paddingTop: 2, + paddingBottom: 4 }}> - - - If you like Chebifier, please cite: Glauer, Martin, et al. "Chebifier: Automating Semantic - Classification in ChEBI to Accelerate Data-driven Discovery." - Digital Discovery, 2024, 3, - 896. - - - Predicted class - +