From ab176b40cdf3c103d70152f2558ef2a2a5345e6e Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 23 Jul 2026 01:51:11 +0800 Subject: [PATCH] Add RPG semantic-ID recommender --- README.md | 1 + cornac/eval_methods/next_item_evaluation.py | 9 + cornac/models/__init__.py | 1 + cornac/models/rpg/README.md | 142 ++++ cornac/models/rpg/__init__.py | 22 + cornac/models/rpg/recom_rpg.py | 682 ++++++++++++++++++ cornac/models/rpg/requirements.txt | 4 + cornac/models/rpg/rpg.py | 255 +++++++ cornac/models/rpg/rpg_config.py | 117 +++ docs/source/api_ref/models.rst | 5 + examples/README.md | 2 + examples/rpg_example.py | 111 +++ .../eval_methods/test_next_item_evaluation.py | 77 +- 13 files changed, 1426 insertions(+), 2 deletions(-) create mode 100644 cornac/models/rpg/README.md create mode 100644 cornac/models/rpg/__init__.py create mode 100644 cornac/models/rpg/recom_rpg.py create mode 100644 cornac/models/rpg/requirements.txt create mode 100644 cornac/models/rpg/rpg.py create mode 100644 cornac/models/rpg/rpg_config.py create mode 100644 examples/rpg_example.py diff --git a/README.md b/README.md index 4be2cd2b9..e0b2d7ba8 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ The table below lists the recommendation models/algorithms featured in Cornac. E | Year | Model and Paper | Type | Environment | Example | | :--: | --------------- | :--: | :---------: | :-----: | +| 2025 | [Generating Long Semantic IDs in Parallel for Recommendation (RPG)](cornac/models/rpg), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.rpg.recom_rpg), [paper](https://arxiv.org/abs/2506.05781) | Next-Item / Content-Based | [requirements](cornac/models/rpg/requirements.txt), CPU / GPU | [quick-start](examples/rpg_example.py) | 2024 | [Comparative Aspects and Opinions Ranking for Recommendation Explanations (Companion)](cornac/models/companion), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.companion.recom_companion), [paper](https://lthoang.com/assets/publications/mlj24.pdf) | Hybrid / Sentiment / Explainable | CPU | [quick-start](examples/companion_example.py) | | [Hypergraphs with Attention on Reviews (HypAR)](cornac/models/hypar), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.hypar.recom_hypar), [paper](https://doi.org/10.1007/978-3-031-56027-9_14)| Hybrid / Sentiment / Explainable | [requirements](cornac/models/hypar/requirements_cu118.txt), CPU / GPU | [quick-start](https://github.com/PreferredAI/HypAR) | 2023 | [Recommender Systems with Generative Retrieval (TIGER)](cornac/models/tiger), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.tiger.recom_tiger), [paper](https://arxiv.org/pdf/2305.05065.pdf) | Next-Item / Content-Based | [requirements](cornac/models/tiger/requirements.txt), CPU / GPU | [quick-start](examples/tiger_example.py) diff --git a/cornac/eval_methods/next_item_evaluation.py b/cornac/eval_methods/next_item_evaluation.py index e7cedc02d..aceb1a26c 100644 --- a/cornac/eval_methods/next_item_evaluation.py +++ b/cornac/eval_methods/next_item_evaluation.py @@ -192,6 +192,14 @@ class NextItemEvaluation(BaseMethod): Notes ----- + **Item content.** Modalities (e.g., + ``item_feature=FeatureModality(features=..., ids=...)``) can be passed as + keyword arguments to the constructor or any of the class-method builders + (:meth:`from_splits`, :meth:`from_timestamps`, :meth:`leave_last_out`). + They are built against the global item-ID map and attached to all splits, + so content-based next-item models (e.g., TIGER) can read + ``train_set.item_feature.features`` with rows aligned to item indices. + **Data splitting.** Ratio-based splitting (inherited from :obj:`BaseMethod`) and per-user leave-last-out both leak future information into training: a random split trains on interactions that @@ -277,6 +285,7 @@ def _build_datasets(self, train_data, test_data, val_data=None): fmt=self.fmt, global_uid_map=self.global_uid_map, global_iid_map=self.global_iid_map, + global_sid_map=self.global_sid_map, seed=self.seed, exclude_unknowns=self.exclude_unknowns, ) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 1e9e3e5a3..c354d3fa6 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -75,6 +75,7 @@ from .pcrl import PCRL from .pmf import PMF from .recvae import RecVAE +from .rpg import RPG from .sansa import SANSA from .sasrec import SASRec from .sbpr import SBPR diff --git a/cornac/models/rpg/README.md b/cornac/models/rpg/README.md new file mode 100644 index 000000000..4cd84808b --- /dev/null +++ b/cornac/models/rpg/README.md @@ -0,0 +1,142 @@ +# RPG + +Cornac implementation of **RPG** (Generating Long Semantic IDs in Parallel for +Recommendation, Hou et al., KDD 2025, +[arXiv:2506.05781](https://arxiv.org/abs/2506.05781)). RPG represents each item +with a long, unordered Semantic ID produced by product quantization and predicts +all of the next item's Semantic-ID digits in parallel. This port follows the +[official implementation](https://github.com/facebookresearch/RPG_KDD2025) for +OPQ tokenization, causal-session training, the GPT-2 multi-token-prediction +(MTP) backbone, and similarity-graph-guided decoding. + +## Requirements + +RPG needs PyTorch, Transformers, FAISS, and scikit-learn: + +```bash +pip install -r cornac/models/rpg/requirements.txt +``` + +The FAISS tokenizer uses 8-bit product quantization, so the supported +`codebook_size` is 256. Item content embeddings must cover every item known to +the train, validation, and test splits. + +## Usage + +Supply precomputed item content embeddings through a `FeatureModality`. Its +`features` rows must align with `ids` (see +[`examples/rpg_example.py`](../../../examples/rpg_example.py) for an end-to-end +example): + +```python +from cornac.data import FeatureModality +from cornac.eval_methods import NextItemEvaluation +from cornac.models import RPG +from cornac.models.rpg import RPG_BEAUTY_CONFIG + +eval_method = NextItemEvaluation.from_splits( + train_data=train, + val_data=val, + test_data=test, + mode="last", + item_feature=FeatureModality(features=item_embeddings, ids=item_ids), +) + +model = RPG(**{**RPG_BEAUTY_CONFIG, "device": "auto", "seed": 2024}) +``` + +The paper uses 3072-dimensional OpenAI `text-embedding-3-large` item +embeddings, followed by whitened PCA to 512 dimensions. Other content +embeddings are accepted, but Semantic-ID quality and recommendation results +depend on them. + +Constructor defaults provide a lighter training setup. Use `RPG_CONFIG` for +the official repository defaults or a per-dataset configuration for the +paper-best Amazon recipe: + +| Config | LR | Temperature | Codebooks | Beams | Graph edges | Propagation steps | +| ------------------- | -----: | ----------: | --------: | ----: | ----------: | ----------------: | +| `RPG_CONFIG` | 0.0003 | 0.07 | 32 | 50 | 50 | 3 | +| `RPG_BEAUTY_CONFIG` | 0.01 | 0.03 | 32 | 20 | 200 | 3 | +| `RPG_SPORTS_CONFIG` | 0.003 | 0.03 | 16 | 100 | 30 | 5 | +| `RPG_TOYS_CONFIG` | 0.003 | 0.03 | 16 | 200 | 20 | 3 | + +All shipped configurations use a maximum history length of 50, batch size 256, +AdamW, cosine scheduling with 10,000 warmup steps, and at most 150 epochs. The +validation NDCG@10 is evaluated every epoch in batches of 32; training stops +after 20 consecutive non-improving epochs and restores the best checkpoint. + +## Training iteration + +RPG does **not** expand a short session into independent prefix-target rows as +TIGER, LETTER, and SASRec do. For a session `[a, b, c, d]`, RPG creates one +causal training row: + +```text +input: [a, b, c, PAD, ...] +labels: [b, c, d, -100, ...] +``` + +The GPT-2 backbone produces a hidden state at every input position. Each valid +label contributes an MTP loss that predicts all Semantic-ID codebooks in +parallel; `-100` labels are ignored. Consequently, the three next-item targets +share one forward pass and one optimizer step. + +For a session longer than `max_len + 1`, the first window supervises every +position. Later sliding windows supervise only their final position, ensuring +that every next-item target is counted exactly once. A session of length `T` +therefore produces `max(T - max_len, 1)` training rows, rather than `T - 1` +prefix rows. + +## Scoring modes + +- `scoring="graph"` follows the paper: it propagates a beam over a similarity + graph built from item Semantic IDs and assigns real scores only to the final + candidates. Set `n_beams` at least as large as the largest evaluation cutoff. +- `scoring="exact"` scores the full catalog by gathering and averaging the + parallel per-codebook logits. It is useful as a deterministic diagnostic and + for modest catalogs, but it is not the paper's retrieval path. + +Both modes use the same trained model. On the Amazon experiments below, their +Recall/NDCG results differ by at most 0.0018 in absolute value. + +## Results + +Paper-style graph-decoding results on the three Amazon Reviews 2014 5-core +datasets: + +| Dataset | R@5 ours/paper | N@5 ours/paper | R@10 ours/paper | N@10 ours/paper | +| ------- | --------------: | --------------: | --------------: | --------------: | +| Beauty | 0.0533 / 0.0550 | 0.0372 / 0.0381 | 0.0789 / 0.0809 | 0.0454 / 0.0464 | +| Sports | 0.0288 / 0.0314 | 0.0198 / 0.0216 | 0.0420 / 0.0463 | 0.0241 / 0.0263 | +| Toys | 0.0613 / 0.0592 | 0.0415 / 0.0401 | 0.0898 / 0.0869 | 0.0506 / 0.0490 | + +**Setting:** per-user leave-last-out (`train = sequence[:-2]`, validation target += second-to-last, test target = last), `mode="last"`, `max_len=50`, and +`seed=123`. Items use the official RPG `text-embedding-3-large` embeddings. +Each category uses its per-dataset configuration above. Beauty and Sports are +within 3.1% and 8.3% of the paper's R@5, respectively; Toys is 3.5% higher. + +### Graph versus exact scoring + +| Dataset | Scoring | R@5 | N@5 | R@10 | N@10 | R@20 | N@20 | MRR | Total (s) | +| ------- | ------- | -----: | -----: | -----: | -----: | -----: | -----: | -----: | --------: | +| Beauty | graph | 0.0533 | 0.0372 | 0.0789 | 0.0454 | 0.1113 | 0.0536 | 0.0380 | 2502 | +| Beauty | exact | 0.0534 | 0.0373 | 0.0798 | 0.0457 | 0.1125 | 0.0539 | 0.0412 | — | +| Sports | graph | 0.0288 | 0.0198 | 0.0420 | 0.0241 | 0.0619 | 0.0291 | 0.0219 | 4095 | +| Sports | exact | 0.0291 | 0.0201 | 0.0428 | 0.0245 | 0.0627 | 0.0295 | 0.0229 | — | +| Toys | graph | 0.0613 | 0.0415 | 0.0898 | 0.0506 | 0.1233 | 0.0591 | 0.0439 | 1682 | +| Toys | exact | 0.0611 | 0.0414 | 0.0903 | 0.0508 | 0.1251 | 0.0596 | 0.0444 | — | + +## Practical notes + +- `n_codebook` controls Semantic-ID length. The paper-best Sports and Toys + recipes use 16 codebooks, Beauty uses 32, and the paper's long-ID + CDs-and-Vinyl experiment uses 64. +- OPQ is trained only on items present in the training interactions, while the + resulting tokenizer assigns codes to all items with supplied features. +- Content embeddings are part of the model recipe. Random features are useful + for smoke tests but do not provide meaningful semantic IDs. +- Graph decoding is approximate and stochastic because its initial beam is + sampled. Set `seed` for repeatability, or use exact scoring when deterministic + full-catalog ranks are required. diff --git a/cornac/models/rpg/__init__.py b/cornac/models/rpg/__init__.py new file mode 100644 index 000000000..7b14a5b05 --- /dev/null +++ b/cornac/models/rpg/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from .recom_rpg import RPG +from .rpg_config import ( + RPG_CONFIG, + RPG_BEAUTY_CONFIG, + RPG_SPORTS_CONFIG, + RPG_TOYS_CONFIG, +) diff --git a/cornac/models/rpg/recom_rpg.py b/cornac/models/rpg/recom_rpg.py new file mode 100644 index 000000000..c66f0f12a --- /dev/null +++ b/cornac/models/rpg/recom_rpg.py @@ -0,0 +1,682 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import math + +import numpy as np +from tqdm.auto import trange + +from cornac.models.recommender import NextItemRecommender + +from ...utils import get_rng + +SUPPORTED_SCORING = ("graph", "exact") +SUPPORTED_LR_SCHEDULES = ("constant", "cosine") +RPG_IGNORE_INDEX = -100 + + +def _rpg_num_training_rows(train_set, max_len): + """Number of rows produced by the official RPG windowing scheme.""" + return sum( + max(len(mapped_ids) - max_len, 1) + for mapped_ids in train_set.sessions.values() + if len(mapped_ids) >= 2 + ) + + +def _rpg_session_iter( + train_set, + pad_index, + batch_size=256, + max_len=50, + rng=None, + shuffle=True, +): + """Yield official RPG causal-session training rows. + + A session of at most ``max_len + 1`` items becomes one right-padded row: + ``items[:-1]`` predicts ``items[1:]`` at every valid position. For a longer + session, the first window supervises every position and each subsequent + sliding window supervises only its last position. This covers every + next-item target exactly once while bounding the GPT input length. + """ + rng = rng if rng is not None else get_rng(None) + sessions = train_set.sessions + examples = [] + for sid, mapped_ids in sessions.items(): + if len(mapped_ids) < 2: + continue + for start in range(max(len(mapped_ids) - max_len, 1)): + examples.append((sid, start)) + if shuffle: + rng.shuffle(examples) + + uir_tuple = train_set.uir_tuple + buffer_uids, buffer_inputs, buffer_labels = [], [], [] + for sid, start in examples: + mapped_ids = sessions[sid] + items = np.asarray(uir_tuple[1][mapped_ids], dtype="int64") + window = items[start : start + max_len + 1] + n_inputs = len(window) - 1 + + input_iids = np.full(max_len, pad_index, dtype="int64") + input_iids[:n_inputs] = window[:-1] + labels = np.full(max_len, RPG_IGNORE_INDEX, dtype="int64") + if start == 0: + labels[:n_inputs] = window[1:] + else: + labels[n_inputs - 1] = window[-1] + + buffer_uids.append(int(uir_tuple[0][mapped_ids[0]])) + buffer_inputs.append(input_iids) + buffer_labels.append(labels) + if len(buffer_uids) == batch_size: + yield ( + np.asarray(buffer_uids, dtype="int64"), + np.asarray(buffer_inputs, dtype="int64"), + np.asarray(buffer_labels, dtype="int64"), + ) + buffer_uids, buffer_inputs, buffer_labels = [], [], [] + + if buffer_uids: + yield ( + np.asarray(buffer_uids, dtype="int64"), + np.asarray(buffer_inputs, dtype="int64"), + np.asarray(buffer_labels, dtype="int64"), + ) + + +class RPG(NextItemRecommender): + """RPG: Generating Long Semantic IDs in Parallel for Recommendation. + + RPG is the architectural counterpoint to TIGER. Instead of short *ordered* + RQ-VAE semantic IDs decoded autoregressively, RPG quantizes precomputed + item content embeddings into **long unordered** semantic IDs with **product + quantization (OPQ, via faiss)** and predicts **all digits in parallel** with + a multi-token-prediction (MTP) loss. A GPT-2 decoder consumes a session as a + sequence of items (each item embedded as the mean of its ``n_codebook`` + semantic-ID token embeddings); one residual head per codebook predicts every + digit of every supervised next item's semantic ID at once, scored by + temperature-scaled cosine similarity to the shared token-embedding table. + Inference uses a similarity graph over item semantic IDs to guide beam + decoding to valid items, so scoring cost is independent of the corpus size. + + Item content embeddings must be provided through the evaluation method, + e.g.:: + + NextItemEvaluation.from_splits( + ..., item_feature=FeatureModality(features=embs, ids=item_ids) + ) + + where ``embs`` are precomputed text/content embeddings covering every known + item. A ready-made :data:`~cornac.models.rpg.RPG_CONFIG` (the official + Amazon-2014 recipe) ships with the model. + + Parameters + ---------- + name: str, default: 'RPG' + The name of the recommender model. + + n_codebook: int, default: 32 + Number of OPQ codebooks = the semantic-ID length (number of digits). + This is the knob behind RPG's "scaling ID length" result: set to 64 for + the long-ID configuration. + + codebook_size: int, default: 256 + Number of codes per codebook (PQ uses 8 bits, hence 256). + + pca_dim: int, default: 512 + Target dimensionality of the whitened PCA applied to the item + embeddings before OPQ. Values ``<= 0`` or ``>=`` the embedding + dimension skip PCA. + + feature_standardize: bool, default: False + When True, z-score the item features per dimension before PCA/OPQ. + + d_model: int, default: 448 + n_layer: int, default: 2 + n_head: int, default: 4 + n_inner: int, default: 1024 + activation: str, default: 'gelu_new' + resid_dropout: float, default: 0.0 + embd_dropout: float, default: 0.5 + attn_dropout: float, default: 0.5 + layer_norm_eps: float, default: 1e-12 + initializer_range: float, default: 0.02 + GPT-2 backbone architecture settings (defaults per the paper). + + max_len: int, default: 50 + Maximum number of history items fed to the backbone. + + temperature: float, default: 0.07 + Temperature of the cosine-similarity MTP logits (train and score). + + n_epochs: int, default: 20 + learning_rate: float, default: 3e-4 + weight_decay: float, default: 0.0 + batch_size: int, default: 256 + max_grad_norm: float or None, default: 1.0 + Backbone training settings (AdamW). The official recipe trains 150 + epochs with early stopping; ``RPG_CONFIG`` carries the full recipe. + + lr_schedule: str, default: 'constant' + 'constant' keeps ``learning_rate`` fixed; 'cosine' does linear warmup + over ``warmup_steps`` then cosine decay (as in the official trainer). + + warmup_steps: int, default: 10000 + Linear-warmup steps when ``lr_schedule='cosine'``. + + scoring: str, default: 'graph' + 'graph' (paper-faithful similarity-graph-guided beam; only the final + ``n_beams`` candidates get real scores) + or 'exact' (full-catalog parallel-MTP score of every item; exact full + ranking, cheap because the backbone runs once per user). + + n_beams: int, default: 50 + Beam width for scoring='graph'. + + graph_edges: int, default: 50 + Number of kNN neighbours per item in the similarity graph. + + propagation_steps: int, default: 3 + Number of graph-propagation steps during graph decoding. + + graph_chunk_size: int, default: 1024 + Item chunk size when building the similarity graph. + + model_selection: str, default: 'last' + 'last' or 'best'. When 'best' and a ``val_set`` is given, the backbone + weights with the highest validation score (evaluated every + ``val_eval_every`` epochs on up to ``val_sample`` val sessions, with + batched exact scoring) are restored at the end of ``fit``. + + val_metric: str, default: 'ndcg' + val_eval_every: int, default: 5 + val_batch_size: int, default: 32 + early_stopping_patience: int or None, default: None + val_k: int, default: 10 + val_sample: int, default: 2000 + Metric, cadence, validation batch size, non-improving evaluation + patience, cutoff K and session cap for best-on-val selection. + + device: str, default: 'auto' + 'auto' selects 'cuda' if available, otherwise 'cpu'. + + trainable: bool, default: True + verbose: bool, default: False + seed: int, default: None + Random seed for weight init, OPQ, and graph decoding. + + References + ---------- + Hou, Y. et al. (2025). Generating Long Semantic IDs in Parallel for + Recommendation. KDD. https://arxiv.org/abs/2506.05781 + (official code: https://github.com/facebookresearch/RPG_KDD2025) + """ + + def __init__( + self, + name="RPG", + n_codebook=32, + codebook_size=256, + pca_dim=512, + feature_standardize=False, + d_model=448, + n_layer=2, + n_head=4, + n_inner=1024, + activation="gelu_new", + resid_dropout=0.0, + embd_dropout=0.5, + attn_dropout=0.5, + layer_norm_eps=1e-12, + initializer_range=0.02, + max_len=50, + temperature=0.07, + n_epochs=20, + learning_rate=3e-4, + weight_decay=0.0, + batch_size=256, + max_grad_norm=1.0, + lr_schedule="constant", + warmup_steps=10000, + scoring="graph", + n_beams=50, + graph_edges=50, + propagation_steps=3, + graph_chunk_size=1024, + model_selection="last", + val_metric="ndcg", + val_eval_every=5, + val_batch_size=32, + early_stopping_patience=None, + val_k=10, + val_sample=2000, + device="auto", + trainable=True, + verbose=False, + seed=None, + ): + super().__init__(name, trainable=trainable, verbose=verbose) + if scoring not in SUPPORTED_SCORING: + raise ValueError( + f"scoring='{scoring}' not supported; choose from {SUPPORTED_SCORING}" + ) + if lr_schedule not in SUPPORTED_LR_SCHEDULES: + raise ValueError( + f"lr_schedule='{lr_schedule}' not supported; choose from {SUPPORTED_LR_SCHEDULES}" + ) + if model_selection not in ("last", "best"): + raise ValueError( + f"model_selection='{model_selection}' not supported; choose 'last' or 'best'" + ) + if val_eval_every <= 0: + raise ValueError("val_eval_every must be positive") + if val_batch_size <= 0: + raise ValueError("val_batch_size must be positive") + if early_stopping_patience is not None and early_stopping_patience <= 0: + raise ValueError("early_stopping_patience must be positive or None") + self.n_codebook = n_codebook + self.codebook_size = codebook_size + self.pca_dim = pca_dim + self.feature_standardize = feature_standardize + self.d_model = d_model + self.n_layer = n_layer + self.n_head = n_head + self.n_inner = n_inner + self.activation = activation + self.resid_dropout = resid_dropout + self.embd_dropout = embd_dropout + self.attn_dropout = attn_dropout + self.layer_norm_eps = layer_norm_eps + self.initializer_range = initializer_range + self.max_len = max_len + self.temperature = temperature + self.n_epochs = n_epochs + self.learning_rate = learning_rate + self.weight_decay = weight_decay + self.batch_size = batch_size + self.max_grad_norm = max_grad_norm + self.lr_schedule = lr_schedule + self.warmup_steps = warmup_steps + self.scoring = scoring + self.n_beams = n_beams + self.graph_edges = graph_edges + self.propagation_steps = propagation_steps + self.graph_chunk_size = graph_chunk_size + self.model_selection = model_selection + self.val_metric = val_metric + self.val_eval_every = val_eval_every + self.val_batch_size = val_batch_size + self.early_stopping_patience = early_stopping_patience + self.val_k = val_k + self.val_sample = val_sample + self.device = device + self.seed = seed + self.rng = get_rng(seed) + self.graph_rng = get_rng(seed) + + def _get_item_features(self): + item_feature = getattr(self.train_set, "item_feature", None) + features = getattr(item_feature, "features", None) + if features is None: + raise ValueError( + "RPG requires precomputed item content embeddings. Provide them " + "via NextItemEvaluation.from_splits(..., item_feature=" + "FeatureModality(features=..., ids=...))." + ) + if features.shape[0] < self.total_items: + raise ValueError( + f"item_feature has {features.shape[0]} rows but {self.total_items} " + "items are known; every item (train/val/test) needs a feature vector." + ) + return np.asarray(features[: self.total_items], dtype="float32") + + def _opq_tokenize(self, feats, train_mask=None): + """OPQ (faiss) tokenizer -> (n_items, n_codebook) un-offset codes. + + Faithful to ``genrec/models/RPG/tokenizer.py``: whitened PCA to + ``pca_dim`` then a faiss ``OPQ{M},IVF1,PQ{M}x8`` index; the per-item + 8-bit PQ codes are read straight out of the (single) inverted list and + reordered into item order. Each byte is one digit's code in ``[0, 256)``. + """ + import faiss + from sklearn.decomposition import PCA + + if self.codebook_size != 256: + raise ValueError( + "the faiss OPQ path uses 8-bit PQ (256 codes per codebook); set " + f"codebook_size=256 (got {self.codebook_size})." + ) + n_items, dim = feats.shape + if self.pca_dim and 0 < self.pca_dim < dim: + n_comp = min(self.pca_dim, n_items - 1, dim) + feats = PCA(n_components=n_comp, whiten=True).fit_transform(feats) + feats = np.ascontiguousarray(feats, dtype="float32") + if train_mask is None: + train_feats = feats + else: + train_mask = np.asarray(train_mask, dtype=bool) + if train_mask.shape != (n_items,): + raise ValueError( + f"train_mask must have shape ({n_items},), got {train_mask.shape}" + ) + train_feats = np.ascontiguousarray(feats[train_mask], dtype="float32") + + factory = f"OPQ{self.n_codebook},IVF1,PQ{self.n_codebook}x8" + index = faiss.index_factory( + feats.shape[1], factory, faiss.METRIC_INNER_PRODUCT + ) + index.train(train_feats) + index.add(feats) + + index_ivf = faiss.extract_index_ivf(index) + invlists = index_ivf.invlists # IVF1 -> a single inverted list (list 0) + list_size = invlists.list_size(0) + code_size = invlists.code_size # bytes per item = n_codebook for 8-bit PQ + codes = faiss.rev_swig_ptr( + invlists.get_codes(0), list_size * code_size + ).reshape(list_size, code_size)[:, : self.n_codebook] + ids = faiss.rev_swig_ptr(invlists.get_ids(0), list_size).copy() + sid_table = np.zeros((n_items, self.n_codebook), dtype="int64") + sid_table[ids] = codes.astype("int64") + return sid_table + + def _build_backbone(self): + from .rpg import RPGBackbone + + model = RPGBackbone( + n_codebook=self.n_codebook, + codebook_size=self.codebook_size, + max_len=self.max_len, + d_model=self.d_model, + n_layer=self.n_layer, + n_head=self.n_head, + n_inner=self.n_inner, + activation=self.activation, + resid_dropout=self.resid_dropout, + embd_dropout=self.embd_dropout, + attn_dropout=self.attn_dropout, + layer_norm_eps=self.layer_norm_eps, + initializer_range=self.initializer_range, + temperature=self.temperature, + ).to(self.device_) + model.set_item_tokens(self.sid_table) + return model + + def _make_lr_scheduler(self, torch, opt): + if self.lr_schedule != "cosine": + return None + n_rows = _rpg_num_training_rows(self.train_set, self.max_len) + steps_per_epoch = max(1, math.ceil(n_rows / self.batch_size)) + total_steps = max(1, steps_per_epoch * self.n_epochs) + + def lr_lambda(step): + if step < self.warmup_steps: + return step / max(1, self.warmup_steps) + progress = (step - self.warmup_steps) / max( + 1, total_steps - self.warmup_steps + ) + return 0.5 * (1.0 + math.cos(math.pi * min(1.0, progress))) + + return torch.optim.lr_scheduler.LambdaLR(opt, lr_lambda) + + def _make_val_metric(self): + from ...metrics import AUC, MRR, NDCG, Recall + + name = self.val_metric.lower() + if name == "recall": + return Recall(k=self.val_k) + if name == "ndcg": + return NDCG(k=self.val_k) + if name == "auc": + return AUC() + if name == "mrr": + return MRR() + raise ValueError( + f"val_metric='{self.val_metric}' not supported; choose from recall/ndcg/auc/mrr" + ) + + def _val_sessions(self, val_set): + sessions = [] + for [_], [mapped_ids], [session_items] in val_set.si_iter( + batch_size=1, shuffle=False + ): + if len(session_items) < 2: + continue + user_idx = int(val_set.uir_tuple[0][mapped_ids[0]]) + sessions.append((user_idx, [int(i) for i in session_items])) + if self.val_sample is not None and len(sessions) > self.val_sample: + idx = self.rng.choice(len(sessions), size=self.val_sample, replace=False) + sessions = [sessions[i] for i in sorted(idx)] + return sessions + + def _exact_item_scores(self, logits, n_items=None): + """Score catalog items from batched per-codebook logits.""" + import torch + + if logits.dim() == 2: + logits = logits.unsqueeze(0) + n_items = self.total_items if n_items is None else n_items + codes = torch.as_tensor( + self.sid_table[:n_items], dtype=torch.long, device=logits.device + ) + scores = logits.new_zeros((logits.size(0), n_items)) + for digit in range(self.n_codebook): + scores += logits[:, digit].index_select(1, codes[:, digit]) + return scores / self.n_codebook + + def _validate(self, val_sessions, metric): + """Mean metric over last-item sessions using batched exact scoring.""" + import torch + + num_items = self.train_set.num_items + item_indices = np.arange(num_items) + results = [] + self.model.eval() + device = next(self.model.parameters()).device + for start in range(0, len(val_sessions), self.val_batch_size): + batch = [ + session + for session in val_sessions[start : start + self.val_batch_size] + if session[1][-1] < num_items + ] + if not batch: + continue + input_ids = np.full( + (len(batch), self.max_len), self.pad_idx, dtype="int64" + ) + for row, (_, session_items) in enumerate(batch): + history = session_items[:-1][-self.max_len :] + input_ids[row, -len(history) :] = history + input_ids = torch.as_tensor(input_ids, device=device) + attention_mask = (input_ids != self.pad_idx).float() + with torch.no_grad(): + logits = self.model.next_item_logits(input_ids, attention_mask) + batch_scores = ( + self._exact_item_scores(logits, num_items).cpu().numpy() + ) + + for row, (_, session_items) in enumerate(batch): + target = session_items[-1] + item_scores = batch_scores[row] + item_rank = item_indices[item_scores.argsort()[::-1]] + results.append( + metric.compute( + gt_pos=np.array([target]), + gt_neg=np.delete(item_indices, target), + pd_rank=item_rank, + pd_scores=item_scores, + item_indices=item_indices, + ) + ) + return float(np.mean(results)) if results else 0.0 + + def _fit_backbone(self, torch, val_set): + self.pad_idx = self.total_items + opt = torch.optim.AdamW( + self.model.parameters(), + lr=self.learning_rate, + weight_decay=self.weight_decay, + ) + scheduler = self._make_lr_scheduler(torch, opt) + + best_state, best_val = None, -float("inf") + non_improving = 0 + select_best = self.model_selection == "best" and val_set is not None + val_sessions = self._val_sessions(val_set) if select_best else None + val_metric = self._make_val_metric() if select_best else None + + progress_bar = trange(1, self.n_epochs + 1, disable=not self.verbose, desc="RPG") + for epoch_id in progress_bar: + self.current_epoch = epoch_id + self.model.train() + total_loss, cnt = 0.0, 0 + for inc, (_, input_iids, labels) in enumerate( + _rpg_session_iter( + self.train_set, + pad_index=self.pad_idx, + batch_size=self.batch_size, + max_len=self.max_len, + rng=self.rng, + shuffle=True, + ) + ): + input_ids = torch.tensor( + input_iids, dtype=torch.long, device=self.device_ + ) + attn_mask = (input_ids != self.pad_idx).float() + target_iids = torch.tensor( + labels, dtype=torch.long, device=self.device_ + ) + opt.zero_grad() + loss = self.model(input_ids, attn_mask, target_iids) + loss.backward() + if self.max_grad_norm is not None: + torch.nn.utils.clip_grad_norm_( + self.model.parameters(), self.max_grad_norm + ) + opt.step() + if scheduler is not None: + scheduler.step() + total_loss += loss.item() + cnt += 1 + if inc % 10 == 0 and cnt > 0: + progress_bar.set_postfix(loss=(total_loss / cnt)) + + if select_best and epoch_id % self.val_eval_every == 0: + score = self._validate(val_sessions, val_metric) + if score > best_val: + best_val = score + non_improving = 0 + self.best_value = score + self.best_epoch = epoch_id + self.wait = 0 + best_state = { + n: p.detach().clone() + for n, p in self.model.state_dict().items() + } + else: + non_improving += 1 + self.wait = non_improving + if ( + self.early_stopping_patience is not None + and non_improving >= self.early_stopping_patience + ): + self.stopped_epoch = epoch_id + break + + if best_state is not None: + self.model.load_state_dict(best_state) + + def fit(self, train_set, val_set=None): + super().fit(train_set, val_set) + if not self.trainable: + return self + + import torch + + torch.manual_seed(self.seed if self.seed is not None else 0) + self.device_ = ( + ("cuda" if torch.cuda.is_available() else "cpu") + if self.device == "auto" + else self.device + ) + + feats = self._get_item_features() + if self.feature_standardize: + mean = feats.mean(axis=0) + std = feats.std(axis=0) + std[std == 0] = 1.0 + feats = ((feats - mean) / std).astype("float32") + + train_mask = np.zeros(self.total_items, dtype=bool) + train_mask[np.asarray(self.train_set.uir_tuple[1], dtype="int64")] = True + self.sid_table = self._opq_tokenize(feats, train_mask) + self.item_tokens = ( + self.sid_table + self.codebook_size * np.arange(self.n_codebook) + 1 + ).astype("int64") + + self.model = self._build_backbone() + self._fit_backbone(torch, val_set) + + self.model.eval() + self.adjacency = self.model.build_adjacency( + self.total_items, self.graph_edges, self.graph_chunk_size + ) + # keep pickles portable across GPU/CPU boxes; moved back in score() + self.model.to("cpu").eval() + return self + + def _ensure_device(self, torch): + if self.device_ == "cuda" and not torch.cuda.is_available(): + self.device_ = "cpu" + if next(self.model.parameters()).device.type != torch.device(self.device_).type: + self.model.to(self.device_) + + def score(self, user_idx, history_items, **kwargs): + import torch + + from .rpg import graph_decode + + if len(history_items) == 0: + return np.ones(self.total_items, dtype="float") + self._ensure_device(torch) + hist = list(history_items)[-self.max_len :] + hist = [self.pad_idx] * (self.max_len - len(hist)) + hist + input_ids = torch.tensor([hist], dtype=torch.long, device=self.device_) + attn_mask = (input_ids != self.pad_idx).float() + self.model.eval() + with torch.no_grad(): + logits = self.model.next_item_logits(input_ids, attn_mask)[0] # (M, K) + + if self.scoring == "exact": + return ( + self._exact_item_scores(logits)[0].cpu().numpy().astype("float") + ) + + # graph-guided beam decoding over the final propagated beam + flat = logits.reshape(-1).cpu().numpy() + candidates, candidate_scores = graph_decode( + flat, + self.adjacency, + self.item_tokens, + self.n_beams, + self.propagation_steps, + self.graph_rng, + ) + scores = np.full(self.total_items, -1e10, dtype="float") + scores[candidates] = candidate_scores + return scores diff --git a/cornac/models/rpg/requirements.txt b/cornac/models/rpg/requirements.txt new file mode 100644 index 000000000..edeb2c7f8 --- /dev/null +++ b/cornac/models/rpg/requirements.txt @@ -0,0 +1,4 @@ +torch>=1.12.0 +transformers>=4.30.0 +faiss-cpu>=1.7.0 +scikit-learn>=1.0.0 diff --git a/cornac/models/rpg/rpg.py b/cornac/models/rpg/rpg.py new file mode 100644 index 000000000..4fd0badbf --- /dev/null +++ b/cornac/models/rpg/rpg.py @@ -0,0 +1,255 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Neural modules for RPG (Hou et al., KDD 2025). + +``RPGBackbone`` is a GPT-2 decoder over items whose embedding is the mean of +their (unordered) OPQ semantic-ID token embeddings, with a multi-token- + prediction (MTP) head per codebook that predicts every digit of the next + items' semantic IDs *in parallel* at all supervised sequence positions via + temperature-scaled cosine similarity to the (shared) token embedding table. + ``build_adjacency`` and ``graph_decode`` +implement the similarity-graph-guided beam decoding used at inference so that +scoring cost does not depend on scoring every catalog item exhaustively. + +Faithful to the official implementation +https://github.com/facebookresearch/RPG_KDD2025 (``genrec/models/RPG/model.py``): +the ``ResBlock`` heads, the ``wte(tokens).mean(dim=-2)`` item embedding, the +``F.normalize`` + cosine/temperature MTP loss, ``build_ii_sim_mat`` (mean over +digits of per-digit code cosine similarity) and ``graph_propagation``. +""" + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import GPT2Config +from transformers.models.gpt2.modeling_gpt2 import GPT2Model + + +class ResBlock(nn.Module): + """Residual SiLU block used as an MTP prediction head (zero-init linear so + each head starts as the identity).""" + + def __init__(self, hidden_size): + super().__init__() + self.linear = nn.Linear(hidden_size, hidden_size) + nn.init.zeros_(self.linear.weight) + self.act = nn.SiLU() + + def forward(self, x): + return x + self.act(self.linear(x)) + + +class RPGBackbone(nn.Module): + """GPT-2 backbone with one MTP head per codebook (RPG paper, Sec. 3). + + A single token-embedding table (``gpt2.wte``) holds all digits' codes with + per-digit offsets; index 0 is padding and the final index is an unused + ``eos`` slot so ``wte.weight[1:-1]`` is exactly the ``n_codebook * + codebook_size`` code embeddings (laid out digit-by-digit). An item's input + embedding is the mean of its ``n_codebook`` token embeddings; each + supervised position's hidden state feeds ``n_codebook`` residual heads, + each scored against its codebook by temperature-scaled cosine similarity. + """ + + def __init__( + self, + n_codebook, + codebook_size, + max_len, + d_model=448, + n_layer=2, + n_head=4, + n_inner=1024, + activation="gelu_new", + resid_dropout=0.0, + embd_dropout=0.5, + attn_dropout=0.5, + layer_norm_eps=1e-12, + initializer_range=0.02, + temperature=0.07, + ): + super().__init__() + self.n_codebook = int(n_codebook) + self.codebook_size = int(codebook_size) + self.temperature = temperature + # token 0 = pad; digit d codes occupy [1 + d*K, 1 + (d+1)*K); last = unused eos + vocab_size = self.n_codebook * self.codebook_size + 2 + cfg = GPT2Config( + vocab_size=vocab_size, + n_positions=max_len, + n_embd=d_model, + n_layer=n_layer, + n_head=n_head, + n_inner=n_inner, + activation_function=activation, + resid_pdrop=resid_dropout, + embd_pdrop=embd_dropout, + attn_pdrop=attn_dropout, + layer_norm_epsilon=layer_norm_eps, + initializer_range=initializer_range, + use_cache=False, + ) + self.gpt2 = GPT2Model(cfg) + self.pred_heads = nn.ModuleList([ResBlock(d_model) for _ in range(self.n_codebook)]) + + offsets = self.codebook_size * torch.arange(self.n_codebook, dtype=torch.long) + 1 + self.register_buffer("offsets", offsets) + # item -> offset tokens; row per item plus a trailing all-pad (0) row for + # the history pad index. Filled by ``set_item_tokens`` after tokenizing. + self.register_buffer( + "item_id2tokens", torch.zeros(1, self.n_codebook, dtype=torch.long) + ) + + def set_item_tokens(self, sid_table): + """Register the per-item OPQ codes as offset tokens (plus a pad row). + + ``sid_table``: (n_items, n_codebook) un-offset codes in ``[0, K)``. + """ + codes = torch.as_tensor(np.asarray(sid_table), dtype=torch.long) + tokens = codes + self.offsets.cpu() + pad_row = torch.zeros(1, self.n_codebook, dtype=torch.long) + table = torch.cat([tokens, pad_row], dim=0).to(self.offsets.device) + self.item_id2tokens = table + + def _hidden_states(self, input_ids, attention_mask): + """GPT-2 hidden states with real-item positions starting at zero.""" + input_tokens = self.item_id2tokens[input_ids] # (B, L, n_codebook) + input_embs = self.gpt2.wte(input_tokens).mean(dim=-2) # (B, L, d) + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids = position_ids.clamp_min(0) + out = self.gpt2( + inputs_embeds=input_embs, + attention_mask=attention_mask, + position_ids=position_ids, + ) + return out.last_hidden_state + + def _last_hidden(self, input_ids, attention_mask): + """GPT-2 hidden state at the most-recent real position for each row.""" + hidden = self._hidden_states(input_ids, attention_mask) + positions = torch.arange(input_ids.size(1), device=input_ids.device) + last_positions = (attention_mask.long() * positions).max(dim=-1).values + rows = torch.arange(input_ids.size(0), device=input_ids.device) + return hidden[rows, last_positions] + + def _mtp_logits(self, hidden): + """Per-digit code logits from ``(..., d)`` hidden states. + + Returns ``(..., n_codebook, codebook_size)``: temperature-scaled cosine + similarity between each head's normalized output and the normalized + codebook token embeddings. + """ + states = torch.stack( + [self.pred_heads[i](hidden) for i in range(self.n_codebook)], dim=-2 + ) + states = F.normalize(states, dim=-1) + token_emb = F.normalize(self.gpt2.wte.weight[1:-1], dim=-1) + token_emb = token_emb.view(self.n_codebook, self.codebook_size, -1) + logits = ( + torch.einsum("...id,ikd->...ik", states, token_emb) / self.temperature + ) + return logits + + def forward(self, input_ids, attention_mask, target_iids): + """Official masked all-position MTP loss. + + ``target_iids`` has shape ``(B, L)``. Valid entries are next-item IDs; + ``-100`` entries are ignored. Cross-entropy is averaged over all valid + positions for each codebook and then averaged across codebooks. + """ + hidden = self._hidden_states(input_ids, attention_mask) + label_mask = target_iids != -100 + logits = self._mtp_logits(hidden[label_mask]) + target_codes = self.item_id2tokens[target_iids[label_mask]] - self.offsets + losses = [ + F.cross_entropy(logits[:, i], target_codes[:, i]) + for i in range(self.n_codebook) + ] + return torch.stack(losses).mean() + + @torch.no_grad() + def next_item_logits(self, input_ids, attention_mask): + """Per-digit next-item code logits, shape (B, n_codebook, codebook_size).""" + return self._mtp_logits(self._last_hidden(input_ids, attention_mask)) + + @torch.no_grad() + def build_adjacency(self, n_items, edges, chunk_size=1024): + """kNN item-item similarity graph over the OPQ codes (``build_ii_sim_mat``). + + sim(i, j) = mean over digits d of the [0, 1]-rescaled cosine similarity + between the digit-d code embeddings of items i and j. Returns the top + ``edges`` neighbours per item as an int64 numpy array (n_items, k), + ``k = min(edges, n_items)``. As in the reference implementation, an + item's own node can be one of its neighbours. + """ + device = self.offsets.device + token_emb = F.normalize( + self.gpt2.wte.weight[1:-1].view(self.n_codebook, self.codebook_size, -1), + dim=-1, + ) + token_sims = torch.bmm(token_emb, token_emb.transpose(1, 2)) + token_sims = 0.5 * (token_sims + 1.0) # (n_codebook, K, K) in [0, 1] + + codes = self.item_id2tokens[:n_items] - self.offsets # (n_items, n_codebook) + k = min(int(edges), n_items) + adj = torch.empty(n_items, k, dtype=torch.long, device=device) + for start in range(0, n_items, chunk_size): + end = min(start + chunk_size, n_items) + ci = codes[start:end] # (b, n_codebook) + sim = torch.zeros(end - start, n_items, device=device) + for d in range(self.n_codebook): + sim += token_sims[d][ci[:, d]][:, codes[:, d]] + sim /= self.n_codebook + adj[start:end] = sim.topk(k, dim=1).indices + return adj.cpu().numpy() + + +def graph_decode(token_logits_flat, adjacency, item_tokens, num_beams, prop_steps, rng): + """Similarity-graph-guided beam decoding (``graph_propagation``). + + Starting from ``num_beams`` random items, repeatedly expand to the graph + neighbours of the current beam, score each candidate by the mean of its + ``n_codebook`` next-item token logits, and keep the top ``num_beams`` for + the next step. The final beam is returned after propagation. + + Parameters + ---------- + token_logits_flat : numpy.ndarray, shape (n_codebook * codebook_size,) + Flattened per-digit next-item logits (digit-by-digit). + adjacency : numpy.ndarray, shape (n_items, edges) + Precomputed kNN neighbour indices. + item_tokens : numpy.ndarray, shape (n_items, n_codebook) + Per-item *offset* tokens (so ``item_tokens - 1`` indexes + ``token_logits_flat``). + num_beams, prop_steps : int + rng : numpy.random.RandomState + + Returns + ------- + (beam, scores) : two aligned numpy arrays over the final beam item indices. + """ + n_items = adjacency.shape[0] + + def score_nodes(nodes): + return token_logits_flat[item_tokens[nodes] - 1].mean(axis=1) + + beam = rng.randint(0, n_items, size=min(num_beams, n_items)) + for _ in range(prop_steps): + neighbors = np.unique(adjacency[beam].reshape(-1)) + nb_scores = score_nodes(neighbors) + top = np.argsort(-nb_scores)[:num_beams] + beam = neighbors[top] + return beam.astype("int64", copy=False), score_nodes(beam) diff --git a/cornac/models/rpg/rpg_config.py b/cornac/models/rpg/rpg_config.py new file mode 100644 index 000000000..fa55f11ac --- /dev/null +++ b/cornac/models/rpg/rpg_config.py @@ -0,0 +1,117 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Official RPG recipe (Amazon 2014) as an importable config. + +Transcribed from the official code +https://github.com/facebookresearch/RPG_KDD2025 -- backbone/tokenizer/decoding +from ``genrec/models/RPG/config.yaml`` and training hyperparameters from +``genrec/default.yaml``: + +* embeddings ``text-embedding-3-large`` (3072-d) -> whitened PCA to 512; +* OPQ tokenizer ``OPQ32,IVF1,PQ32x8`` (32 codebooks x 256 codes); +* GPT-2 backbone: seq len 50, d_model 448, 2 layers, 4 heads, inner 1024, + ``gelu_new``, dropout embd 0.5 / attn 0.5 / resid 0.0, layer-norm eps 1e-12; +* MTP temperature 0.07; +* graph decoding: beam 50, edges 50, propagation 3, chunk 1024; +* AdamW lr 3e-4, weight decay 0.0, batch 256, cosine schedule with 10k warmup, + grad-norm clip 1.0. + +Usage:: + + from cornac.models.rpg import RPG_CONFIG, RPG + + model = RPG(**{**RPG_CONFIG, "seed": 2024}) + +The official trainer runs up to 150 epochs, evaluates every epoch in batches +of 32, and early-stops after 20 non-improving ``ndcg@10`` epochs. These settings +are reproduced below. Per-dataset paper-best settings (from the README's +reproduce commands) ship as :data:`RPG_BEAUTY_CONFIG` / +:data:`RPG_SPORTS_CONFIG` / :data:`RPG_TOYS_CONFIG`; ``n_codebook=64`` is the +CDs_and_Vinyl long-semantic-ID setting. +""" + +RPG_CONFIG = dict( + pca_dim=512, + n_codebook=32, + codebook_size=256, + max_len=50, + d_model=448, + n_layer=2, + n_head=4, + n_inner=1024, + activation="gelu_new", + resid_dropout=0.0, + embd_dropout=0.5, + attn_dropout=0.5, + layer_norm_eps=1e-12, + initializer_range=0.02, + temperature=0.07, + scoring="graph", + n_beams=50, + graph_edges=50, + propagation_steps=3, + graph_chunk_size=1024, + learning_rate=3e-4, + weight_decay=0.0, + batch_size=256, + n_epochs=150, + lr_schedule="cosine", + warmup_steps=10000, + max_grad_norm=1.0, + model_selection="best", + val_metric="ndcg", + val_k=10, + val_eval_every=1, + val_batch_size=32, + early_stopping_patience=20, + val_sample=None, +) + +# Per-dataset reproduce settings, verbatim from the official README's +# reproduce commands (the repo's config.yaml/default.yaml above are its +# *defaults*, not the paper-best values). lr, temperature and n_codebook +# change training; n_beams/graph_edges/propagation_steps only the graph +# decode. Note Sports/Toys tune *shorter* IDs (16 codebooks); 64 is the +# CDs_and_Vinyl setting. + +RPG_BEAUTY_CONFIG = dict( + RPG_CONFIG, + learning_rate=0.01, + temperature=0.03, + n_codebook=32, + n_beams=20, + graph_edges=200, + propagation_steps=3, +) + +RPG_SPORTS_CONFIG = dict( + RPG_CONFIG, + learning_rate=0.003, + temperature=0.03, + n_codebook=16, + n_beams=100, + graph_edges=30, + propagation_steps=5, +) + +RPG_TOYS_CONFIG = dict( + RPG_CONFIG, + learning_rate=0.003, + temperature=0.03, + n_codebook=16, + n_beams=200, + graph_edges=20, + propagation_steps=3, +) diff --git a/docs/source/api_ref/models.rst b/docs/source/api_ref/models.rst index 333264812..d85920754 100644 --- a/docs/source/api_ref/models.rst +++ b/docs/source/api_ref/models.rst @@ -16,6 +16,11 @@ Comparative Aspects and Opinions Ranking for Recommendation Explanations (Compan .. automodule:: cornac.models.companion.recom_companion :members: +Generating Long Semantic IDs in Parallel for Recommendation (RPG) +----------------------------------------------------------------- +.. automodule:: cornac.models.rpg.recom_rpg + :members: + Disentangled Multimodal Representation Learning for Recommendation (DMRL) ------------------------------------------------------------------------- .. automodule:: cornac.models.dmrl.recom_dmrl diff --git a/examples/README.md b/examples/README.md index 9aa1592da..fb4b1330c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -132,6 +132,8 @@ [tiger_example.py](tiger_example.py) - Generative retrieval with semantic IDs (TIGER) on Amazon Beauty with Sentence-T5 item content embeddings, reproducing the paper's leave-last-out protocol. +[rpg_example.py](rpg_example.py) - Parallel generation of long unordered semantic IDs (RPG): OPQ tokenizer + multi-token prediction + graph-guided decoding, with Diginetica dataset. + ---- ## Next-Basket Algorithms diff --git a/examples/rpg_example.py b/examples/rpg_example.py new file mode 100644 index 000000000..0e494031b --- /dev/null +++ b/examples/rpg_example.py @@ -0,0 +1,111 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""RPG (parallel long semantic IDs) on Diginetica. + +RPG quantizes item CONTENT embeddings into long *unordered* semantic IDs with +product quantization (OPQ, via faiss), then trains a GPT-2 decoder that predicts +every digit of the next item's semantic ID in parallel (multi-token prediction). +Inference uses a similarity graph over item semantic IDs to guide beam decoding. + +The content embeddings are precomputed and passed to the evaluation method as a +FeatureModality. Diginetica ships without item text in Cornac, so this example +uses random vectors as a stand-in -- replace them with real content embeddings +for meaningful semantic IDs, e.g. with the OpenAI ``text-embedding-3-large`` +embeddings used in the paper, or with sentence-transformers:: + + from sentence_transformers import SentenceTransformer + encoder = SentenceTransformer("sentence-t5-base") + features = encoder.encode([titles[iid] for iid in item_ids]) + +Requires ``faiss-cpu`` and ``scikit-learn`` (see cornac/models/rpg/requirements.txt). +""" + +import numpy as np +import torch + +import cornac +from cornac.data import FeatureModality +from cornac.datasets import diginetica +from cornac.eval_methods import NextItemEvaluation +from cornac.metrics import MRR, NDCG, Recall +from cornac.models import RPG, SASRec + +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" +print(f"using device: {DEVICE}") + +train_data = diginetica.load_train() +val_data = diginetica.load_val() +test_data = diginetica.load_test() +print("data loaded") + +item_ids = sorted({tup[2] for tup in train_data + val_data + test_data}) +print("NOTE: using random item features as a stand-in; " "replace with real content embeddings (see module docstring).") +features = np.random.RandomState(123).randn(len(item_ids), 768).astype("float32") + +next_item_eval = NextItemEvaluation.from_splits( + train_data=train_data, + val_data=val_data, + test_data=test_data, + exclude_unknowns=True, + verbose=True, + fmt="USIT", + item_feature=FeatureModality(features=features, ids=item_ids), +) + +models = [ + RPG( + n_codebook=32, # semantic-ID length; set to 64 for the long-ID config + codebook_size=256, + pca_dim=512, + n_epochs=50, + batch_size=256, + max_len=50, + scoring="graph", # or "exact" for the full-catalog parallel-MTP ranking + n_beams=50, + graph_edges=50, + propagation_steps=3, + device=DEVICE, + verbose=True, + seed=2024, + ), + SASRec( + embedding_dim=64, + loss="cross-entropy", + n_sample=512, + batch_size=128, + n_epochs=100, + max_len=20, + num_blocks=2, + num_heads=2, + learning_rate=0.01, + device=DEVICE, + verbose=True, + seed=123, + ), +] + +metrics = [ + NDCG(k=10), + NDCG(k=50), + Recall(k=10), + Recall(k=50), + MRR(), +] + +cornac.Experiment( + eval_method=next_item_eval, + models=models, + metrics=metrics, +).run() diff --git a/tests/cornac/eval_methods/test_next_item_evaluation.py b/tests/cornac/eval_methods/test_next_item_evaluation.py index 6e3847068..0f2a4f73a 100644 --- a/tests/cornac/eval_methods/test_next_item_evaluation.py +++ b/tests/cornac/eval_methods/test_next_item_evaluation.py @@ -16,7 +16,9 @@ import unittest import warnings -from cornac.data import Reader +import numpy as np + +from cornac.data import FeatureModality, Reader from cornac.eval_methods import NextItemEvaluation from cornac.metrics import HitRatio, Recall from cornac.models import SPop @@ -145,13 +147,18 @@ def test_empty_train_raises(self): def test_empty_val_warns(self): # No session ends in [55, 65): train={s1,s2,s3}, test={s4,s5,s6}, val # empty -> warn and fall back to no validation set. - with self.assertWarns(UserWarning): + # catch_warnings instead of assertWarns: assertWarns probes + # __warningregistry__ on every module in sys.modules, which trips + # transformers 5.x's lazy __getattr__ when model tests ran earlier. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") m = NextItemEvaluation.from_timestamps( self.usit, test_timestamp=65, val_timestamp=55, fmt="USIT", ) + self.assertTrue(any(issubclass(w.category, UserWarning) for w in caught)) self.assertIsNone(m.val_set) def test_verbose(self): @@ -272,5 +279,71 @@ def test_verbose(self): self.assertEqual(_split_sids(m.test_set), {"u1", "u2"}) +class TestGlobalSidMap(unittest.TestCase): + def test_val_set_shares_global_sid_map(self): + # The same raw session (user) appears in train/val/test under + # leave_last_out; its mapped session id must agree across splits. + uirt = [ + ("u1", "a", 1.0, 10), + ("u1", "b", 1.0, 20), + ("u1", "c", 1.0, 30), + ("u2", "b", 1.0, 15), + ("u2", "a", 1.0, 25), + ("u2", "d", 1.0, 35), + ] + m = NextItemEvaluation.leave_last_out(uirt, exclude_unknowns=False) + for raw_sid in ["u1", "u2"]: + self.assertEqual(m.val_set.sid_map[raw_sid], m.train_set.sid_map[raw_sid]) + self.assertEqual(m.test_set.sid_map[raw_sid], m.train_set.sid_map[raw_sid]) + + +class TestItemFeature(unittest.TestCase): + """Content (item_feature) support: modalities passed as kwargs must be + built against the global item-ID map and attached to every split, so + content-based next-item models (e.g., TIGER) can read + ``train_set.item_feature.features`` with rows aligned to item indices.""" + + def setUp(self): + self.items = ["a", "b", "c", "d"] + self.features = np.arange(4 * 3, dtype="float").reshape(4, 3) + self.uirt = [ + ("u1", "a", 1.0, 10), + ("u1", "b", 1.0, 20), + ("u1", "c", 1.0, 30), + ("u2", "b", 1.0, 15), + ("u2", "d", 1.0, 25), + ("u2", "a", 1.0, 35), + ] + + def _modality(self): + return FeatureModality( + features=self.features, ids=self.items, normalized=False + ) + + def test_leave_last_out_attaches_and_aligns(self): + m = NextItemEvaluation.leave_last_out( + self.uirt, exclude_unknowns=False, item_feature=self._modality() + ) + for split in [m.train_set, m.val_set, m.test_set]: + self.assertIsNotNone(split.item_feature) + feats = m.train_set.item_feature.features + # row k of built features == raw features of the item mapped to index k + for raw_id, mapped_idx in m.global_iid_map.items(): + np.testing.assert_array_equal( + feats[mapped_idx], self.features[self.items.index(raw_id)] + ) + + def test_from_splits_attaches(self): + usit = [(u, u, i, t) for u, i, _, t in self.uirt] + m = NextItemEvaluation.from_splits( + train_data=usit[:4], + test_data=usit[4:], + fmt="USIT", + item_feature=self._modality(), + ) + self.assertIsNotNone(m.train_set.item_feature) + self.assertIsNotNone(m.test_set.item_feature) + + if __name__ == "__main__": unittest.main()