Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 23 additions & 5 deletions deepmd/dpmodel/utils/lmdb_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,18 @@ def _remap_keys(frame: dict[str, Any]) -> dict[str, Any]:
return out


def _remap_atom_types(atype: np.ndarray, type_remap: np.ndarray) -> np.ndarray:
"""Remap real atom types while preserving negative virtual sentinels.

Positive indices retain NumPy's normal bounds checking, so malformed LMDB
data cannot be silently reinterpreted as a different species.
"""
remapped_atype = atype.astype(np.int64, copy=True)
real_atom_mask = remapped_atype >= 0
remapped_atype[real_atom_mask] = type_remap[remapped_atype[real_atom_mask]]
return remapped_atype


def is_lmdb(systems: str) -> bool:
"""Check if systems points to an LMDB dataset."""
return systems.endswith(".lmdb") or Path(systems, "data.mdb").is_file()
Expand Down Expand Up @@ -466,10 +478,16 @@ def __init__(
def _compute_natoms_vec(self, atype: np.ndarray) -> np.ndarray:
"""Compute natoms_vec from a frame's atype array.

Negative virtual types are excluded from the per-type counts, matching
mixed-type NPY data handling. This function also excludes positive
indices outside the configured type map. The leading nloc entries still
include every atom slot.

Returns [nloc, nloc, count_type0, count_type1, ...] with length ntypes+2.
"""
nloc = len(atype)
counts = np.bincount(atype, minlength=self._ntypes)[: self._ntypes]
real_atype = atype[(atype >= 0) & (atype < self._ntypes)]
counts = np.bincount(real_atype, minlength=self._ntypes)
vec = np.empty(self._ntypes + 2, dtype=np.int64)
vec[0] = nloc
vec[1] = nloc
Expand Down Expand Up @@ -579,7 +597,7 @@ def __getitem__(self, index: int) -> dict[str, Any]:
frame["atype"] = frame["atype"].reshape(-1).astype(np.int64)
# Remap atom types from LMDB's type_map to model's type_map
if self._type_remap is not None:
frame["atype"] = self._type_remap[frame["atype"]].astype(np.int64)
frame["atype"] = _remap_atom_types(frame["atype"], self._type_remap)
if "virial" in frame and isinstance(frame["virial"], np.ndarray):
frame["virial"] = (
frame["virial"].reshape(9).astype(self._resolve_dtype("virial"))
Expand Down Expand Up @@ -1459,9 +1477,9 @@ def __init__(
and "atype" in frame
and isinstance(frame["atype"], np.ndarray)
):
frame["atype"] = self._type_remap[
frame["atype"].reshape(-1)
].astype(np.int64)
frame["atype"] = _remap_atom_types(
frame["atype"].reshape(-1), self._type_remap
)
self._frames.append(frame)

# Shuffle if requested
Expand Down
5 changes: 3 additions & 2 deletions deepmd/dpmodel/utils/stat.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ def collect_observed_types(sampled: list[dict], type_map: list[str]) -> list[str
Returns
-------
list[str]
Sorted list of observed element symbols.
Sorted list of observed element symbols. Negative virtual atom types
and indices outside ``type_map`` are ignored.
"""
from deepmd.utils.econf_embd import (
sort_element_type,
Expand All @@ -54,7 +55,7 @@ def collect_observed_types(sampled: list[dict], type_map: list[str]) -> list[str
atype = to_numpy_array(system["atype"]) # shape: [nframes, natoms]
observed_indices.update(np.unique(atype).tolist())
observed_types = [
type_map[i] for i in sorted(observed_indices) if i < len(type_map)
type_map[i] for i in sorted(observed_indices) if 0 <= i < len(type_map)
]
return sort_element_type(observed_types)

Expand Down
56 changes: 56 additions & 0 deletions source/tests/common/dpmodel/test_lmdb_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
LmdbTestDataNlocView,
SameNlocBatchSampler,
_expand_indices_by_blocks,
_remap_atom_types,
compute_block_targets,
is_lmdb,
make_neighbor_stat_data,
Expand Down Expand Up @@ -165,6 +166,44 @@ def _create_lmdb_with_type_map(
return path


def _create_lmdb_with_virtual_type(path: str) -> str:
"""Create one LMDB frame containing a negative virtual atom type."""
atype = np.array([0, -1, 1], dtype=np.int64)
frame = _make_frame(natoms=len(atype))
frame["atom_types"] = {
"type": "<i8",
"shape": atype.shape,
"data": atype.tobytes(),
}
frame["atom_numbs"] = [
{
"type": "<i8",
"shape": (1,),
"data": np.array([1], dtype=np.int64).tobytes(),
},
{
"type": "<i8",
"shape": (1,),
"data": np.array([1], dtype=np.int64).tobytes(),
},
]

env = lmdb.open(path, map_size=10 * 1024 * 1024)
with env.begin(write=True) as txn:
meta = {
"nframes": 1,
"frame_idx_fmt": "012d",
"type_map": ["O", "H"],
# Virtual atoms occupy coordinate slots but are not real species.
"system_info": {"natoms": [1, 1]},
"frame_nlocs": [len(atype)],
}
txn.put(b"__metadata__", msgpack.packb(meta, use_bin_type=True))
txn.put(b"000000000000", msgpack.packb(frame, use_bin_type=True))
env.close()
return path


def _create_lmdb_with_system_ids(
path: str,
system_frames: list[int],
Expand Down Expand Up @@ -571,6 +610,23 @@ def test_testdata_no_type_map_in_metadata(self):
self.assertIsNone(td._type_remap)
tmpdir.cleanup()

def test_virtual_type_preserved_during_remap(self):
"""Both LMDB consumers must retain virtual sentinels when remapping."""
path = _create_lmdb_with_virtual_type(f"{self._tmpdir.name}/virtual_type.lmdb")

reader = LmdbDataReader(path, ["H", "O"])
frame = reader[0]
np.testing.assert_array_equal(frame["atype"], [1, -1, 0])
np.testing.assert_array_equal(frame["natoms"], [3, 3, 1, 1])

test_data = LmdbTestData(path, type_map=["H", "O"], shuffle_test=False)
np.testing.assert_array_equal(test_data.get_test()["type"], [[1, -1, 0]])

def test_positive_out_of_range_type_still_raises(self):
"""A malformed real type must not be mistaken for a virtual sentinel."""
with self.assertRaises(IndexError):
_remap_atom_types(np.array([0, 2]), np.array([1, 0]))


# ============================================================
# auto_prob / frame_system_ids tests
Expand Down
9 changes: 9 additions & 0 deletions source/tests/common/dpmodel/test_observed_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,15 @@ def test_out_of_range_index_ignored(self) -> None:
result = collect_observed_types(sampled, type_map)
self.assertEqual(result, ["O"])

def test_virtual_type_ignored(self) -> None:
"""Negative virtual types must not alias the end of the type map."""
sampled = [
{"atype": np.array([[0, -1, 1]])},
]
type_map = ["O", "H", "Au"]
result = collect_observed_types(sampled, type_map)
self.assertEqual(result, ["H", "O"])


class TestObservedTypeStatFile(unittest.TestCase):
"""Test stat file save/load round-trip for observed_type (dpmodel)."""
Expand Down
Loading