Skip to content

Issue 246: add kVariableUTF8 and kVariableASCII #247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jkotan merged 2 commits into from
Aug 12, 2026
Merged

Issue 246: add kVariableUTF8 and kVariableASCII #247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a78c429
add kVariableUTF8 and kVariableASCII
jkotan Aug 12, 2026
d4e6ff2
update tests
jkotan Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/debian11_py3/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ RUN chmod 644 /etc/apt/trusted.gpg.d/debian-hdri-repo.gpg
RUN add-apt-repository "deb http://repos.pni-hdri.de/apt/debian bullseye main" -y
# RUN apt-get -qq update && apt-get -qq -y dist-upgrade
RUN apt-get -qq update && apt-get -qq install -y python3-sphinx apt-utils net-tools
RUN apt-get install -y libpninexus3.5.0-dev libpninexus3.5.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev libatlas-base-dev cython3 python3-setuptools libboost-python-dev python3-numpy-abi9 g++ python3-h5py hdf5-plugin-bshuf hdf5-plugin-bz2 hdf5-plugin-lz4 git python3-pytest cmake doxygen python3-sphinx libboost-all-dev
RUN apt-get install -y libpninexus3.6.0-dev libpninexus3.6.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev libatlas-base-dev cython3 python3-setuptools libboost-python-dev python3-numpy-abi9 g++ python3-h5py hdf5-plugin-bshuf hdf5-plugin-bz2 hdf5-plugin-lz4 git python3-pytest cmake doxygen python3-sphinx libboost-all-dev
RUN useradd -ms /bin/bash tango
RUN apt-get -qq install -y adduser
RUN /bin/bash -c 'sleep 10'
Expand Down
2 changes: 1 addition & 1 deletion .ci/debian12_py3/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ RUN chmod 644 /etc/apt/trusted.gpg.d/debian-hdri-repo.gpg
RUN cd /etc/apt/sources.list.d && wget http://repos.pni-hdri.de/bookworm-pni-hdri.list
RUN apt-get -qq update && apt-get -qq -y dist-upgrade
RUN apt-get -qq update && apt-get -qq install -y python3-sphinx apt-utils net-tools
RUN apt-get install -y libpninexus3.5.0-dev libpninexus3.5.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev libatlas-base-dev cython3 python3-setuptools libboost-python-dev python3-numpy-abi9 g++ python3-h5py git python3-pytest cmake doxygen python3-sphinx libboost-all-dev hdf5-filter-plugin hdf5-plugin-bshuf
RUN apt-get install -y libpninexus3.6.0-dev libpninexus3.6.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev libatlas-base-dev cython3 python3-setuptools libboost-python-dev python3-numpy-abi9 g++ python3-h5py git python3-pytest cmake doxygen python3-sphinx libboost-all-dev hdf5-filter-plugin hdf5-plugin-bshuf
# hdf5-plugin-bz2 hdf5-plugin-lz4

RUN useradd -ms /bin/bash tango
Expand Down
2 changes: 1 addition & 1 deletion .ci/debian13_py3/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ RUN chmod 644 /etc/apt/trusted.gpg.d/debian-hdri-repo.gpg
RUN cd /etc/apt/sources.list.d && wget http://repos.pni-hdri.de/trixie-pni-hdri.list
RUN apt-get -qq update && apt-get -qq -y dist-upgrade
RUN apt-get -qq update && apt-get -qq install -y python3-sphinx apt-utils net-tools
RUN apt-get install -y libpninexus3.5.0-dev libpninexus3.5.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev cython3 python3-setuptools libboost-python-dev python3-numpy-dev g++ python3-h5py git python3-pytest cmake doxygen python3-sphinx libboost-all-dev hdf5-filter-plugin bitshuffle
RUN apt-get install -y libpninexus3.6.0-dev libpninexus3.6.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev cython3 python3-setuptools libboost-python-dev python3-numpy-dev g++ python3-h5py git python3-pytest cmake doxygen python3-sphinx libboost-all-dev hdf5-filter-plugin bitshuffle
# hdf5-plugin-bz2 hdf5-plugin-lz4

RUN useradd -ms /bin/bash tango
Expand Down
2 changes: 1 addition & 1 deletion .ci/ubuntu22.04_py3/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ RUN chmod 644 /etc/apt/trusted.gpg.d/debian-hdri-repo.gpg
RUN add-apt-repository "deb http://repos.pni-hdri.de/apt/ubuntu jammy main" -y
RUN apt-get -qq update && export DEBIAN_FRONTEND=noninteractive && apt-get -qq dist-upgrade
RUN apt-get -qq update && export DEBIAN_FRONTEND=noninteractive && apt-get -qq install -y python3-sphinx apt-utils net-tools
RUN apt-get -qq install -y libpninexus3.5.0-dev libpninexus3.5.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev libatlas-base-dev cython3 python3-setuptools libboost-python-dev python3-numpy-abi9 g++ python3-h5py
RUN apt-get -qq install -y libpninexus3.6.0-dev libpninexus3.6.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev libatlas-base-dev cython3 python3-setuptools libboost-python-dev python3-numpy-abi9 g++ python3-h5py
RUN apt-get -qq install -y hdf5-plugin-bshuf hdf5-plugin-bz2 hdf5-plugin-lz4 python3-pytest git cmake doxygen python3-sphinx libboost-all-dev
RUN useradd -ms /bin/bash tango

Expand Down
2 changes: 1 addition & 1 deletion .ci/ubuntu24.04_py3/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ RUN cd /etc/apt/sources.list.d && wget http://repos.pni-hdri.de/noble-pni-hdri.l

RUN apt-get -qq update && export DEBIAN_FRONTEND=noninteractive && apt-get -qq dist-upgrade
RUN apt-get -qq update && export DEBIAN_FRONTEND=noninteractive && apt-get -qq install -y python3-sphinx apt-utils net-tools
RUN apt-get -qq install -y libpninexus3.5.0-dev libpninexus3.5.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev libatlas-base-dev cython3 python3-setuptools libboost-python-dev python3-numpy-abi9 g++ python3-h5py
RUN apt-get -qq install -y libpninexus3.6.0-dev libpninexus3.6.0 libh5cpp0.8.0-dev libh5cpp0.8.0 python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev libatlas-base-dev cython3 python3-setuptools libboost-python-dev python3-numpy-abi9 g++ python3-h5py
RUN apt-get -qq install -y hdf5-filter-plugin hdf5-plugin-bshuf python3-pytest git cmake doxygen python3-sphinx libboost-all-dev
RUN useradd -ms /bin/bash tango

Expand Down
2 changes: 1 addition & 1 deletion .ci/ubuntu26.04_py3/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ RUN apt-get -qq update && export DEBIAN_FRONTEND=noninteractive && apt-get -qq d
RUN apt-get -qq update && export DEBIAN_FRONTEND=noninteractive && apt-get -qq install -y python3-sphinx apt-utils net-tools
RUN apt-get -qq install -y python3-numpy libhdf5-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-date-time-dev libboost-filesystem-dev libblas-dev cython3 python3-setuptools libboost-python-dev python3-numpy-dev g++ python3-h5py
RUN apt-get -qq install -y hdf5-filter-plugin bitshuffle python3-pytest git cmake doxygen python3-sphinx libboost-all-dev
RUN apt-get -qq install -y libpninexus3.5.0-dev libpninexus3.5.0 libh5cpp0.8.0-dev libh5cpp0.8.0
RUN apt-get -qq install -y libpninexus3.6.0-dev libpninexus3.6.0 libh5cpp0.8.0-dev libh5cpp0.8.0
RUN useradd -ms /bin/bash tango

# # install latest h5cpp
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pixi/recipe.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
context:
version: 3.5.0
version: 3.6.0

package:
name: pninexus
Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# ChangeLog

## 3.6.0 - 2026-08-12
- add kVariableUTF8 ([#247](https://github.com/pni-libraries/python-pninexus/pull/242)
- technical release for libpninexus v3.6.0 (https://github.com/pni-libraries/libpninexus/issues/285) and libh5cpp v0.8.0 (https://github.com/ess-dmsc/h5cpp/issues/679)

## 3.5.0 - 2026-08-10
- add wrapper for file drivers ([#242](https://github.com/pni-libraries/python-pninexus/pull/242)
- technical release for libpninexus v3.5.0 (https://github.com/pni-libraries/libpninexus/issues/280) and libh5cpp v0.8.0 (https://github.com/ess-dmsc/h5cpp/issues/679)
Expand Down
1 change: 1 addition & 0 deletions doc/sphinx/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@

html_context = {
"docs_versions" : [
"v3.6.0",
"v3.5.0",
"v3.4.0",
"v3.3.0",
Expand Down
6 changes: 3 additions & 3 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@
BuildDoc = None

name = "pninexus"
version = "3.5.0"
release = "3.5.0"
version = "3.6.0"
release = "3.6.0"
# documentation for release with one '.' is stored as v:latest
# release = "3.5"
# release = "3.6"

if release.count(".") == 1:
docs_release = '(latest)'
Expand Down
5 changes: 5 additions & 0 deletions src/cpp/h5cpp/datatype/datatype.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,9 @@ BOOST_PYTHON_MODULE(_datatype)

scope current;

auto utf8_type = hdf5::datatype::create<std::string>();
utf8_type.encoding(hdf5::datatype::CharacterEncoding::UTF8);

current.attr("kUInt8") = hdf5::datatype::create<uint8_t>();
current.attr("kInt8") = hdf5::datatype::create<int8_t>();
current.attr("kUInt16") = hdf5::datatype::create<uint16_t>();
Expand All @@ -293,6 +296,8 @@ BOOST_PYTHON_MODULE(_datatype)
current.attr("kComplex128") = hdf5::datatype::create<std::complex<double>>();
current.attr("kComplex256") = hdf5::datatype::create<std::complex<long double>>();
current.attr("kVariableString") = hdf5::datatype::create<std::string>();
current.attr("kVariableASCII") = hdf5::datatype::create<std::string>();
current.attr("kVariableUTF8") = utf8_type;
current.attr("kEBool") = hdf5::datatype::create<hdf5::datatype::EBool>();

//need some functions
Expand Down
4 changes: 3 additions & 1 deletion src/pninexus/h5cpp/datatype/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
from pninexus.h5cpp._datatype import kComplex128
from pninexus.h5cpp._datatype import kComplex256
from pninexus.h5cpp._datatype import kVariableString
from pninexus.h5cpp._datatype import kVariableASCII
from pninexus.h5cpp._datatype import kVariableUTF8
from pninexus.h5cpp._datatype import kEBool

from pninexus.h5cpp._datatype import is_bool
Expand Down Expand Up @@ -179,5 +181,5 @@ def compound_getitem(self, key):
"kUInt64", "kInt64",
"kFloat16", "kFloat32", "kFloat64", "kFloat128",
"kComplex32", "kComplex64", "kComplex128", "kComplex256",
"kVariableString",
"kVariableString", "kVariableASCII", "kVariableUTF8",
"Factory", "kFactory", "to_numpy", "kEBool", "is_bool", "Enum"]
2 changes: 1 addition & 1 deletion src/pninexus/nexus/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ def field_factory_create(parent, name, dtype, shape=None, max_shape=None,
raise TypeError("`units` must be an instance of `str`!")

unit_attr = dataset.attributes.create(
"units", h5cpp.datatype.kVariableString)
"units", h5cpp.datatype.kVariableUTF8)
unit_attr.write(units)

return dataset
Expand Down
19 changes: 15 additions & 4 deletions test/h5cpp_tests/attribute_tests/attribute_io_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@
from pninexus import h5cpp
from pninexus.h5cpp.file import AccessFlags
from pninexus.h5cpp.datatype import (
kInt32, kFloat32, kUInt8, kVariableString, kEBool)
kInt32, kFloat32, kUInt8, kVariableString,
kVariableUTF8, kVariableASCII, kEBool)
from pninexus.h5cpp.datatype import StringPad, String


Expand Down Expand Up @@ -129,13 +130,23 @@ def testStringScalarVariableLength(self):
r = a.read()
self.assertEqual(r, data)

def testStringASCIIScalarVariableLength(self):

data = "hello world"
a = self.root.attributes.create(
"StringASCIIScalarVLength", kVariableASCII)
a.write(data)
r = a.read()
self.assertEqual(r, data)

def testStringUTF8ScalarVariableLength(self):

data = u"\u03bcm"
bdata = b"\xce\xbcm"
dtype = String.variable()
dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
a = self.root.attributes.create("StringUTF8ScalarVLength", dtype)
# dtype = String.variable()
# dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
a = self.root.attributes.create(
"StringUTF8ScalarVLength", kVariableUTF8)
a.write(data)
r = a.read()
if sys.version_info > (3,):
Expand Down
8 changes: 4 additions & 4 deletions test/h5cpp_tests/file_image_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

from pninexus.h5cpp.file import File
from pninexus.h5cpp.file import ImageFlags
from pninexus.h5cpp.datatype import kVariableString
from pninexus.h5cpp.datatype import kVariableUTF8
from pninexus.h5cpp.file import AccessFlags
from pninexus.h5cpp.file import create, from_buffer
from pninexus.h5cpp.file import open as h5open
Expand Down Expand Up @@ -62,7 +62,7 @@ def test_image_from_buffer(self):
hdf5_version = "1.0.4"
f1 = create(self.filename, AccessFlags.TRUNCATE)
r1 = f1.root()
a1 = r1.attributes.create("HDF5_Version", kVariableString)
a1 = r1.attributes.create("HDF5_Version", kVariableUTF8)
a1.write(hdf5_version)
a1.close()
r1.close()
Expand All @@ -84,7 +84,7 @@ def test_image_to_buffer(self):
hdf5_version = "1.0.3"
f1 = create(self.filename, AccessFlags.TRUNCATE)
r1 = f1.root()
a1 = r1.attributes.create("HDF5_Version", kVariableString)
a1 = r1.attributes.create("HDF5_Version", kVariableUTF8)
a1.write(hdf5_version)
size = f1.buffer_size
obuffer = np.zeros(shape=[size], dtype='uint8')
Expand All @@ -110,7 +110,7 @@ def test_image_buffer(self):
hdf5_version = "1.0.2"
f1 = create(self.filename, AccessFlags.TRUNCATE)
r1 = f1.root()
a1 = r1.attributes.create("HDF5_Version", kVariableString)
a1 = r1.attributes.create("HDF5_Version", kVariableUTF8)
a1.write(hdf5_version)

size = f1.buffer_size
Expand Down
Loading