Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/release.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@

## Unreleased

### Fixes

* Validate Blosc frames before decompression, allowing valid zero-length frames to round-trip
while rejecting truncated or size-mismatched frames. System Blosc builds now require version
1.16.0 or newer. By {user}`Igor Stadnyk <fallenmi>`, {issue}`831`.

### Maintenance

* **Migrate build system from setuptools/setup.py to meson-python.** This replaces the
Expand Down
14 changes: 11 additions & 3 deletions src/numcodecs/blosc.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ cdef extern from "blosc.h":
int numinternalthreads) nogil
int blosc_decompress_ctx(const void* src, void* dest, size_t destsize,
int numinternalthreads) nogil
int blosc_cbuffer_validate(const void* cbuffer, size_t cbytes, size_t* nbytes)
void blosc_cbuffer_sizes(const void* cbuffer, size_t* nbytes, size_t* cbytes,
size_t* blocksize)
char* blosc_cbuffer_complib(const void* cbuffer)
Expand Down Expand Up @@ -350,7 +351,7 @@ def decompress(source, dest=None):
memoryview dest_mv
Py_buffer* dest_pb
char* dest_ptr
size_t nbytes, cbytes, blocksize
size_t nbytes

# obtain source memoryview
source_mv = ensure_contiguous_memoryview(source)
Expand All @@ -359,8 +360,10 @@ def decompress(source, dest=None):
# get source pointer
source_ptr = <const char*>source_pb.buf

# determine buffer size
blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize)
# validate source and determine decompressed buffer size
ret = blosc_cbuffer_validate(source_ptr, <size_t>source_pb.len, &nbytes)
if ret != 0:
raise RuntimeError('error during blosc decompression: %d' % ret)

# setup destination buffer
if dest is None:
Expand All @@ -382,6 +385,11 @@ def decompress(source, dest=None):
raise ValueError('destination buffer too small; expected at least %s, '
'got %s' % (nbytes, dest_nbytes))

# Blosc reports the number of decompressed bytes, so zero is successful
# for a validated empty frame rather than an error.
if nbytes == 0:
return dest

# perform decompression
if _get_use_threads():
# allow blosc to use threads internally
Expand Down
6 changes: 5 additions & 1 deletion src/numcodecs/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,11 @@ zlib_dep = declare_dependency(
)

# --- Vendored blosc ---
blosc_dep = dependency('blosc', required: get_option('system_blosc'))
blosc_dep = dependency(
'blosc',
version: '>=1.16.0',
required: get_option('system_blosc'),
)

if not blosc_dep.found()
blosc_sources = files(
Expand Down
8 changes: 8 additions & 0 deletions tests/test_blosc.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,14 @@ def test_encode_decode(array, codec):
check_encode_decode(array, codec)


def test_empty_encode_decode(use_threads):
blosc.use_threads = use_threads
try:
check_encode_decode(np.empty(0, dtype='u1'), Blosc())
finally:
blosc.use_threads = None


def test_config():
codec = Blosc(cname='zstd', clevel=3, shuffle=1)
check_config(codec)
Expand Down