From b7865626c4548a120621870d11b7a99a79277664 Mon Sep 17 00:00:00 2001 From: "Phong X. Nguyen" Date: Wed, 10 Jun 2026 21:38:00 +0000 Subject: [PATCH 1/8] Add lz4 support to CLFUS --- CMakeLists.txt | 5 +++ ci/docker/deb/Dockerfile | 2 +- cmake/FindLZ4.cmake | 43 +++++++++++++++++++ doc/admin-guide/files/records.yaml.en.rst | 3 +- doc/admin-guide/storage/index.en.rst | 1 + .../cache-architecture/ram-cache.en.rst | 21 +++++---- include/iocore/cache/Cache.h | 12 +++++- include/tscore/ink_config.h.cmake.in | 1 + src/iocore/cache/CMakeLists.txt | 4 ++ src/iocore/cache/CacheProcessor.cc | 5 +++ src/iocore/cache/RamCacheCLFUS.cc | 34 ++++++++++++++- src/traffic_layout/CMakeLists.txt | 4 ++ src/traffic_layout/info.cc | 15 +++++++ 13 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 cmake/FindLZ4.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index fcd8b2eef13..2bdbc12dfe9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -393,6 +393,11 @@ else() set(HAVE_ZSTD_H FALSE) endif() +find_package(LZ4) +if(LZ4_FOUND) + set(HAVE_LZ4_H TRUE) +endif() + # ncurses is used in traffic_top find_package(Curses) set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H}) diff --git a/ci/docker/deb/Dockerfile b/ci/docker/deb/Dockerfile index 85815163e2f..cd7d4bb89bf 100644 --- a/ci/docker/deb/Dockerfile +++ b/ci/docker/deb/Dockerfile @@ -56,7 +56,7 @@ RUN apt-get update; apt-get -y dist-upgrade; \ libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \ tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \ libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \ - libzstd-dev; \ + libzstd-dev liblz4-dev; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \ groupadd -g 665 jenkins && \ diff --git a/cmake/FindLZ4.cmake b/cmake/FindLZ4.cmake new file mode 100644 index 00000000000..c36a6573c32 --- /dev/null +++ b/cmake/FindLZ4.cmake @@ -0,0 +1,43 @@ +find_path( + LZ4_INCLUDE_DIR + NAMES lz4.h + DOC "lz4 include directory" +) +mark_as_advanced(LZ4_INCLUDE_DIR) +find_library( + LZ4_LIBRARY + NAMES lz4 liblz4 + DOC "lz4 library" +) +mark_as_advanced(LZ4_LIBRARY) + +if(LZ4_INCLUDE_DIR) + file(STRINGS "${LZ4_INCLUDE_DIR}/lz4.h" _lz4_version_lines REGEX "#define[ \t]+LZ4_VERSION_(MAJOR|MINOR|RELEASE)") + string(REGEX REPLACE ".*LZ4_VERSION_MAJOR *\([0-9]*\).*" "\\1" _lz4_version_major "${_lz4_version_lines}") + string(REGEX REPLACE ".*LZ4_VERSION_MINOR *\([0-9]*\).*" "\\1" _lz4_version_minor "${_lz4_version_lines}") + string(REGEX REPLACE ".*LZ4_VERSION_RELEASE *\([0-9]*\).*" "\\1" _lz4_version_release "${_lz4_version_lines}") + set(LZ4_VERSION "${_lz4_version_major}.${_lz4_version_minor}.${_lz4_version_release}") + unset(_lz4_version_major) + unset(_lz4_version_minor) + unset(_lz4_version_release) + unset(_lz4_version_lines) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + LZ4 + REQUIRED_VARS LZ4_LIBRARY LZ4_INCLUDE_DIR + VERSION_VAR LZ4_VERSION +) + +if(LZ4_FOUND) + set(LZ4_INCLUDE_DIRS "${LZ4_INCLUDE_DIR}") + set(LZ4_LIBRARIES "${LZ4_LIBRARY}") + + if(NOT TARGET LZ4::LZ4) + add_library(LZ4::LZ4 UNKNOWN IMPORTED) + set_target_properties( + LZ4::LZ4 PROPERTIES IMPORTED_LOCATION "${LZ4_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}" + ) + endif() +endif() diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index b362089c894..c7ed80b0d9e 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2927,9 +2927,10 @@ RAM Cache Value Description ======== =================================================================== ``0`` No compression - ``1`` Fastlz (extremely fast, relatively low compression) + ``1`` Fastlz (extremely fast, relatively low compression) - prefer lz4 ``2`` Libz (moderate speed, reasonable compression) ``3`` Liblzma (very slow, high compression) + ``4`` lz4 (extremely fast, relatively low compression) ======== =================================================================== Compression runs on task threads. To use more cores for RAM cache diff --git a/doc/admin-guide/storage/index.en.rst b/doc/admin-guide/storage/index.en.rst index f68808b1016..fa7d94e6ed8 100644 --- a/doc/admin-guide/storage/index.en.rst +++ b/doc/admin-guide/storage/index.en.rst @@ -105,6 +105,7 @@ Value Meaning 1 *fastlz* compression 2 *libz* compression 3 *liblzma* compression +4 *lz4* compression ======= ============================= .. _changing-the-size-of-the-ram-cache: diff --git a/doc/developer-guide/cache-architecture/ram-cache.en.rst b/doc/developer-guide/cache-architecture/ram-cache.en.rst index bb6851e2e2d..690888df5b3 100644 --- a/doc/developer-guide/cache-architecture/ram-cache.en.rst +++ b/doc/developer-guide/cache-architecture/ram-cache.en.rst @@ -37,7 +37,7 @@ following features: * Is Scan Resistant and extracts robust hit rates even when the working set does not fit in the RAM Cache. -* Supports compression at 3 levels: fastlz, gzip (libz), and xz (liblzma). +* Supports compression at 4 levels: fastlz, gzip (libz), xz (liblzma) and lz4. Compression can be moved to another thread. * Has very low CPU overhead, only slightly more than a basic LRU. Rather than @@ -72,7 +72,7 @@ len Length of the object, which differs from *size* because of compression and padding). compressed_len Compressed length of the object. compressed Compression type, or ``none`` if no compression. Possible types - are: *fastlz*, *libz*, and *liblzma*. + are: *fastlz*, *libz*, *liblzma*, and *lz4*. uncompressible Flag indicating that content cannot be compressed (true), or that it mat be compressed (false). copy Whether or not this object should be copied in and copied out @@ -147,17 +147,20 @@ since we need to make a copy anyway. Those not tagged ``copy`` are inserted uncompressed in the hope that they can be reused in uncompressed form. This is a compile time option and may be something we want to change. -There are 3 algorithms and levels of compression (speed on an Intel i7 920 -series processor using one thread): +There are 4 algorithms and levels of compression (speed on an Intel Xeon Gold +6338 processor using lzbench and the silesia XML benchmark): ======= ================ ================== ==================================== Method Compression Rate Decompression Rate Notes ======= ================ ================== ==================================== -fastlz 173 MB/sec 442 MB/sec Basically free since disk or network - will limit first; ~53% final size. -libz 55 MB/sec 234 MB/sec Almost free, particularly - decompression; ~37% final size. -liblzma 3 MB/sec 50 MB/sec Expensive; ~27% final size. +fastlz 452 MB/sec 913 MB/sec Effectively obsolete; prefer lz4. + Basically free since disk or network + will limit first; ~26% final size. +libz 54 MB/sec 536 MB/sec Almost free, particularly + decompression; ~13% final size. +liblzma 5 MB/sec 291 MB/sec Expensive; ~8% final size. +lz4 727 MB/sec 3458 MB/sec Basically free since disk or network + will limit first; 23% final size ======= ================ ================== ==================================== These are ballpark numbers, and your millage will vary enormously. JPEG, for diff --git a/include/iocore/cache/Cache.h b/include/iocore/cache/Cache.h index 3a7da523b56..492628fbfa9 100644 --- a/include/iocore/cache/Cache.h +++ b/include/iocore/cache/Cache.h @@ -43,8 +43,16 @@ static constexpr ts::ModuleVersion CACHE_MODULE_VERSION(1, 0); #define CACHE_COMPRESSION_FASTLZ 1 #define CACHE_COMPRESSION_LIBZ 2 #define CACHE_COMPRESSION_LIBLZMA 3 - -enum { RAM_HIT_COMPRESS_NONE = 1, RAM_HIT_COMPRESS_FASTLZ, RAM_HIT_COMPRESS_LIBZ, RAM_HIT_COMPRESS_LIBLZMA, RAM_HIT_LAST_ENTRY }; +#define CACHE_COMPRESSION_LZ4 4 + +enum { + RAM_HIT_COMPRESS_NONE = 1, + RAM_HIT_COMPRESS_FASTLZ, + RAM_HIT_COMPRESS_LIBZ, + RAM_HIT_COMPRESS_LIBLZMA, + RAM_HIT_COMPRESS_LZ4, + RAM_HIT_LAST_ENTRY +}; struct CacheVC; class CacheEvacuateDocVC; diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index 73c8b860fb9..11e78f69e47 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -47,6 +47,7 @@ #cmakedefine HAVE_POSIX_FALLOCATE 1 #cmakedefine HAVE_POSIX_MADVISE 1 #cmakedefine HAVE_ZSTD_H 1 +#cmakedefine HAVE_LZ4_H 1 #cmakedefine HAVE_PTHREAD_GETNAME_NP 1 #cmakedefine HAVE_PTHREAD_GET_NAME_NP 1 diff --git a/src/iocore/cache/CMakeLists.txt b/src/iocore/cache/CMakeLists.txt index f8b3817f722..d0e37e3fadb 100644 --- a/src/iocore/cache/CMakeLists.txt +++ b/src/iocore/cache/CMakeLists.txt @@ -56,6 +56,10 @@ if(HAVE_LZMA_H) target_link_libraries(inkcache PRIVATE LibLZMA::LibLZMA) endif() +if(HAVE_LZ4_H) + target_link_libraries(inkcache PRIVATE LZ4::LZ4) +endif() + if(BUILD_TESTING) # Unit Tests with unit_tests/main.cc macro(add_cache_test name) diff --git a/src/iocore/cache/CacheProcessor.cc b/src/iocore/cache/CacheProcessor.cc index ffff468ef78..f145d89aa87 100644 --- a/src/iocore/cache/CacheProcessor.cc +++ b/src/iocore/cache/CacheProcessor.cc @@ -1648,6 +1648,11 @@ CacheProcessor::cacheInitialized() case CACHE_COMPRESSION_LIBLZMA: #ifndef HAVE_LZMA_H Fatal("lzma not available for RAM cache compression"); +#endif + break; + case CACHE_COMPRESSION_LZ4: +#ifndef HAVE_LZ4_H + Fatal("lz4 not available for RAM cache compression"); #endif break; } diff --git a/src/iocore/cache/RamCacheCLFUS.cc b/src/iocore/cache/RamCacheCLFUS.cc index 639d2faa33c..346191e81e7 100644 --- a/src/iocore/cache/RamCacheCLFUS.cc +++ b/src/iocore/cache/RamCacheCLFUS.cc @@ -35,6 +35,9 @@ #ifdef HAVE_LZMA_H #include #endif +#ifdef HAVE_LZ4_H +#include +#endif #define REQUIRED_COMPRESSION 0.9 // must get to this size or declared incompressible #define REQUIRED_SHRINK 0.8 // must get to this size or keep original buffer (with padding) @@ -164,6 +167,11 @@ RamCacheCLFUSCompressor::mainEvent(int /* event ATS_UNUSED */, Event *e) case CACHE_COMPRESSION_LIBLZMA: #ifndef HAVE_LZMA_H Warning("lzma not available for RAM cache compression"); +#endif + break; + case CACHE_COMPRESSION_LZ4: +#ifndef HAVE_LZ4_H + Warning("lz4 not available for RAM cache compression"); #endif break; } @@ -298,6 +306,16 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey ram_hit_state = RAM_HIT_COMPRESS_LIBLZMA; break; } +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: { + int l = static_cast(e->len); + if (l != LZ4_decompress_safe(e->data->data(), b, e->compressed_len, l)) { + goto Lfailed; + } + ram_hit_state = RAM_HIT_COMPRESS_LZ4; + break; + } #endif } IOBufferData *data = new_xmalloc_IOBufferData(b, e->len); @@ -463,7 +481,12 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) break; #ifdef HAVE_LZMA_H case CACHE_COMPRESSION_LIBLZMA: - l = e->len; + l = static_cast(lzma_stream_buffer_bound(e->len)); + break; +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: + l = static_cast(LZ4_compressBound(e->len)); break; #endif } @@ -504,6 +527,15 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) l = static_cast(pos); break; } +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: { + int ll = l; + if ((l = LZ4_compress_default(edata->data(), b, elen, ll)) == 0) { + failed = true; + } + break; + } #endif } MUTEX_TAKE_LOCK(stripe->mutex, thread); diff --git a/src/traffic_layout/CMakeLists.txt b/src/traffic_layout/CMakeLists.txt index b86d4f2573a..ddc9c2674d8 100644 --- a/src/traffic_layout/CMakeLists.txt +++ b/src/traffic_layout/CMakeLists.txt @@ -35,6 +35,10 @@ if(HAVE_ZSTD_H) target_link_libraries(traffic_layout PRIVATE zstd::zstd) endif() +if(HAVE_LZ4_H) + target_link_libraries(traffic_layout PRIVATE LZ4::LZ4) +endif() + install(TARGETS traffic_layout) clang_tidy_check(traffic_layout) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index e77c2926d3e..30a5c9fd9cc 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -54,6 +54,10 @@ #include #endif +#if HAVE_LZ4_H +#include +#endif + #if HAVE_SSL_CTX_ADD_CERT_COMPRESSION_ALG static constexpr int ts_has_cert_compression_callbacks = 1; #else @@ -133,6 +137,11 @@ produce_features(bool json) print_feature("TS_HAS_ZSTD", 1, json); #else print_feature("TS_HAS_ZSTD", 0, json); +#endif +#ifdef HAVE_LZ4_H + print_feature("TS_HAS_LZ4", 1, json); +#else + print_feature("TS_HAS_LZ4", 0, json); #endif print_feature("TS_HAS_CERT_COMPRESSION", ts_has_cert_compression, json); print_feature("TS_HAS_CERT_COMPRESSION_CALLBACKS", ts_has_cert_compression_callbacks, json); @@ -256,6 +265,12 @@ produce_versions(bool json) #else print_var("zstd", undef, json); #endif +#ifdef HAVE_LZ4_H + print_var("lz4", LBW().print("{}", LZ4_VERSION_STRING).view(), json); + print_var("lz4.run", LBW().print("{}", LZ4_versionString()).view(), json); +#else + print_var("lz4", undef, json); +#endif // This should always be last print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true); From 874ba9c523981a811a1712ac9e7cba45a012bce0 Mon Sep 17 00:00:00 2001 From: "Phong X. Nguyen" Date: Wed, 10 Jun 2026 22:18:32 +0000 Subject: [PATCH 2/8] Extract out definitions from RamCacheCLFUS so we can test compression --- src/iocore/cache/RamCacheCLFUS.cc | 62 +------------------ src/iocore/cache/RamCacheCLFUS.h | 99 +++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 61 deletions(-) create mode 100644 src/iocore/cache/RamCacheCLFUS.h diff --git a/src/iocore/cache/RamCacheCLFUS.cc b/src/iocore/cache/RamCacheCLFUS.cc index 346191e81e7..1587adef8b2 100644 --- a/src/iocore/cache/RamCacheCLFUS.cc +++ b/src/iocore/cache/RamCacheCLFUS.cc @@ -24,6 +24,7 @@ // Clocked Least Frequently Used by Size (CLFUS) replacement policy // See https://cwiki.apache.org/confluence/display/TS/RamCache +#include "RamCacheCLFUS.h" #include "P_RamCache.h" #include "P_CacheInternal.h" #include "StripeSM.h" @@ -65,67 +66,6 @@ DbgCtl dbg_ctl_ram_cache_compare{"ram_cache_compare"}; #endif -struct RamCacheCLFUSEntry { - CryptoHash key; - uint64_t auxkey; - uint64_t hits; - uint32_t size; // memory used including padding in buffer - uint32_t len; // actual data length - uint32_t compressed_len; - union { - struct { - uint32_t compressed : 3; // compression type - uint32_t incompressible : 1; - uint32_t lru : 1; - uint32_t copy : 1; // copy-in-copy-out - } flag_bits; - uint32_t flags; - }; - LINK(RamCacheCLFUSEntry, lru_link); - LINK(RamCacheCLFUSEntry, hash_link); - Ptr data; -}; - -class RamCacheCLFUS : public RamCache -{ -public: - RamCacheCLFUS() {} - - // returns 1 on found/stored, 0 on not found/stored, if provided auxkey1 and auxkey2 must match - int get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey = 0) override; - int put(CryptoHash *key, IOBufferData *data, uint32_t len, bool copy = false, uint64_t auxkey = 0) override; - int fixup(const CryptoHash *key, uint64_t old_auxkey, uint64_t new_auxkey) override; - int64_t size() const override; - - void init(int64_t max_bytes, StripeSM *stripe) override; - - void compress_entries(EThread *thread, int do_at_most = INT_MAX); - - // TODO move it to private. - StripeSM *stripe = nullptr; // for stats -private: - int64_t _max_bytes = 0; - int64_t _bytes = 0; - int64_t _objects = 0; - - double _average_value = 0; - int64_t _history = 0; - int _ibuckets = 0; - int _nbuckets = 0; - DList(RamCacheCLFUSEntry, hash_link) *_bucket = nullptr; - Que(RamCacheCLFUSEntry, lru_link) _lru[2]; - uint16_t *_seen = nullptr; - int _ncompressed = 0; - RamCacheCLFUSEntry *_compressed = nullptr; // first uncompressed lru[0] entry - - void _resize_hashtable(); - void _victimize(RamCacheCLFUSEntry *e); - void _move_compressed(RamCacheCLFUSEntry *e); - RamCacheCLFUSEntry *_destroy(RamCacheCLFUSEntry *e); - void _requeue_victims(Que(RamCacheCLFUSEntry, lru_link) & victims); - void _tick(); // move CLOCK on history -}; - int64_t RamCacheCLFUS::size() const { diff --git a/src/iocore/cache/RamCacheCLFUS.h b/src/iocore/cache/RamCacheCLFUS.h new file mode 100644 index 00000000000..037e0cfb159 --- /dev/null +++ b/src/iocore/cache/RamCacheCLFUS.h @@ -0,0 +1,99 @@ +/** @file + + Clocked Least Frequently Used by Size (CLFUS) RAM cache replacement policy. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +// See https://cwiki.apache.org/confluence/display/TS/RamCache + +#include "P_RamCache.h" + +#include "iocore/eventsystem/IOBuffer.h" +#include "tscore/CryptoHash.h" +#include "tscore/List.h" + +#include +#include + +class EThread; +class StripeSM; + +struct RamCacheCLFUSEntry { + CryptoHash key; + uint64_t auxkey; + uint64_t hits; + uint32_t size; // memory used including padding in buffer + uint32_t len; // actual data length + uint32_t compressed_len; + union { + struct { + uint32_t compressed : 3; // compression type + uint32_t incompressible : 1; + uint32_t lru : 1; + uint32_t copy : 1; // copy-in-copy-out + } flag_bits; + uint32_t flags; + }; + LINK(RamCacheCLFUSEntry, lru_link); + LINK(RamCacheCLFUSEntry, hash_link); + Ptr data; +}; + +class RamCacheCLFUS : public RamCache +{ +public: + RamCacheCLFUS() {} + + // returns 1 on found/stored, 0 on not found/stored, if provided auxkey1 and auxkey2 must match + int get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey = 0) override; + int put(CryptoHash *key, IOBufferData *data, uint32_t len, bool copy = false, uint64_t auxkey = 0) override; + int fixup(const CryptoHash *key, uint64_t old_auxkey, uint64_t new_auxkey) override; + int64_t size() const override; + + void init(int64_t max_bytes, StripeSM *stripe) override; + + void compress_entries(EThread *thread, int do_at_most = INT_MAX); + + // TODO move it to private. + StripeSM *stripe = nullptr; // for stats +private: + int64_t _max_bytes = 0; + int64_t _bytes = 0; + int64_t _objects = 0; + + double _average_value = 0; + int64_t _history = 0; + int _ibuckets = 0; + int _nbuckets = 0; + DList(RamCacheCLFUSEntry, hash_link) *_bucket = nullptr; + Que(RamCacheCLFUSEntry, lru_link) _lru[2]; + uint16_t *_seen = nullptr; + int _ncompressed = 0; + RamCacheCLFUSEntry *_compressed = nullptr; // first uncompressed lru[0] entry + + void _resize_hashtable(); + void _victimize(RamCacheCLFUSEntry *e); + void _move_compressed(RamCacheCLFUSEntry *e); + RamCacheCLFUSEntry *_destroy(RamCacheCLFUSEntry *e); + void _requeue_victims(Que(RamCacheCLFUSEntry, lru_link) & victims); + void _tick(); // move CLOCK on history +}; From 5bd85bae8222c94330d04ae14d4e43a03e3db43e Mon Sep 17 00:00:00 2001 From: "Phong X. Nguyen" Date: Wed, 10 Jun 2026 22:18:41 +0000 Subject: [PATCH 3/8] Add unit tests --- src/iocore/cache/CMakeLists.txt | 1 + .../cache/unit_tests/test_RamCacheCLFUS.cc | 226 ++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc diff --git a/src/iocore/cache/CMakeLists.txt b/src/iocore/cache/CMakeLists.txt index d0e37e3fadb..73f2b10f00c 100644 --- a/src/iocore/cache/CMakeLists.txt +++ b/src/iocore/cache/CMakeLists.txt @@ -95,6 +95,7 @@ if(BUILD_TESTING) add_cache_test(Update_Header unit_tests/test_Update_header.cc) add_cache_test(CacheStripe unit_tests/test_Stripe.cc) add_cache_test(CacheAggregateWriteBuffer unit_tests/test_AggregateWriteBuffer.cc) + add_cache_test(RamCacheCLFUS unit_tests/test_RamCacheCLFUS.cc) # Unit Tests without unit_tests/main.cc add_executable(test_ConfigVolumes unit_tests/test_ConfigVolumes.cc) diff --git a/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc new file mode 100644 index 00000000000..2e6f5dfb4b2 --- /dev/null +++ b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc @@ -0,0 +1,226 @@ +/** @file + + Catch-based unit tests for RAM cache (CLFUS) compression roundtrips. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" + +#include "../RamCacheCLFUS.h" +#include "../P_CacheInternal.h" + +#include "iocore/cache/Cache.h" +#include "tscore/ink_config.h" + +#include +#include +#include +#include + +// Required by main.h +int cache_vols = 1; +bool reuse_existing_cache = false; + +namespace +{ + +// A compression backend to exercise, along with the RAM_HIT_* state get() +// should report once a compressible object has been stored compressed. +struct CompressionCase { + int config; // CACHE_COMPRESSION_* + int expected_hit; // RAM_HIT_COMPRESS_* reported by get() for compressible data + const char *name; +}; + +std::vector +compression_cases() +{ + std::vector cases{ + {CACHE_COMPRESSION_NONE, RAM_HIT_COMPRESS_NONE, "none" }, + {CACHE_COMPRESSION_FASTLZ, RAM_HIT_COMPRESS_FASTLZ, "fastlz"}, + {CACHE_COMPRESSION_LIBZ, RAM_HIT_COMPRESS_LIBZ, "libz" }, + }; +#ifdef HAVE_LZMA_H + cases.push_back({CACHE_COMPRESSION_LIBLZMA, RAM_HIT_COMPRESS_LIBLZMA, "liblzma"}); +#endif +#ifdef HAVE_LZ4_H + cases.push_back({CACHE_COMPRESSION_LZ4, RAM_HIT_COMPRESS_LZ4, "lz4"}); +#endif + return cases; +} + +// Minimal CacheDisk wiring needed to construct a StripeSM. Mirrors the helper +// in test_Stripe.cc. +void +init_disk(CacheDisk &disk) +{ + disk.path = static_cast(ats_malloc(1)); + disk.path[0] = '\0'; + disk.disk_stripes = static_cast(ats_malloc(sizeof(DiskStripe *))); + disk.disk_stripes[0] = nullptr; + disk.header = static_cast(ats_malloc(sizeof(DiskHeader))); + disk.header->num_volumes = 0; +} + +// The CLFUS get/put/compress paths touch only these metrics and the stripe +// mutex, so that is all the stripe needs for these tests. +void +wire_stripe(StripeSM &stripe, CacheVol &cache_vol) +{ + stripe.cache_vol = &cache_vol; + + cache_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.ram_cache.bytes"); + cache_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.hits"); + cache_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.misses"); + cache_vol.vol_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.vol.ram_cache.bytes"); + cache_vol.vol_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.hits"); + cache_vol.vol_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.misses"); +} + +Ptr +make_buffer(const std::vector &bytes) +{ + int64_t idx = iobuffer_size_to_index(bytes.size(), MAX_BUFFER_SIZE_INDEX); + Ptr data{make_ptr(new_IOBufferData(idx, MEMALIGNED))}; + std::memcpy(data->data(), bytes.data(), bytes.size()); + return data; +} + +// Highly compressible: a short repeating pattern. +std::vector +compressible_bytes(std::size_t len) +{ + std::vector bytes(len); + for (std::size_t i = 0; i < len; i++) { + bytes[i] = static_cast('A' + (i % 26)); + } + return bytes; +} + +// Effectively incompressible: a deterministic xorshift byte stream. +std::vector +incompressible_bytes(std::size_t len) +{ + std::vector bytes(len); + uint32_t state = 0x9e3779b9; + for (std::size_t i = 0; i < len; i++) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + bytes[i] = static_cast(state & 0xff); + } + return bytes; +} + +// Store payload under a fresh key, force a synchronous compression pass with +// `config`, then read it back. Returns the RAM_HIT_* state reported by get() +// and the bytes that were returned. +int +store_compress_get(StripeSM &stripe, int config, const std::vector &payload, std::vector &out) +{ + // Initialize with compression disabled so init() does not schedule the + // background compressor (which would retain a pointer to this stack object). + cache_config_ram_cache_compress = CACHE_COMPRESSION_NONE; + cache_config_ram_cache_compress_percent = 100; + cache_config_ram_cache_use_seen_filter = 0; + + RamCacheCLFUS rc; + rc.init(1 << 20, &stripe); + + Ptr in = make_buffer(payload); + uint32_t len = static_cast(payload.size()); + + static uint64_t salt = 0; + ++salt; + CryptoHash key; + key.u64[0] = 0xc0ffee00 + salt; + key.u64[1] = 0xdeadbeef + salt; + + REQUIRE(rc.put(&key, in.get(), len) == 1); + + cache_config_ram_cache_compress = config; + rc.compress_entries(this_ethread()); + + Ptr ret; + int hit = rc.get(&key, &ret); + REQUIRE(ret.get() != nullptr); + out.assign(ret->data(), ret->data() + len); + return hit; +} + +} // namespace + +TEST_CASE("CLFUS compressible objects roundtrip cleanly", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + auto payload = compressible_bytes(8192); + const CompressionCase c = GENERATE(from_range(compression_cases())); + INFO("compression backend: " << c.name); + + std::vector out; + int hit = store_compress_get(stripe, c.config, payload, out); + + CHECK(hit == c.expected_hit); + CHECK(out == payload); +} + +TEST_CASE("CLFUS incompressible objects fall back to uncompressed storage", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + auto payload = incompressible_bytes(8192); + + // Only the backends that actually attempt compression are interesting here; + // skip the NONE case. + auto cases = compression_cases(); + cases.erase(cases.begin()); + const CompressionCase c = GENERATE_REF(from_range(cases)); + INFO("compression backend: " << c.name); + + std::vector out; + int hit = store_compress_get(stripe, c.config, payload, out); + + // Incompressible data is kept verbatim, so a read reports no compression. + CHECK(hit == RAM_HIT_COMPRESS_NONE); + CHECK(out == payload); +} + +TEST_CASE("CLFUS single-byte payload roundtrips", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + std::vector out; + int hit = store_compress_get(stripe, CACHE_COMPRESSION_NONE, compressible_bytes(1), out); + CHECK(hit == RAM_HIT_COMPRESS_NONE); + CHECK(out == compressible_bytes(1)); +} From 788f1ff6bf8f02830253ae224116bcc4f4927b94 Mon Sep 17 00:00:00 2001 From: "Phong X. Nguyen" Date: Wed, 10 Jun 2026 23:17:18 +0000 Subject: [PATCH 4/8] Add zstd support --- CMakeLists.txt | 6 ++- cmake/FindZSTD.cmake | 43 +++++++++++++++++++ doc/admin-guide/files/records.yaml.en.rst | 3 +- doc/admin-guide/storage/index.en.rst | 1 + .../cache-architecture/ram-cache.en.rst | 13 +++--- include/iocore/cache/Cache.h | 2 + src/iocore/cache/CMakeLists.txt | 4 ++ src/iocore/cache/CacheProcessor.cc | 5 +++ src/iocore/cache/RamCacheCLFUS.cc | 37 ++++++++++++++++ .../cache/unit_tests/test_RamCacheCLFUS.cc | 3 ++ 10 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 cmake/FindZSTD.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 2bdbc12dfe9..9778e60d755 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,8 +366,8 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) -find_package(zstd CONFIG QUIET) -if(zstd_FOUND) +find_package(ZSTD) +if(ZSTD_FOUND) # Provide a compatibility target name if the upstream package does not export it # Our code links against `zstd::zstd`; upstream zstd usually exports @@ -388,6 +388,8 @@ if(zstd_FOUND) else() set(HAVE_ZSTD_H FALSE) endif() + else() + set(HAVE_ZSTD_H TRUE) endif() else() set(HAVE_ZSTD_H FALSE) diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake new file mode 100644 index 00000000000..737be3b584b --- /dev/null +++ b/cmake/FindZSTD.cmake @@ -0,0 +1,43 @@ +find_path( + ZSTD_INCLUDE_DIR + NAMES zstd.h + DOC "zstd include directory" +) +mark_as_advanced(ZSTD_INCLUDE_DIR) +find_library( + ZSTD_LIBRARY + NAMES zstd libzstd + DOC "zstd library" +) +mark_as_advanced(ZSTD_LIBRARY) + +if(ZSTD_INCLUDE_DIR) + file(STRINGS "${ZSTD_INCLUDE_DIR}/zstd.h" _zstd_version_lines REGEX "#define[ \t]+ZSTD_VERSION_(MAJOR|MINOR|RELEASE)") + string(REGEX REPLACE ".*ZSTD_VERSION_MAJOR *\([0-9]*\).*" "\\1" _zstd_version_major "${_zstd_version_lines}") + string(REGEX REPLACE ".*ZSTD_VERSION_MINOR *\([0-9]*\).*" "\\1" _zstd_version_minor "${_zstd_version_lines}") + string(REGEX REPLACE ".*ZSTD_VERSION_RELEASE *\([0-9]*\).*" "\\1" _zstd_version_release "${_zstd_version_lines}") + set(ZSTD_VERSION "${_zstd_version_major}.${_zstd_version_minor}.${_zstd_version_release}") + unset(_zstd_version_major) + unset(_zstd_version_minor) + unset(_zstd_version_release) + unset(_zstd_version_lines) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + ZSTD + REQUIRED_VARS ZSTD_LIBRARY ZSTD_INCLUDE_DIR + VERSION_VAR ZSTD_VERSION +) + +if(ZSTD_FOUND) + set(ZSTD_INCLUDE_DIRS "${ZSTD_INCLUDE_DIR}") + set(ZSTD_LIBRARIES "${ZSTD_LIBRARY}") + + if(NOT TARGET zstd::zstd) + add_library(zstd::zstd UNKNOWN IMPORTED) + set_target_properties( + zstd::zstd PROPERTIES IMPORTED_LOCATION "${ZSTD_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}" + ) + endif() +endif() diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index c7ed80b0d9e..6433d62aa56 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2928,9 +2928,10 @@ RAM Cache ======== =================================================================== ``0`` No compression ``1`` Fastlz (extremely fast, relatively low compression) - prefer lz4 - ``2`` Libz (moderate speed, reasonable compression) + ``2`` Libz (moderate speed, reasonable compression) - prefer zstd ``3`` Liblzma (very slow, high compression) ``4`` lz4 (extremely fast, relatively low compression) + ``5`` zstd (fast speed, reasonable compression) ======== =================================================================== Compression runs on task threads. To use more cores for RAM cache diff --git a/doc/admin-guide/storage/index.en.rst b/doc/admin-guide/storage/index.en.rst index fa7d94e6ed8..dce15c88e43 100644 --- a/doc/admin-guide/storage/index.en.rst +++ b/doc/admin-guide/storage/index.en.rst @@ -106,6 +106,7 @@ Value Meaning 2 *libz* compression 3 *liblzma* compression 4 *lz4* compression +5 *zstd* compression ======= ============================= .. _changing-the-size-of-the-ram-cache: diff --git a/doc/developer-guide/cache-architecture/ram-cache.en.rst b/doc/developer-guide/cache-architecture/ram-cache.en.rst index 690888df5b3..f557899793c 100644 --- a/doc/developer-guide/cache-architecture/ram-cache.en.rst +++ b/doc/developer-guide/cache-architecture/ram-cache.en.rst @@ -37,7 +37,7 @@ following features: * Is Scan Resistant and extracts robust hit rates even when the working set does not fit in the RAM Cache. -* Supports compression at 4 levels: fastlz, gzip (libz), xz (liblzma) and lz4. +* Supports compression at 5 levels: fastlz, gzip (libz), xz (liblzma), lz4 and zstd. Compression can be moved to another thread. * Has very low CPU overhead, only slightly more than a basic LRU. Rather than @@ -72,7 +72,7 @@ len Length of the object, which differs from *size* because of compression and padding). compressed_len Compressed length of the object. compressed Compression type, or ``none`` if no compression. Possible types - are: *fastlz*, *libz*, *liblzma*, and *lz4*. + are: *fastlz*, *libz*, *liblzma*, *lz4* and *zstd*. uncompressible Flag indicating that content cannot be compressed (true), or that it mat be compressed (false). copy Whether or not this object should be copied in and copied out @@ -147,8 +147,8 @@ since we need to make a copy anyway. Those not tagged ``copy`` are inserted uncompressed in the hope that they can be reused in uncompressed form. This is a compile time option and may be something we want to change. -There are 4 algorithms and levels of compression (speed on an Intel Xeon Gold -6338 processor using lzbench and the silesia XML benchmark): +There are 5 algorithms and levels of compression (speed on an Intel Xeon Gold +6338 processor using lzbench and the silesia XML corpus): ======= ================ ================== ==================================== Method Compression Rate Decompression Rate Notes @@ -156,11 +156,14 @@ Method Compression Rate Decompression Rate Notes fastlz 452 MB/sec 913 MB/sec Effectively obsolete; prefer lz4. Basically free since disk or network will limit first; ~26% final size. -libz 54 MB/sec 536 MB/sec Almost free, particularly +libz 54 MB/sec 536 MB/sec Effectively obsolete; prefer zstd. + Almost free, particularly decompression; ~13% final size. liblzma 5 MB/sec 291 MB/sec Expensive; ~8% final size. lz4 727 MB/sec 3458 MB/sec Basically free since disk or network will limit first; 23% final size +zstd 508 MB/sec 1690 MB/sec Basically free since disk or network + will limit first; ~12% final size. ======= ================ ================== ==================================== These are ballpark numbers, and your millage will vary enormously. JPEG, for diff --git a/include/iocore/cache/Cache.h b/include/iocore/cache/Cache.h index 492628fbfa9..2b5a473f93b 100644 --- a/include/iocore/cache/Cache.h +++ b/include/iocore/cache/Cache.h @@ -44,6 +44,7 @@ static constexpr ts::ModuleVersion CACHE_MODULE_VERSION(1, 0); #define CACHE_COMPRESSION_LIBZ 2 #define CACHE_COMPRESSION_LIBLZMA 3 #define CACHE_COMPRESSION_LZ4 4 +#define CACHE_COMPRESSION_ZSTD 5 enum { RAM_HIT_COMPRESS_NONE = 1, @@ -51,6 +52,7 @@ enum { RAM_HIT_COMPRESS_LIBZ, RAM_HIT_COMPRESS_LIBLZMA, RAM_HIT_COMPRESS_LZ4, + RAM_HIT_COMPRESS_ZSTD, RAM_HIT_LAST_ENTRY }; diff --git a/src/iocore/cache/CMakeLists.txt b/src/iocore/cache/CMakeLists.txt index 73f2b10f00c..a48cdc85335 100644 --- a/src/iocore/cache/CMakeLists.txt +++ b/src/iocore/cache/CMakeLists.txt @@ -60,6 +60,10 @@ if(HAVE_LZ4_H) target_link_libraries(inkcache PRIVATE LZ4::LZ4) endif() +if(HAVE_ZSTD_H) + target_link_libraries(inkcache PRIVATE zstd::zstd) +endif() + if(BUILD_TESTING) # Unit Tests with unit_tests/main.cc macro(add_cache_test name) diff --git a/src/iocore/cache/CacheProcessor.cc b/src/iocore/cache/CacheProcessor.cc index f145d89aa87..8320bec945b 100644 --- a/src/iocore/cache/CacheProcessor.cc +++ b/src/iocore/cache/CacheProcessor.cc @@ -1653,6 +1653,11 @@ CacheProcessor::cacheInitialized() case CACHE_COMPRESSION_LZ4: #ifndef HAVE_LZ4_H Fatal("lz4 not available for RAM cache compression"); +#endif + break; + case CACHE_COMPRESSION_ZSTD: +#ifndef HAVE_ZSTD_H + Fatal("zstd not available for RAM cache compression"); #endif break; } diff --git a/src/iocore/cache/RamCacheCLFUS.cc b/src/iocore/cache/RamCacheCLFUS.cc index 1587adef8b2..aa9168d924c 100644 --- a/src/iocore/cache/RamCacheCLFUS.cc +++ b/src/iocore/cache/RamCacheCLFUS.cc @@ -39,6 +39,9 @@ #ifdef HAVE_LZ4_H #include #endif +#ifdef HAVE_ZSTD_H +#include +#endif #define REQUIRED_COMPRESSION 0.9 // must get to this size or declared incompressible #define REQUIRED_SHRINK 0.8 // must get to this size or keep original buffer (with padding) @@ -112,6 +115,11 @@ RamCacheCLFUSCompressor::mainEvent(int /* event ATS_UNUSED */, Event *e) case CACHE_COMPRESSION_LZ4: #ifndef HAVE_LZ4_H Warning("lz4 not available for RAM cache compression"); +#endif + break; + case CACHE_COMPRESSION_ZSTD: +#ifndef HAVE_ZSTD_H + Warning("zstd not available for RAM cache compression"); #endif break; } @@ -256,6 +264,18 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey ram_hit_state = RAM_HIT_COMPRESS_LZ4; break; } +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: { + size_t l = static_cast(e->len); + size_t ll = 0; + ll = ZSTD_decompress(b, l, e->data->data(), e->compressed_len); + if (ZSTD_isError(ll) || l != ll) { + goto Lfailed; + } + ram_hit_state = RAM_HIT_COMPRESS_ZSTD; + break; + } #endif } IOBufferData *data = new_xmalloc_IOBufferData(b, e->len); @@ -428,6 +448,11 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) case CACHE_COMPRESSION_LZ4: l = static_cast(LZ4_compressBound(e->len)); break; +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: + l = static_cast(ZSTD_compressBound(e->len)); + break; #endif } // store transient data for lock release @@ -476,6 +501,18 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) } break; } +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: { + size_t ll = l; + size_t zret = ZSTD_compress(b, ll, edata->data(), elen, ZSTD_CLEVEL_DEFAULT); + if (ZSTD_isError(zret)) { + failed = true; + } else { + l = static_cast(zret); + } + break; + } #endif } MUTEX_TAKE_LOCK(stripe->mutex, thread); diff --git a/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc index 2e6f5dfb4b2..c4c51eadc04 100644 --- a/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc +++ b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc @@ -62,6 +62,9 @@ compression_cases() #endif #ifdef HAVE_LZ4_H cases.push_back({CACHE_COMPRESSION_LZ4, RAM_HIT_COMPRESS_LZ4, "lz4"}); +#endif +#ifdef HAVE_ZSTD_H + cases.push_back({CACHE_COMPRESSION_ZSTD, RAM_HIT_COMPRESS_ZSTD, "zstd"}); #endif return cases; } From a4803b55f890153de3685d7616d791996a1b6dba Mon Sep 17 00:00:00 2001 From: "Phong X. Nguyen" Date: Thu, 11 Jun 2026 00:46:53 +0000 Subject: [PATCH 5/8] Claude PR review remediation --- CMakeLists.txt | 3 +++ ci/docker/yum/Dockerfile | 2 +- src/iocore/cache/RamCacheCLFUS.cc | 7 ++++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9778e60d755..24ea61629af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -372,6 +372,9 @@ if(ZSTD_FOUND) # Provide a compatibility target name if the upstream package does not export it # Our code links against `zstd::zstd`; upstream zstd usually exports # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed. + # Normally this will be dead code if we use the packaged FindZSTD.cmake; but + # if CMAKE_FIND_PACKAGE_PREFER_CONFIG=1 and zstd-config.cmake is found, this + # may be useful. if(NOT TARGET zstd::zstd) if(TARGET zstd::libzstd_shared) set(_zstd_target zstd::libzstd_shared) diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 4f53cadea60..c97237b21ab 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -52,7 +52,7 @@ RUN yum -y update; \ # Devel packages that ATS needs yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \ - ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \ + ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel lz4-devel \ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 httpd-tools procps-ng nmap-ncat \ diff --git a/src/iocore/cache/RamCacheCLFUS.cc b/src/iocore/cache/RamCacheCLFUS.cc index aa9168d924c..5b3130cc2f5 100644 --- a/src/iocore/cache/RamCacheCLFUS.cc +++ b/src/iocore/cache/RamCacheCLFUS.cc @@ -41,6 +41,7 @@ #endif #ifdef HAVE_ZSTD_H #include +constexpr int CLFUS_ZSTD_LEVEL = 3; #endif #define REQUIRED_COMPRESSION 0.9 // must get to this size or declared incompressible @@ -103,9 +104,7 @@ RamCacheCLFUSCompressor::mainEvent(int /* event ATS_UNUSED */, Event *e) Warning("unknown RAM cache compression type: %d", cache_config_ram_cache_compress); case CACHE_COMPRESSION_NONE: case CACHE_COMPRESSION_FASTLZ: - break; case CACHE_COMPRESSION_LIBZ: - Warning("libz not available for RAM cache compression"); break; case CACHE_COMPRESSION_LIBLZMA: #ifndef HAVE_LZMA_H @@ -269,6 +268,7 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey case CACHE_COMPRESSION_ZSTD: { size_t l = static_cast(e->len); size_t ll = 0; + // TODO: Use a thread_local ZSTD_DCtx ll = ZSTD_decompress(b, l, e->data->data(), e->compressed_len); if (ZSTD_isError(ll) || l != ll) { goto Lfailed; @@ -505,7 +505,8 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) #ifdef HAVE_ZSTD_H case CACHE_COMPRESSION_ZSTD: { size_t ll = l; - size_t zret = ZSTD_compress(b, ll, edata->data(), elen, ZSTD_CLEVEL_DEFAULT); + // TODO: Use a thread_local ZSTD_CCtx + size_t zret = ZSTD_compress(b, ll, edata->data(), elen, CLFUS_ZSTD_LEVEL); if (ZSTD_isError(zret)) { failed = true; } else { From 1689450fd1962ad08d73118ab0b27ac9727f9c5d Mon Sep 17 00:00:00 2001 From: "Phong X. Nguyen" Date: Thu, 11 Jun 2026 01:49:15 +0000 Subject: [PATCH 6/8] Reuse zstd context --- CMakeLists.txt | 4 ++- src/iocore/cache/RamCacheCLFUS.cc | 51 ++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 24ea61629af..1c81017d578 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,7 +366,9 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) -find_package(ZSTD) +# 1.4.0 stabilized the advanced one-shot API (ZSTD_compress2 et al.) used by +# the RAM cache. +find_package(ZSTD 1.4.0) if(ZSTD_FOUND) # Provide a compatibility target name if the upstream package does not export it diff --git a/src/iocore/cache/RamCacheCLFUS.cc b/src/iocore/cache/RamCacheCLFUS.cc index 5b3130cc2f5..71faf0e3580 100644 --- a/src/iocore/cache/RamCacheCLFUS.cc +++ b/src/iocore/cache/RamCacheCLFUS.cc @@ -41,7 +41,39 @@ #endif #ifdef HAVE_ZSTD_H #include +#include constexpr int CLFUS_ZSTD_LEVEL = 3; + +namespace +{ + +// One-shot ZSTD_compress/ZSTD_decompress allocate and free a context on every +// call, so reuse a per-thread context instead. May return nullptr if zstd +// fails to allocate one. The compression level is a sticky parameter set once +// here; no explicit ZSTD_CCtx_reset() is needed because ZSTD_compress2() +// starts a new session on every call (resets are only for interrupting the +// streaming API or changing sticky parameters). +ZSTD_CCtx * +zstd_cctx() +{ + thread_local std::unique_ptr ctx = [] { + std::unique_ptr c{ZSTD_createCCtx(), ZSTD_freeCCtx}; + if (c && ZSTD_isError(ZSTD_CCtx_setParameter(c.get(), ZSTD_c_compressionLevel, CLFUS_ZSTD_LEVEL))) { + c.reset(); + } + return c; + }(); + return ctx.get(); +} + +ZSTD_DCtx * +zstd_dctx() +{ + thread_local std::unique_ptr ctx{ZSTD_createDCtx(), ZSTD_freeDCtx}; + return ctx.get(); +} + +} // end anonymous namespace #endif #define REQUIRED_COMPRESSION 0.9 // must get to this size or declared incompressible @@ -266,10 +298,12 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey #endif #ifdef HAVE_ZSTD_H case CACHE_COMPRESSION_ZSTD: { - size_t l = static_cast(e->len); - size_t ll = 0; - // TODO: Use a thread_local ZSTD_DCtx - ll = ZSTD_decompress(b, l, e->data->data(), e->compressed_len); + size_t l = static_cast(e->len); + ZSTD_DCtx *dctx = zstd_dctx(); + if (dctx == nullptr) { + goto Lfailed; + } + size_t ll = ZSTD_decompressDCtx(dctx, b, l, e->data->data(), e->compressed_len); if (ZSTD_isError(ll) || l != ll) { goto Lfailed; } @@ -504,9 +538,12 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) #endif #ifdef HAVE_ZSTD_H case CACHE_COMPRESSION_ZSTD: { - size_t ll = l; - // TODO: Use a thread_local ZSTD_CCtx - size_t zret = ZSTD_compress(b, ll, edata->data(), elen, CLFUS_ZSTD_LEVEL); + ZSTD_CCtx *cctx = zstd_cctx(); + if (cctx == nullptr) { + failed = true; + break; + } + size_t zret = ZSTD_compress2(cctx, b, l, edata->data(), elen); if (ZSTD_isError(zret)) { failed = true; } else { From aa880e146db2a119e57f3ed7c542dbf1f532327e Mon Sep 17 00:00:00 2001 From: "Phong X. Nguyen" Date: Thu, 11 Jun 2026 02:46:39 +0000 Subject: [PATCH 7/8] Add licenses to CMake files --- NOTICE | 7 +++++++ cmake/FindLZ4.cmake | 35 +++++++++++++++++++++++++++++++++++ cmake/FindZSTD.cmake | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/NOTICE b/NOTICE index 31787fecbf0..7669e1fb6c6 100644 --- a/NOTICE +++ b/NOTICE @@ -118,3 +118,10 @@ LS-HPACK provides functionality to encode and decode HTTP headers using HPACK compression mechanism specified in RFC 7541. Copyright (c) 2018 - 2023 LiteSpeed Technologies Inc, (MIT License) https://github.com/litespeedtech/ls-hpack.git + +~~ + +cmake/FindLZ4.cmake and cmake/FindZSTD.cmake derived from: +VTK: open-source software system for image processing, 3D graphics, volume rendering and visualization +Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen (BSD-3-Clause License) +https://gitlab.kitware.com/vtk/vtk diff --git a/cmake/FindLZ4.cmake b/cmake/FindLZ4.cmake index c36a6573c32..1c53b1bf121 100644 --- a/cmake/FindLZ4.cmake +++ b/cmake/FindLZ4.cmake @@ -1,3 +1,38 @@ +#========================================================================= +# +# Sourced from the Visualization Toolkit (VTK), CMake/FindLZ4.cmake: +# https://gitlab.kitware.com/vtk/vtk +# +# Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names +# of any contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#========================================================================= + find_path( LZ4_INCLUDE_DIR NAMES lz4.h diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake index 737be3b584b..9634a96a401 100644 --- a/cmake/FindZSTD.cmake +++ b/cmake/FindZSTD.cmake @@ -1,3 +1,38 @@ +#========================================================================= +# +# Derived from the Visualization Toolkit (VTK), CMake/FindLZ4.cmake: +# https://gitlab.kitware.com/vtk/vtk +# +# Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names +# of any contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#========================================================================= + find_path( ZSTD_INCLUDE_DIR NAMES zstd.h From 764b2e4f1e3b0e2b16eb90389ef57a73b6065e51 Mon Sep 17 00:00:00 2001 From: "Phong X. Nguyen" Date: Thu, 11 Jun 2026 21:22:06 +0000 Subject: [PATCH 8/8] PR remediations --- ci/rat-exclude.txt | 2 + .../statistics/core/cache-volume.en.rst | 5 ++ .../monitoring/statistics/core/cache.en.rst | 5 ++ .../cache-architecture/ram-cache.en.rst | 32 +++---- include/iocore/cache/Cache.h | 9 ++ src/iocore/cache/CacheProcessor.cc | 75 ++++++++-------- src/iocore/cache/P_CacheStats.h | 75 ++++++++-------- src/iocore/cache/RamCacheCLFUS.cc | 40 ++++++++- .../cache/unit_tests/test_RamCacheCLFUS.cc | 90 +++++++++++++------ 9 files changed, 213 insertions(+), 120 deletions(-) diff --git a/ci/rat-exclude.txt b/ci/rat-exclude.txt index 39ad1b11ca8..795da6f8f8e 100644 --- a/ci/rat-exclude.txt +++ b/ci/rat-exclude.txt @@ -79,3 +79,5 @@ tools/http_load/** **/clang-tidy.conf build*/** cmake-build*/** +cmake/FindLZ4.cmake +cmake/FindZSTD.cmake diff --git a/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst b/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst index c6e39eb7449..eb6b8b2c512 100644 --- a/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst @@ -132,6 +132,11 @@ a configuration with only one cache volume: :literal:`0`. Accumulates the number of misses to the LRU RAM cache for this volume. Note that this count includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk. +.. ts:stat:: global proxy.process.cache.volume_0.ram_cache.decompress.failure integer + :type: counter + + Accumulates the number of RAM cache entries that failed to decompress on read, for this volume. A failed entry is dropped from the RAM cache and the read is treated as a miss. A nonzero value indicates data corruption or a compression library error, not ordinary cache churn. + .. ts:stat:: global proxy.process.cache.volume_0.last_open_read.hits integer :type: counter diff --git a/doc/admin-guide/monitoring/statistics/core/cache.en.rst b/doc/admin-guide/monitoring/statistics/core/cache.en.rst index bb2f7d17f42..b06bb9802a5 100644 --- a/doc/admin-guide/monitoring/statistics/core/cache.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/cache.en.rst @@ -95,6 +95,11 @@ Cache Accumulates the number of misses to the LRU RAM cache for all volumes. Note that this includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk. +.. ts:stat:: global proxy.process.cache.ram_cache.decompress.failure integer + :type: counter + + Accumulates the number of RAM cache entries that failed to decompress on read, for all volumes. A failed entry is dropped from the RAM cache and the read is treated as a miss. A nonzero value indicates data corruption or a compression library error, not ordinary cache churn. + .. ts:stat:: global proxy.process.cache.last_open_read.hits integer :type: counter diff --git a/doc/developer-guide/cache-architecture/ram-cache.en.rst b/doc/developer-guide/cache-architecture/ram-cache.en.rst index f557899793c..2fa9c2caa46 100644 --- a/doc/developer-guide/cache-architecture/ram-cache.en.rst +++ b/doc/developer-guide/cache-architecture/ram-cache.en.rst @@ -73,7 +73,7 @@ len Length of the object, which differs from *size* because of compressed_len Compressed length of the object. compressed Compression type, or ``none`` if no compression. Possible types are: *fastlz*, *libz*, *liblzma*, *lz4* and *zstd*. -uncompressible Flag indicating that content cannot be compressed (true), or that +incompressible Flag indicating that content cannot be compressed (true), or that it mat be compressed (false). copy Whether or not this object should be copied in and copied out (e.g. HTTP HDR). @@ -150,21 +150,21 @@ a compile time option and may be something we want to change. There are 5 algorithms and levels of compression (speed on an Intel Xeon Gold 6338 processor using lzbench and the silesia XML corpus): -======= ================ ================== ==================================== -Method Compression Rate Decompression Rate Notes -======= ================ ================== ==================================== -fastlz 452 MB/sec 913 MB/sec Effectively obsolete; prefer lz4. - Basically free since disk or network - will limit first; ~26% final size. -libz 54 MB/sec 536 MB/sec Effectively obsolete; prefer zstd. - Almost free, particularly - decompression; ~13% final size. -liblzma 5 MB/sec 291 MB/sec Expensive; ~8% final size. -lz4 727 MB/sec 3458 MB/sec Basically free since disk or network - will limit first; 23% final size -zstd 508 MB/sec 1690 MB/sec Basically free since disk or network - will limit first; ~12% final size. -======= ================ ================== ==================================== +======= ===== ================= ================== ==================================== +Method Level Compression Rate Decompression Rate Notes +======= ===== ================= ================== ==================================== +fastlz 1 452 MB/sec 913 MB/sec Effectively obsolete; prefer lz4. + Basically free since disk or network + will limit first; ~26% final size. +libz 6 54 MB/sec 536 MB/sec Effectively obsolete; prefer zstd. + Almost free, particularly + decompression; ~13% final size. +liblzma 6 5 MB/sec 291 MB/sec Expensive; ~8% final size. +lz4 1 727 MB/sec 3458 MB/sec Basically free since disk or network + will limit first; ~23% final size. +zstd 3 508 MB/sec 1690 MB/sec Basically free since disk or network + will limit first; ~12% final size. +======= ===== ================= ================== ==================================== These are ballpark numbers, and your millage will vary enormously. JPEG, for example, will not compress with any of these (or at least will only do so at diff --git a/include/iocore/cache/Cache.h b/include/iocore/cache/Cache.h index 2b5a473f93b..72568b5f636 100644 --- a/include/iocore/cache/Cache.h +++ b/include/iocore/cache/Cache.h @@ -56,6 +56,15 @@ enum { RAM_HIT_LAST_ENTRY }; +// The RAM_HIT_COMPRESS_* values are the CACHE_COMPRESSION_* values offset by +// one; keep the two sequences from silently desyncing when a codec is added. +static_assert(RAM_HIT_COMPRESS_NONE == CACHE_COMPRESSION_NONE + 1); +static_assert(RAM_HIT_COMPRESS_FASTLZ == CACHE_COMPRESSION_FASTLZ + 1); +static_assert(RAM_HIT_COMPRESS_LIBZ == CACHE_COMPRESSION_LIBZ + 1); +static_assert(RAM_HIT_COMPRESS_LIBLZMA == CACHE_COMPRESSION_LIBLZMA + 1); +static_assert(RAM_HIT_COMPRESS_LZ4 == CACHE_COMPRESSION_LZ4 + 1); +static_assert(RAM_HIT_COMPRESS_ZSTD == CACHE_COMPRESSION_ZSTD + 1); + struct CacheVC; class CacheEvacuateDocVC; struct CacheDisk; diff --git a/src/iocore/cache/CacheProcessor.cc b/src/iocore/cache/CacheProcessor.cc index 8320bec945b..725724f3625 100644 --- a/src/iocore/cache/CacheProcessor.cc +++ b/src/iocore/cache/CacheProcessor.cc @@ -1170,43 +1170,44 @@ register_cache_stats(CacheStatsBlock *rsb, const std::string &prefix) rsb->fragment_document_count[2] = ts::Metrics::Counter::createPtr(prefix + ".frags_per_doc.3+"); // And then everything else - rsb->bytes_used = ts::Metrics::Gauge::createPtr(prefix + ".bytes_used"); - rsb->bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".bytes_total"); - rsb->stripes = ts::Metrics::Gauge::createPtr(prefix + ".stripes"); - rsb->ram_cache_bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.total_bytes"); - rsb->ram_cache_bytes = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.bytes_used"); - rsb->ram_cache_hits = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.hits"); - rsb->last_open_read_hits = ts::Metrics::Counter::createPtr(prefix + ".last_open_read.hits"); - rsb->agg_buffer_hits = ts::Metrics::Counter::createPtr(prefix + ".aggregation_buffer.hits"); - rsb->ram_cache_misses = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.misses"); - rsb->all_mem_misses = ts::Metrics::Counter::createPtr(prefix + ".all_memory_caches.misses"); - rsb->pread_count = ts::Metrics::Counter::createPtr(prefix + ".pread_count"); - rsb->percent_full = ts::Metrics::Gauge::createPtr(prefix + ".percent_full"); - rsb->read_seek_fail = ts::Metrics::Counter::createPtr(prefix + ".read.seek.failure"); - rsb->read_invalid = ts::Metrics::Counter::createPtr(prefix + ".read.invalid"); - rsb->write_backlog_failure = ts::Metrics::Counter::createPtr(prefix + ".write.backlog.failure"); - rsb->direntries_total = ts::Metrics::Gauge::createPtr(prefix + ".direntries.total"); - rsb->direntries_used = ts::Metrics::Gauge::createPtr(prefix + ".direntries.used"); - rsb->directory_collision = ts::Metrics::Counter::createPtr(prefix + ".directory_collision"); - rsb->read_busy_success = ts::Metrics::Counter::createPtr(prefix + ".read_busy.success"); - rsb->read_busy_failure = ts::Metrics::Counter::createPtr(prefix + ".read_busy.failure"); - rsb->write_bytes = ts::Metrics::Counter::createPtr(prefix + ".write_bytes_stat"); - rsb->hdr_vector_marshal = ts::Metrics::Counter::createPtr(prefix + ".vector_marshals"); - rsb->hdr_marshal = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshals"); - rsb->hdr_marshal_bytes = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshal_bytes"); - rsb->gc_bytes_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_bytes_evacuated"); - rsb->gc_frags_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_frags_evacuated"); - rsb->directory_wrap = ts::Metrics::Counter::createPtr(prefix + ".wrap_count"); - rsb->directory_sync_count = ts::Metrics::Counter::createPtr(prefix + ".sync.count"); - rsb->directory_sync_bytes = ts::Metrics::Counter::createPtr(prefix + ".sync.bytes"); - rsb->directory_sync_time = ts::Metrics::Counter::createPtr(prefix + ".sync.time"); - rsb->span_errors_read = ts::Metrics::Counter::createPtr(prefix + ".span.errors.read"); - rsb->span_errors_write = ts::Metrics::Counter::createPtr(prefix + ".span.errors.write"); - rsb->span_failing = ts::Metrics::Gauge::createPtr(prefix + ".span.failing"); - rsb->span_offline = ts::Metrics::Gauge::createPtr(prefix + ".span.offline"); - rsb->span_online = ts::Metrics::Gauge::createPtr(prefix + ".span.online"); - rsb->stripe_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".stripe.lock_contention"); - rsb->writer_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".writer.lock_contention"); + rsb->bytes_used = ts::Metrics::Gauge::createPtr(prefix + ".bytes_used"); + rsb->bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".bytes_total"); + rsb->stripes = ts::Metrics::Gauge::createPtr(prefix + ".stripes"); + rsb->ram_cache_bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.total_bytes"); + rsb->ram_cache_bytes = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.bytes_used"); + rsb->ram_cache_hits = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.hits"); + rsb->last_open_read_hits = ts::Metrics::Counter::createPtr(prefix + ".last_open_read.hits"); + rsb->agg_buffer_hits = ts::Metrics::Counter::createPtr(prefix + ".aggregation_buffer.hits"); + rsb->ram_cache_misses = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.misses"); + rsb->ram_cache_decompress_failures = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.decompress.failure"); + rsb->all_mem_misses = ts::Metrics::Counter::createPtr(prefix + ".all_memory_caches.misses"); + rsb->pread_count = ts::Metrics::Counter::createPtr(prefix + ".pread_count"); + rsb->percent_full = ts::Metrics::Gauge::createPtr(prefix + ".percent_full"); + rsb->read_seek_fail = ts::Metrics::Counter::createPtr(prefix + ".read.seek.failure"); + rsb->read_invalid = ts::Metrics::Counter::createPtr(prefix + ".read.invalid"); + rsb->write_backlog_failure = ts::Metrics::Counter::createPtr(prefix + ".write.backlog.failure"); + rsb->direntries_total = ts::Metrics::Gauge::createPtr(prefix + ".direntries.total"); + rsb->direntries_used = ts::Metrics::Gauge::createPtr(prefix + ".direntries.used"); + rsb->directory_collision = ts::Metrics::Counter::createPtr(prefix + ".directory_collision"); + rsb->read_busy_success = ts::Metrics::Counter::createPtr(prefix + ".read_busy.success"); + rsb->read_busy_failure = ts::Metrics::Counter::createPtr(prefix + ".read_busy.failure"); + rsb->write_bytes = ts::Metrics::Counter::createPtr(prefix + ".write_bytes_stat"); + rsb->hdr_vector_marshal = ts::Metrics::Counter::createPtr(prefix + ".vector_marshals"); + rsb->hdr_marshal = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshals"); + rsb->hdr_marshal_bytes = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshal_bytes"); + rsb->gc_bytes_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_bytes_evacuated"); + rsb->gc_frags_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_frags_evacuated"); + rsb->directory_wrap = ts::Metrics::Counter::createPtr(prefix + ".wrap_count"); + rsb->directory_sync_count = ts::Metrics::Counter::createPtr(prefix + ".sync.count"); + rsb->directory_sync_bytes = ts::Metrics::Counter::createPtr(prefix + ".sync.bytes"); + rsb->directory_sync_time = ts::Metrics::Counter::createPtr(prefix + ".sync.time"); + rsb->span_errors_read = ts::Metrics::Counter::createPtr(prefix + ".span.errors.read"); + rsb->span_errors_write = ts::Metrics::Counter::createPtr(prefix + ".span.errors.write"); + rsb->span_failing = ts::Metrics::Gauge::createPtr(prefix + ".span.failing"); + rsb->span_offline = ts::Metrics::Gauge::createPtr(prefix + ".span.offline"); + rsb->span_online = ts::Metrics::Gauge::createPtr(prefix + ".span.online"); + rsb->stripe_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".stripe.lock_contention"); + rsb->writer_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".writer.lock_contention"); } void diff --git a/src/iocore/cache/P_CacheStats.h b/src/iocore/cache/P_CacheStats.h index 7673cf56e0e..08120605b47 100644 --- a/src/iocore/cache/P_CacheStats.h +++ b/src/iocore/cache/P_CacheStats.h @@ -37,41 +37,42 @@ struct CacheStatsBlock { ts::Metrics::Counter::AtomicType *fragment_document_count[3] = {nullptr, nullptr, nullptr}; // For 1, 2 and 3+ fragments - ts::Metrics::Gauge::AtomicType *bytes_used = nullptr; - ts::Metrics::Gauge::AtomicType *bytes_total = nullptr; - ts::Metrics::Gauge::AtomicType *stripes = nullptr; - ts::Metrics::Gauge::AtomicType *ram_cache_bytes = nullptr; - ts::Metrics::Gauge::AtomicType *ram_cache_bytes_total = nullptr; - ts::Metrics::Gauge::AtomicType *direntries_total = nullptr; - ts::Metrics::Gauge::AtomicType *direntries_used = nullptr; - ts::Metrics::Counter::AtomicType *ram_cache_hits = nullptr; - ts::Metrics::Counter::AtomicType *last_open_read_hits = nullptr; - ts::Metrics::Counter::AtomicType *agg_buffer_hits = nullptr; - ts::Metrics::Counter::AtomicType *ram_cache_misses = nullptr; - ts::Metrics::Counter::AtomicType *all_mem_misses = nullptr; - ts::Metrics::Counter::AtomicType *pread_count = nullptr; - ts::Metrics::Gauge::AtomicType *percent_full = nullptr; - ts::Metrics::Counter::AtomicType *read_seek_fail = nullptr; - ts::Metrics::Counter::AtomicType *read_invalid = nullptr; - ts::Metrics::Counter::AtomicType *write_backlog_failure = nullptr; - ts::Metrics::Counter::AtomicType *directory_collision = nullptr; - ts::Metrics::Counter::AtomicType *read_busy_success = nullptr; - ts::Metrics::Counter::AtomicType *read_busy_failure = nullptr; - ts::Metrics::Counter::AtomicType *gc_bytes_evacuated = nullptr; - ts::Metrics::Counter::AtomicType *gc_frags_evacuated = nullptr; - ts::Metrics::Counter::AtomicType *write_bytes = nullptr; - ts::Metrics::Counter::AtomicType *hdr_vector_marshal = nullptr; - ts::Metrics::Counter::AtomicType *hdr_marshal = nullptr; - ts::Metrics::Counter::AtomicType *hdr_marshal_bytes = nullptr; - ts::Metrics::Counter::AtomicType *directory_wrap = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_count = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_time = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_bytes = nullptr; - ts::Metrics::Counter::AtomicType *span_errors_read = nullptr; - ts::Metrics::Counter::AtomicType *span_errors_write = nullptr; - ts::Metrics::Gauge::AtomicType *span_offline = nullptr; - ts::Metrics::Gauge::AtomicType *span_online = nullptr; - ts::Metrics::Gauge::AtomicType *span_failing = nullptr; - ts::Metrics::Counter::AtomicType *stripe_lock_contention = nullptr; - ts::Metrics::Counter::AtomicType *writer_lock_contention = nullptr; + ts::Metrics::Gauge::AtomicType *bytes_used = nullptr; + ts::Metrics::Gauge::AtomicType *bytes_total = nullptr; + ts::Metrics::Gauge::AtomicType *stripes = nullptr; + ts::Metrics::Gauge::AtomicType *ram_cache_bytes = nullptr; + ts::Metrics::Gauge::AtomicType *ram_cache_bytes_total = nullptr; + ts::Metrics::Gauge::AtomicType *direntries_total = nullptr; + ts::Metrics::Gauge::AtomicType *direntries_used = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_hits = nullptr; + ts::Metrics::Counter::AtomicType *last_open_read_hits = nullptr; + ts::Metrics::Counter::AtomicType *agg_buffer_hits = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_misses = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_decompress_failures = nullptr; + ts::Metrics::Counter::AtomicType *all_mem_misses = nullptr; + ts::Metrics::Counter::AtomicType *pread_count = nullptr; + ts::Metrics::Gauge::AtomicType *percent_full = nullptr; + ts::Metrics::Counter::AtomicType *read_seek_fail = nullptr; + ts::Metrics::Counter::AtomicType *read_invalid = nullptr; + ts::Metrics::Counter::AtomicType *write_backlog_failure = nullptr; + ts::Metrics::Counter::AtomicType *directory_collision = nullptr; + ts::Metrics::Counter::AtomicType *read_busy_success = nullptr; + ts::Metrics::Counter::AtomicType *read_busy_failure = nullptr; + ts::Metrics::Counter::AtomicType *gc_bytes_evacuated = nullptr; + ts::Metrics::Counter::AtomicType *gc_frags_evacuated = nullptr; + ts::Metrics::Counter::AtomicType *write_bytes = nullptr; + ts::Metrics::Counter::AtomicType *hdr_vector_marshal = nullptr; + ts::Metrics::Counter::AtomicType *hdr_marshal = nullptr; + ts::Metrics::Counter::AtomicType *hdr_marshal_bytes = nullptr; + ts::Metrics::Counter::AtomicType *directory_wrap = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_count = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_time = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_bytes = nullptr; + ts::Metrics::Counter::AtomicType *span_errors_read = nullptr; + ts::Metrics::Counter::AtomicType *span_errors_write = nullptr; + ts::Metrics::Gauge::AtomicType *span_offline = nullptr; + ts::Metrics::Gauge::AtomicType *span_online = nullptr; + ts::Metrics::Gauge::AtomicType *span_failing = nullptr; + ts::Metrics::Counter::AtomicType *stripe_lock_contention = nullptr; + ts::Metrics::Counter::AtomicType *writer_lock_contention = nullptr; }; diff --git a/src/iocore/cache/RamCacheCLFUS.cc b/src/iocore/cache/RamCacheCLFUS.cc index 71faf0e3580..daf8e00859c 100644 --- a/src/iocore/cache/RamCacheCLFUS.cc +++ b/src/iocore/cache/RamCacheCLFUS.cc @@ -32,6 +32,9 @@ #include "iocore/eventsystem/Tasks.h" #include "fastlz/fastlz.h" #include "tscore/CryptoHash.h" +#include "tscore/Throttler.h" + +#include #include #ifdef HAVE_LZMA_H #include @@ -49,7 +52,8 @@ namespace // One-shot ZSTD_compress/ZSTD_decompress allocate and free a context on every // call, so reuse a per-thread context instead. May return nullptr if zstd -// fails to allocate one. The compression level is a sticky parameter set once +// fails to allocate one; that failure is sticky for the life of the thread, so +// warn when it happens. The compression level is a sticky parameter set once // here; no explicit ZSTD_CCtx_reset() is needed because ZSTD_compress2() // starts a new session on every call (resets are only for interrupting the // streaming API or changing sticky parameters). @@ -61,6 +65,9 @@ zstd_cctx() if (c && ZSTD_isError(ZSTD_CCtx_setParameter(c.get(), ZSTD_c_compressionLevel, CLFUS_ZSTD_LEVEL))) { c.reset(); } + if (!c) { + Warning("unable to allocate zstd compression context; RAM cache entries will not be compressed on this thread"); + } return c; }(); return ctx.get(); @@ -69,13 +76,23 @@ zstd_cctx() ZSTD_DCtx * zstd_dctx() { - thread_local std::unique_ptr ctx{ZSTD_createDCtx(), ZSTD_freeDCtx}; + thread_local std::unique_ptr ctx = [] { + std::unique_ptr c{ZSTD_createDCtx(), ZSTD_freeDCtx}; + if (!c) { + Warning("unable to allocate zstd decompression context; compressed RAM cache entries will miss on this thread"); + } + return c; + }(); return ctx.get(); } } // end anonymous namespace #endif +// The compression type is stored in the 3-bit RamCacheCLFUSEntry +// flag_bits.compressed field; a new codec value must still fit. +static_assert(CACHE_COMPRESSION_ZSTD < (1 << 3)); + #define REQUIRED_COMPRESSION 0.9 // must get to this size or declared incompressible #define REQUIRED_SHRINK 0.8 // must get to this size or keep original buffer (with padding) #define HISTORY_HYSTERIA 10 // extra temporary history @@ -301,7 +318,10 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey size_t l = static_cast(e->len); ZSTD_DCtx *dctx = zstd_dctx(); if (dctx == nullptr) { - goto Lfailed; + // This thread can't decompress, but the entry itself is fine: + // miss instead of evicting it. + ats_free(b); + goto Lerror; } size_t ll = ZSTD_decompressDCtx(dctx, b, l, e->data->data(), e->compressed_len); if (ZSTD_isError(ll) || l != ll) { @@ -355,6 +375,20 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey return 0; Lfailed: ats_free(b); + { + // A failure here is data corruption or a codec error, not an ordinary + // miss; make it visible beyond the debug-gated trace below. + static Throttler decompress_failure_throttler(std::chrono::seconds(60)); + + uint64_t suppressed = 0; + if (!decompress_failure_throttler.is_throttled(suppressed)) { + Warning("RAM cache decompression failed: type %d len %u compressed_len %u key %X; entry dropped" + " (%" PRIu64 " similar failures suppressed)", + static_cast(e->flag_bits.compressed), e->len, e->compressed_len, key->slice32(3), suppressed); + } + ts::Metrics::Counter::increment(cache_rsb.ram_cache_decompress_failures); + ts::Metrics::Counter::increment(stripe->cache_vol->vol_rsb.ram_cache_decompress_failures); + } this->_destroy(e); DDbg(dbg_ctl_ram_cache, "get %X %" PRId64 " Z_ERR", key->slice32(3), auxkey); goto Lerror; diff --git a/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc index c4c51eadc04..cb234e682df 100644 --- a/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc +++ b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc @@ -89,12 +89,15 @@ wire_stripe(StripeSM &stripe, CacheVol &cache_vol) { stripe.cache_vol = &cache_vol; - cache_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.ram_cache.bytes"); - cache_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.hits"); - cache_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.misses"); - cache_vol.vol_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.vol.ram_cache.bytes"); - cache_vol.vol_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.hits"); - cache_vol.vol_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.misses"); + cache_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.ram_cache.bytes"); + cache_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.hits"); + cache_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.misses"); + cache_rsb.ram_cache_decompress_failures = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.decompress.failure"); + cache_vol.vol_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.vol.ram_cache.bytes"); + cache_vol.vol_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.hits"); + cache_vol.vol_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.misses"); + cache_vol.vol_rsb.ram_cache_decompress_failures = + ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.decompress.failure"); } Ptr @@ -132,11 +135,17 @@ incompressible_bytes(std::size_t len) return bytes; } +struct RoundtripResult { + int hit = 0; + int64_t size_before = 0; // rc.size() after put, before the compression pass + int64_t size_after = 0; // rc.size() after the compression pass + std::vector out; +}; + // Store payload under a fresh key, force a synchronous compression pass with -// `config`, then read it back. Returns the RAM_HIT_* state reported by get() -// and the bytes that were returned. -int -store_compress_get(StripeSM &stripe, int config, const std::vector &payload, std::vector &out) +// `config`, then read it back. +RoundtripResult +store_compress_get(StripeSM &stripe, int config, const std::vector &payload) { // Initialize with compression disabled so init() does not schedule the // background compressor (which would retain a pointer to this stack object). @@ -158,14 +167,19 @@ store_compress_get(StripeSM &stripe, int config, const std::vector &payloa REQUIRE(rc.put(&key, in.get(), len) == 1); + RoundtripResult r; + + r.size_before = rc.size(); cache_config_ram_cache_compress = config; rc.compress_entries(this_ethread()); + r.size_after = rc.size(); Ptr ret; - int hit = rc.get(&key, &ret); + + r.hit = rc.get(&key, &ret); REQUIRE(ret.get() != nullptr); - out.assign(ret->data(), ret->data() + len); - return hit; + r.out.assign(ret->data(), ret->data() + len); + return r; } } // namespace @@ -178,15 +192,21 @@ TEST_CASE("CLFUS compressible objects roundtrip cleanly", "[cache][ramcache][com CacheVol cache_vol; wire_stripe(stripe, cache_vol); - auto payload = compressible_bytes(8192); + // Large enough to exercise the *_compressBound() arithmetic and uint32_t + // casts in compress_entries(), not just small-buffer paths. + auto payload = compressible_bytes(256 * 1024); const CompressionCase c = GENERATE(from_range(compression_cases())); INFO("compression backend: " << c.name); - std::vector out; - int hit = store_compress_get(stripe, c.config, payload, out); + RoundtripResult r = store_compress_get(stripe, c.config, payload); - CHECK(hit == c.expected_hit); - CHECK(out == payload); + CHECK(r.hit == c.expected_hit); + CHECK(r.out == payload); + if (c.config != CACHE_COMPRESSION_NONE) { + // The feature's contract is that compression saves memory, not merely + // that the entry is tagged compressed. + CHECK(r.size_after < r.size_before); + } } TEST_CASE("CLFUS incompressible objects fall back to uncompressed storage", "[cache][ramcache][compress]") @@ -197,7 +217,7 @@ TEST_CASE("CLFUS incompressible objects fall back to uncompressed storage", "[ca CacheVol cache_vol; wire_stripe(stripe, cache_vol); - auto payload = incompressible_bytes(8192); + auto payload = incompressible_bytes(256 * 1024); // Only the backends that actually attempt compression are interesting here; // skip the NONE case. @@ -206,12 +226,11 @@ TEST_CASE("CLFUS incompressible objects fall back to uncompressed storage", "[ca const CompressionCase c = GENERATE_REF(from_range(cases)); INFO("compression backend: " << c.name); - std::vector out; - int hit = store_compress_get(stripe, c.config, payload, out); + RoundtripResult r = store_compress_get(stripe, c.config, payload); // Incompressible data is kept verbatim, so a read reports no compression. - CHECK(hit == RAM_HIT_COMPRESS_NONE); - CHECK(out == payload); + CHECK(r.hit == RAM_HIT_COMPRESS_NONE); + CHECK(r.out == payload); } TEST_CASE("CLFUS single-byte payload roundtrips", "[cache][ramcache][compress]") @@ -222,8 +241,25 @@ TEST_CASE("CLFUS single-byte payload roundtrips", "[cache][ramcache][compress]") CacheVol cache_vol; wire_stripe(stripe, cache_vol); - std::vector out; - int hit = store_compress_get(stripe, CACHE_COMPRESSION_NONE, compressible_bytes(1), out); - CHECK(hit == RAM_HIT_COMPRESS_NONE); - CHECK(out == compressible_bytes(1)); + RoundtripResult r = store_compress_get(stripe, CACHE_COMPRESSION_NONE, compressible_bytes(1)); + + CHECK(r.hit == RAM_HIT_COMPRESS_NONE); + CHECK(r.out == compressible_bytes(1)); +} + +// A backend that is not compiled in silently disappears from the parametrized +// cases above; make that visible in the test output rather than shipping an +// untested backend behind a green run. +TEST_CASE("CLFUS compression backends compiled in", "[cache][ramcache][compress]") +{ +#ifndef HAVE_LZMA_H + WARN("liblzma is not compiled in; the liblzma RAM cache compression backend is NOT tested"); +#endif +#ifndef HAVE_LZ4_H + WARN("lz4 is not compiled in; the lz4 RAM cache compression backend is NOT tested"); +#endif +#ifndef HAVE_ZSTD_H + WARN("zstd is not compiled in; the zstd RAM cache compression backend is NOT tested"); +#endif + CHECK(compression_cases().size() >= 3); }