diff --git a/CMakeLists.txt b/CMakeLists.txt index fcd8b2eef13..1c81017d578 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,12 +366,17 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) -find_package(zstd CONFIG QUIET) -if(zstd_FOUND) +# 1.4.0 stabilized the advanced one-shot API (ZSTD_compress2 et al.) used by +# the RAM cache. +find_package(ZSTD 1.4.0) +if(ZSTD_FOUND) # Provide a compatibility target name if the upstream package does not export it # Our code links against `zstd::zstd`; upstream zstd usually exports # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed. + # Normally this will be dead code if we use the packaged FindZSTD.cmake; but + # if CMAKE_FIND_PACKAGE_PREFER_CONFIG=1 and zstd-config.cmake is found, this + # may be useful. if(NOT TARGET zstd::zstd) if(TARGET zstd::libzstd_shared) set(_zstd_target zstd::libzstd_shared) @@ -388,11 +393,18 @@ if(zstd_FOUND) else() set(HAVE_ZSTD_H FALSE) endif() + else() + set(HAVE_ZSTD_H TRUE) endif() else() set(HAVE_ZSTD_H FALSE) endif() +find_package(LZ4) +if(LZ4_FOUND) + set(HAVE_LZ4_H TRUE) +endif() + # ncurses is used in traffic_top find_package(Curses) set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H}) diff --git a/NOTICE b/NOTICE index 31787fecbf0..7669e1fb6c6 100644 --- a/NOTICE +++ b/NOTICE @@ -118,3 +118,10 @@ LS-HPACK provides functionality to encode and decode HTTP headers using HPACK compression mechanism specified in RFC 7541. Copyright (c) 2018 - 2023 LiteSpeed Technologies Inc, (MIT License) https://github.com/litespeedtech/ls-hpack.git + +~~ + +cmake/FindLZ4.cmake and cmake/FindZSTD.cmake derived from: +VTK: open-source software system for image processing, 3D graphics, volume rendering and visualization +Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen (BSD-3-Clause License) +https://gitlab.kitware.com/vtk/vtk diff --git a/ci/docker/deb/Dockerfile b/ci/docker/deb/Dockerfile index 85815163e2f..cd7d4bb89bf 100644 --- a/ci/docker/deb/Dockerfile +++ b/ci/docker/deb/Dockerfile @@ -56,7 +56,7 @@ RUN apt-get update; apt-get -y dist-upgrade; \ libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \ tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \ libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \ - libzstd-dev; \ + libzstd-dev liblz4-dev; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \ groupadd -g 665 jenkins && \ diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 4f53cadea60..c97237b21ab 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -52,7 +52,7 @@ RUN yum -y update; \ # Devel packages that ATS needs yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \ - ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \ + ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel lz4-devel \ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 httpd-tools procps-ng nmap-ncat \ diff --git a/ci/rat-exclude.txt b/ci/rat-exclude.txt index 39ad1b11ca8..795da6f8f8e 100644 --- a/ci/rat-exclude.txt +++ b/ci/rat-exclude.txt @@ -79,3 +79,5 @@ tools/http_load/** **/clang-tidy.conf build*/** cmake-build*/** +cmake/FindLZ4.cmake +cmake/FindZSTD.cmake diff --git a/cmake/FindLZ4.cmake b/cmake/FindLZ4.cmake new file mode 100644 index 00000000000..1c53b1bf121 --- /dev/null +++ b/cmake/FindLZ4.cmake @@ -0,0 +1,78 @@ +#========================================================================= +# +# Sourced from the Visualization Toolkit (VTK), CMake/FindLZ4.cmake: +# https://gitlab.kitware.com/vtk/vtk +# +# Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names +# of any contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#========================================================================= + +find_path( + LZ4_INCLUDE_DIR + NAMES lz4.h + DOC "lz4 include directory" +) +mark_as_advanced(LZ4_INCLUDE_DIR) +find_library( + LZ4_LIBRARY + NAMES lz4 liblz4 + DOC "lz4 library" +) +mark_as_advanced(LZ4_LIBRARY) + +if(LZ4_INCLUDE_DIR) + file(STRINGS "${LZ4_INCLUDE_DIR}/lz4.h" _lz4_version_lines REGEX "#define[ \t]+LZ4_VERSION_(MAJOR|MINOR|RELEASE)") + string(REGEX REPLACE ".*LZ4_VERSION_MAJOR *\([0-9]*\).*" "\\1" _lz4_version_major "${_lz4_version_lines}") + string(REGEX REPLACE ".*LZ4_VERSION_MINOR *\([0-9]*\).*" "\\1" _lz4_version_minor "${_lz4_version_lines}") + string(REGEX REPLACE ".*LZ4_VERSION_RELEASE *\([0-9]*\).*" "\\1" _lz4_version_release "${_lz4_version_lines}") + set(LZ4_VERSION "${_lz4_version_major}.${_lz4_version_minor}.${_lz4_version_release}") + unset(_lz4_version_major) + unset(_lz4_version_minor) + unset(_lz4_version_release) + unset(_lz4_version_lines) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + LZ4 + REQUIRED_VARS LZ4_LIBRARY LZ4_INCLUDE_DIR + VERSION_VAR LZ4_VERSION +) + +if(LZ4_FOUND) + set(LZ4_INCLUDE_DIRS "${LZ4_INCLUDE_DIR}") + set(LZ4_LIBRARIES "${LZ4_LIBRARY}") + + if(NOT TARGET LZ4::LZ4) + add_library(LZ4::LZ4 UNKNOWN IMPORTED) + set_target_properties( + LZ4::LZ4 PROPERTIES IMPORTED_LOCATION "${LZ4_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}" + ) + endif() +endif() diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake new file mode 100644 index 00000000000..9634a96a401 --- /dev/null +++ b/cmake/FindZSTD.cmake @@ -0,0 +1,78 @@ +#========================================================================= +# +# Derived from the Visualization Toolkit (VTK), CMake/FindLZ4.cmake: +# https://gitlab.kitware.com/vtk/vtk +# +# Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names +# of any contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#========================================================================= + +find_path( + ZSTD_INCLUDE_DIR + NAMES zstd.h + DOC "zstd include directory" +) +mark_as_advanced(ZSTD_INCLUDE_DIR) +find_library( + ZSTD_LIBRARY + NAMES zstd libzstd + DOC "zstd library" +) +mark_as_advanced(ZSTD_LIBRARY) + +if(ZSTD_INCLUDE_DIR) + file(STRINGS "${ZSTD_INCLUDE_DIR}/zstd.h" _zstd_version_lines REGEX "#define[ \t]+ZSTD_VERSION_(MAJOR|MINOR|RELEASE)") + string(REGEX REPLACE ".*ZSTD_VERSION_MAJOR *\([0-9]*\).*" "\\1" _zstd_version_major "${_zstd_version_lines}") + string(REGEX REPLACE ".*ZSTD_VERSION_MINOR *\([0-9]*\).*" "\\1" _zstd_version_minor "${_zstd_version_lines}") + string(REGEX REPLACE ".*ZSTD_VERSION_RELEASE *\([0-9]*\).*" "\\1" _zstd_version_release "${_zstd_version_lines}") + set(ZSTD_VERSION "${_zstd_version_major}.${_zstd_version_minor}.${_zstd_version_release}") + unset(_zstd_version_major) + unset(_zstd_version_minor) + unset(_zstd_version_release) + unset(_zstd_version_lines) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + ZSTD + REQUIRED_VARS ZSTD_LIBRARY ZSTD_INCLUDE_DIR + VERSION_VAR ZSTD_VERSION +) + +if(ZSTD_FOUND) + set(ZSTD_INCLUDE_DIRS "${ZSTD_INCLUDE_DIR}") + set(ZSTD_LIBRARIES "${ZSTD_LIBRARY}") + + if(NOT TARGET zstd::zstd) + add_library(zstd::zstd UNKNOWN IMPORTED) + set_target_properties( + zstd::zstd PROPERTIES IMPORTED_LOCATION "${ZSTD_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}" + ) + endif() +endif() diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index b362089c894..6433d62aa56 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2927,9 +2927,11 @@ RAM Cache Value Description ======== =================================================================== ``0`` No compression - ``1`` Fastlz (extremely fast, relatively low compression) - ``2`` Libz (moderate speed, reasonable compression) + ``1`` Fastlz (extremely fast, relatively low compression) - prefer lz4 + ``2`` Libz (moderate speed, reasonable compression) - prefer zstd ``3`` Liblzma (very slow, high compression) + ``4`` lz4 (extremely fast, relatively low compression) + ``5`` zstd (fast speed, reasonable compression) ======== =================================================================== Compression runs on task threads. To use more cores for RAM cache diff --git a/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst b/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst index c6e39eb7449..eb6b8b2c512 100644 --- a/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst @@ -132,6 +132,11 @@ a configuration with only one cache volume: :literal:`0`. Accumulates the number of misses to the LRU RAM cache for this volume. Note that this count includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk. +.. ts:stat:: global proxy.process.cache.volume_0.ram_cache.decompress.failure integer + :type: counter + + Accumulates the number of RAM cache entries that failed to decompress on read, for this volume. A failed entry is dropped from the RAM cache and the read is treated as a miss. A nonzero value indicates data corruption or a compression library error, not ordinary cache churn. + .. ts:stat:: global proxy.process.cache.volume_0.last_open_read.hits integer :type: counter diff --git a/doc/admin-guide/monitoring/statistics/core/cache.en.rst b/doc/admin-guide/monitoring/statistics/core/cache.en.rst index bb2f7d17f42..b06bb9802a5 100644 --- a/doc/admin-guide/monitoring/statistics/core/cache.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/cache.en.rst @@ -95,6 +95,11 @@ Cache Accumulates the number of misses to the LRU RAM cache for all volumes. Note that this includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk. +.. ts:stat:: global proxy.process.cache.ram_cache.decompress.failure integer + :type: counter + + Accumulates the number of RAM cache entries that failed to decompress on read, for all volumes. A failed entry is dropped from the RAM cache and the read is treated as a miss. A nonzero value indicates data corruption or a compression library error, not ordinary cache churn. + .. ts:stat:: global proxy.process.cache.last_open_read.hits integer :type: counter diff --git a/doc/admin-guide/storage/index.en.rst b/doc/admin-guide/storage/index.en.rst index f68808b1016..dce15c88e43 100644 --- a/doc/admin-guide/storage/index.en.rst +++ b/doc/admin-guide/storage/index.en.rst @@ -105,6 +105,8 @@ Value Meaning 1 *fastlz* compression 2 *libz* compression 3 *liblzma* compression +4 *lz4* compression +5 *zstd* compression ======= ============================= .. _changing-the-size-of-the-ram-cache: diff --git a/doc/developer-guide/cache-architecture/ram-cache.en.rst b/doc/developer-guide/cache-architecture/ram-cache.en.rst index bb6851e2e2d..2fa9c2caa46 100644 --- a/doc/developer-guide/cache-architecture/ram-cache.en.rst +++ b/doc/developer-guide/cache-architecture/ram-cache.en.rst @@ -37,7 +37,7 @@ following features: * Is Scan Resistant and extracts robust hit rates even when the working set does not fit in the RAM Cache. -* Supports compression at 3 levels: fastlz, gzip (libz), and xz (liblzma). +* Supports compression at 5 levels: fastlz, gzip (libz), xz (liblzma), lz4 and zstd. Compression can be moved to another thread. * Has very low CPU overhead, only slightly more than a basic LRU. Rather than @@ -72,8 +72,8 @@ len Length of the object, which differs from *size* because of compression and padding). compressed_len Compressed length of the object. compressed Compression type, or ``none`` if no compression. Possible types - are: *fastlz*, *libz*, and *liblzma*. -uncompressible Flag indicating that content cannot be compressed (true), or that + are: *fastlz*, *libz*, *liblzma*, *lz4* and *zstd*. +incompressible Flag indicating that content cannot be compressed (true), or that it mat be compressed (false). copy Whether or not this object should be copied in and copied out (e.g. HTTP HDR). @@ -147,18 +147,24 @@ since we need to make a copy anyway. Those not tagged ``copy`` are inserted uncompressed in the hope that they can be reused in uncompressed form. This is a compile time option and may be something we want to change. -There are 3 algorithms and levels of compression (speed on an Intel i7 920 -series processor using one thread): - -======= ================ ================== ==================================== -Method Compression Rate Decompression Rate Notes -======= ================ ================== ==================================== -fastlz 173 MB/sec 442 MB/sec Basically free since disk or network - will limit first; ~53% final size. -libz 55 MB/sec 234 MB/sec Almost free, particularly - decompression; ~37% final size. -liblzma 3 MB/sec 50 MB/sec Expensive; ~27% final size. -======= ================ ================== ==================================== +There are 5 algorithms and levels of compression (speed on an Intel Xeon Gold +6338 processor using lzbench and the silesia XML corpus): + +======= ===== ================= ================== ==================================== +Method Level Compression Rate Decompression Rate Notes +======= ===== ================= ================== ==================================== +fastlz 1 452 MB/sec 913 MB/sec Effectively obsolete; prefer lz4. + Basically free since disk or network + will limit first; ~26% final size. +libz 6 54 MB/sec 536 MB/sec Effectively obsolete; prefer zstd. + Almost free, particularly + decompression; ~13% final size. +liblzma 6 5 MB/sec 291 MB/sec Expensive; ~8% final size. +lz4 1 727 MB/sec 3458 MB/sec Basically free since disk or network + will limit first; ~23% final size. +zstd 3 508 MB/sec 1690 MB/sec Basically free since disk or network + will limit first; ~12% final size. +======= ===== ================= ================== ==================================== These are ballpark numbers, and your millage will vary enormously. JPEG, for example, will not compress with any of these (or at least will only do so at diff --git a/include/iocore/cache/Cache.h b/include/iocore/cache/Cache.h index 3a7da523b56..72568b5f636 100644 --- a/include/iocore/cache/Cache.h +++ b/include/iocore/cache/Cache.h @@ -43,8 +43,27 @@ static constexpr ts::ModuleVersion CACHE_MODULE_VERSION(1, 0); #define CACHE_COMPRESSION_FASTLZ 1 #define CACHE_COMPRESSION_LIBZ 2 #define CACHE_COMPRESSION_LIBLZMA 3 +#define CACHE_COMPRESSION_LZ4 4 +#define CACHE_COMPRESSION_ZSTD 5 + +enum { + RAM_HIT_COMPRESS_NONE = 1, + RAM_HIT_COMPRESS_FASTLZ, + RAM_HIT_COMPRESS_LIBZ, + RAM_HIT_COMPRESS_LIBLZMA, + RAM_HIT_COMPRESS_LZ4, + RAM_HIT_COMPRESS_ZSTD, + RAM_HIT_LAST_ENTRY +}; -enum { RAM_HIT_COMPRESS_NONE = 1, RAM_HIT_COMPRESS_FASTLZ, RAM_HIT_COMPRESS_LIBZ, RAM_HIT_COMPRESS_LIBLZMA, RAM_HIT_LAST_ENTRY }; +// The RAM_HIT_COMPRESS_* values are the CACHE_COMPRESSION_* values offset by +// one; keep the two sequences from silently desyncing when a codec is added. +static_assert(RAM_HIT_COMPRESS_NONE == CACHE_COMPRESSION_NONE + 1); +static_assert(RAM_HIT_COMPRESS_FASTLZ == CACHE_COMPRESSION_FASTLZ + 1); +static_assert(RAM_HIT_COMPRESS_LIBZ == CACHE_COMPRESSION_LIBZ + 1); +static_assert(RAM_HIT_COMPRESS_LIBLZMA == CACHE_COMPRESSION_LIBLZMA + 1); +static_assert(RAM_HIT_COMPRESS_LZ4 == CACHE_COMPRESSION_LZ4 + 1); +static_assert(RAM_HIT_COMPRESS_ZSTD == CACHE_COMPRESSION_ZSTD + 1); struct CacheVC; class CacheEvacuateDocVC; diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index 73c8b860fb9..11e78f69e47 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -47,6 +47,7 @@ #cmakedefine HAVE_POSIX_FALLOCATE 1 #cmakedefine HAVE_POSIX_MADVISE 1 #cmakedefine HAVE_ZSTD_H 1 +#cmakedefine HAVE_LZ4_H 1 #cmakedefine HAVE_PTHREAD_GETNAME_NP 1 #cmakedefine HAVE_PTHREAD_GET_NAME_NP 1 diff --git a/src/iocore/cache/CMakeLists.txt b/src/iocore/cache/CMakeLists.txt index f8b3817f722..a48cdc85335 100644 --- a/src/iocore/cache/CMakeLists.txt +++ b/src/iocore/cache/CMakeLists.txt @@ -56,6 +56,14 @@ if(HAVE_LZMA_H) target_link_libraries(inkcache PRIVATE LibLZMA::LibLZMA) endif() +if(HAVE_LZ4_H) + target_link_libraries(inkcache PRIVATE LZ4::LZ4) +endif() + +if(HAVE_ZSTD_H) + target_link_libraries(inkcache PRIVATE zstd::zstd) +endif() + if(BUILD_TESTING) # Unit Tests with unit_tests/main.cc macro(add_cache_test name) @@ -91,6 +99,7 @@ if(BUILD_TESTING) add_cache_test(Update_Header unit_tests/test_Update_header.cc) add_cache_test(CacheStripe unit_tests/test_Stripe.cc) add_cache_test(CacheAggregateWriteBuffer unit_tests/test_AggregateWriteBuffer.cc) + add_cache_test(RamCacheCLFUS unit_tests/test_RamCacheCLFUS.cc) # Unit Tests without unit_tests/main.cc add_executable(test_ConfigVolumes unit_tests/test_ConfigVolumes.cc) diff --git a/src/iocore/cache/CacheProcessor.cc b/src/iocore/cache/CacheProcessor.cc index ffff468ef78..725724f3625 100644 --- a/src/iocore/cache/CacheProcessor.cc +++ b/src/iocore/cache/CacheProcessor.cc @@ -1170,43 +1170,44 @@ register_cache_stats(CacheStatsBlock *rsb, const std::string &prefix) rsb->fragment_document_count[2] = ts::Metrics::Counter::createPtr(prefix + ".frags_per_doc.3+"); // And then everything else - rsb->bytes_used = ts::Metrics::Gauge::createPtr(prefix + ".bytes_used"); - rsb->bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".bytes_total"); - rsb->stripes = ts::Metrics::Gauge::createPtr(prefix + ".stripes"); - rsb->ram_cache_bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.total_bytes"); - rsb->ram_cache_bytes = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.bytes_used"); - rsb->ram_cache_hits = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.hits"); - rsb->last_open_read_hits = ts::Metrics::Counter::createPtr(prefix + ".last_open_read.hits"); - rsb->agg_buffer_hits = ts::Metrics::Counter::createPtr(prefix + ".aggregation_buffer.hits"); - rsb->ram_cache_misses = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.misses"); - rsb->all_mem_misses = ts::Metrics::Counter::createPtr(prefix + ".all_memory_caches.misses"); - rsb->pread_count = ts::Metrics::Counter::createPtr(prefix + ".pread_count"); - rsb->percent_full = ts::Metrics::Gauge::createPtr(prefix + ".percent_full"); - rsb->read_seek_fail = ts::Metrics::Counter::createPtr(prefix + ".read.seek.failure"); - rsb->read_invalid = ts::Metrics::Counter::createPtr(prefix + ".read.invalid"); - rsb->write_backlog_failure = ts::Metrics::Counter::createPtr(prefix + ".write.backlog.failure"); - rsb->direntries_total = ts::Metrics::Gauge::createPtr(prefix + ".direntries.total"); - rsb->direntries_used = ts::Metrics::Gauge::createPtr(prefix + ".direntries.used"); - rsb->directory_collision = ts::Metrics::Counter::createPtr(prefix + ".directory_collision"); - rsb->read_busy_success = ts::Metrics::Counter::createPtr(prefix + ".read_busy.success"); - rsb->read_busy_failure = ts::Metrics::Counter::createPtr(prefix + ".read_busy.failure"); - rsb->write_bytes = ts::Metrics::Counter::createPtr(prefix + ".write_bytes_stat"); - rsb->hdr_vector_marshal = ts::Metrics::Counter::createPtr(prefix + ".vector_marshals"); - rsb->hdr_marshal = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshals"); - rsb->hdr_marshal_bytes = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshal_bytes"); - rsb->gc_bytes_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_bytes_evacuated"); - rsb->gc_frags_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_frags_evacuated"); - rsb->directory_wrap = ts::Metrics::Counter::createPtr(prefix + ".wrap_count"); - rsb->directory_sync_count = ts::Metrics::Counter::createPtr(prefix + ".sync.count"); - rsb->directory_sync_bytes = ts::Metrics::Counter::createPtr(prefix + ".sync.bytes"); - rsb->directory_sync_time = ts::Metrics::Counter::createPtr(prefix + ".sync.time"); - rsb->span_errors_read = ts::Metrics::Counter::createPtr(prefix + ".span.errors.read"); - rsb->span_errors_write = ts::Metrics::Counter::createPtr(prefix + ".span.errors.write"); - rsb->span_failing = ts::Metrics::Gauge::createPtr(prefix + ".span.failing"); - rsb->span_offline = ts::Metrics::Gauge::createPtr(prefix + ".span.offline"); - rsb->span_online = ts::Metrics::Gauge::createPtr(prefix + ".span.online"); - rsb->stripe_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".stripe.lock_contention"); - rsb->writer_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".writer.lock_contention"); + rsb->bytes_used = ts::Metrics::Gauge::createPtr(prefix + ".bytes_used"); + rsb->bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".bytes_total"); + rsb->stripes = ts::Metrics::Gauge::createPtr(prefix + ".stripes"); + rsb->ram_cache_bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.total_bytes"); + rsb->ram_cache_bytes = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.bytes_used"); + rsb->ram_cache_hits = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.hits"); + rsb->last_open_read_hits = ts::Metrics::Counter::createPtr(prefix + ".last_open_read.hits"); + rsb->agg_buffer_hits = ts::Metrics::Counter::createPtr(prefix + ".aggregation_buffer.hits"); + rsb->ram_cache_misses = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.misses"); + rsb->ram_cache_decompress_failures = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.decompress.failure"); + rsb->all_mem_misses = ts::Metrics::Counter::createPtr(prefix + ".all_memory_caches.misses"); + rsb->pread_count = ts::Metrics::Counter::createPtr(prefix + ".pread_count"); + rsb->percent_full = ts::Metrics::Gauge::createPtr(prefix + ".percent_full"); + rsb->read_seek_fail = ts::Metrics::Counter::createPtr(prefix + ".read.seek.failure"); + rsb->read_invalid = ts::Metrics::Counter::createPtr(prefix + ".read.invalid"); + rsb->write_backlog_failure = ts::Metrics::Counter::createPtr(prefix + ".write.backlog.failure"); + rsb->direntries_total = ts::Metrics::Gauge::createPtr(prefix + ".direntries.total"); + rsb->direntries_used = ts::Metrics::Gauge::createPtr(prefix + ".direntries.used"); + rsb->directory_collision = ts::Metrics::Counter::createPtr(prefix + ".directory_collision"); + rsb->read_busy_success = ts::Metrics::Counter::createPtr(prefix + ".read_busy.success"); + rsb->read_busy_failure = ts::Metrics::Counter::createPtr(prefix + ".read_busy.failure"); + rsb->write_bytes = ts::Metrics::Counter::createPtr(prefix + ".write_bytes_stat"); + rsb->hdr_vector_marshal = ts::Metrics::Counter::createPtr(prefix + ".vector_marshals"); + rsb->hdr_marshal = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshals"); + rsb->hdr_marshal_bytes = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshal_bytes"); + rsb->gc_bytes_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_bytes_evacuated"); + rsb->gc_frags_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_frags_evacuated"); + rsb->directory_wrap = ts::Metrics::Counter::createPtr(prefix + ".wrap_count"); + rsb->directory_sync_count = ts::Metrics::Counter::createPtr(prefix + ".sync.count"); + rsb->directory_sync_bytes = ts::Metrics::Counter::createPtr(prefix + ".sync.bytes"); + rsb->directory_sync_time = ts::Metrics::Counter::createPtr(prefix + ".sync.time"); + rsb->span_errors_read = ts::Metrics::Counter::createPtr(prefix + ".span.errors.read"); + rsb->span_errors_write = ts::Metrics::Counter::createPtr(prefix + ".span.errors.write"); + rsb->span_failing = ts::Metrics::Gauge::createPtr(prefix + ".span.failing"); + rsb->span_offline = ts::Metrics::Gauge::createPtr(prefix + ".span.offline"); + rsb->span_online = ts::Metrics::Gauge::createPtr(prefix + ".span.online"); + rsb->stripe_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".stripe.lock_contention"); + rsb->writer_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".writer.lock_contention"); } void @@ -1648,6 +1649,16 @@ CacheProcessor::cacheInitialized() case CACHE_COMPRESSION_LIBLZMA: #ifndef HAVE_LZMA_H Fatal("lzma not available for RAM cache compression"); +#endif + break; + case CACHE_COMPRESSION_LZ4: +#ifndef HAVE_LZ4_H + Fatal("lz4 not available for RAM cache compression"); +#endif + break; + case CACHE_COMPRESSION_ZSTD: +#ifndef HAVE_ZSTD_H + Fatal("zstd not available for RAM cache compression"); #endif break; } diff --git a/src/iocore/cache/P_CacheStats.h b/src/iocore/cache/P_CacheStats.h index 7673cf56e0e..08120605b47 100644 --- a/src/iocore/cache/P_CacheStats.h +++ b/src/iocore/cache/P_CacheStats.h @@ -37,41 +37,42 @@ struct CacheStatsBlock { ts::Metrics::Counter::AtomicType *fragment_document_count[3] = {nullptr, nullptr, nullptr}; // For 1, 2 and 3+ fragments - ts::Metrics::Gauge::AtomicType *bytes_used = nullptr; - ts::Metrics::Gauge::AtomicType *bytes_total = nullptr; - ts::Metrics::Gauge::AtomicType *stripes = nullptr; - ts::Metrics::Gauge::AtomicType *ram_cache_bytes = nullptr; - ts::Metrics::Gauge::AtomicType *ram_cache_bytes_total = nullptr; - ts::Metrics::Gauge::AtomicType *direntries_total = nullptr; - ts::Metrics::Gauge::AtomicType *direntries_used = nullptr; - ts::Metrics::Counter::AtomicType *ram_cache_hits = nullptr; - ts::Metrics::Counter::AtomicType *last_open_read_hits = nullptr; - ts::Metrics::Counter::AtomicType *agg_buffer_hits = nullptr; - ts::Metrics::Counter::AtomicType *ram_cache_misses = nullptr; - ts::Metrics::Counter::AtomicType *all_mem_misses = nullptr; - ts::Metrics::Counter::AtomicType *pread_count = nullptr; - ts::Metrics::Gauge::AtomicType *percent_full = nullptr; - ts::Metrics::Counter::AtomicType *read_seek_fail = nullptr; - ts::Metrics::Counter::AtomicType *read_invalid = nullptr; - ts::Metrics::Counter::AtomicType *write_backlog_failure = nullptr; - ts::Metrics::Counter::AtomicType *directory_collision = nullptr; - ts::Metrics::Counter::AtomicType *read_busy_success = nullptr; - ts::Metrics::Counter::AtomicType *read_busy_failure = nullptr; - ts::Metrics::Counter::AtomicType *gc_bytes_evacuated = nullptr; - ts::Metrics::Counter::AtomicType *gc_frags_evacuated = nullptr; - ts::Metrics::Counter::AtomicType *write_bytes = nullptr; - ts::Metrics::Counter::AtomicType *hdr_vector_marshal = nullptr; - ts::Metrics::Counter::AtomicType *hdr_marshal = nullptr; - ts::Metrics::Counter::AtomicType *hdr_marshal_bytes = nullptr; - ts::Metrics::Counter::AtomicType *directory_wrap = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_count = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_time = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_bytes = nullptr; - ts::Metrics::Counter::AtomicType *span_errors_read = nullptr; - ts::Metrics::Counter::AtomicType *span_errors_write = nullptr; - ts::Metrics::Gauge::AtomicType *span_offline = nullptr; - ts::Metrics::Gauge::AtomicType *span_online = nullptr; - ts::Metrics::Gauge::AtomicType *span_failing = nullptr; - ts::Metrics::Counter::AtomicType *stripe_lock_contention = nullptr; - ts::Metrics::Counter::AtomicType *writer_lock_contention = nullptr; + ts::Metrics::Gauge::AtomicType *bytes_used = nullptr; + ts::Metrics::Gauge::AtomicType *bytes_total = nullptr; + ts::Metrics::Gauge::AtomicType *stripes = nullptr; + ts::Metrics::Gauge::AtomicType *ram_cache_bytes = nullptr; + ts::Metrics::Gauge::AtomicType *ram_cache_bytes_total = nullptr; + ts::Metrics::Gauge::AtomicType *direntries_total = nullptr; + ts::Metrics::Gauge::AtomicType *direntries_used = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_hits = nullptr; + ts::Metrics::Counter::AtomicType *last_open_read_hits = nullptr; + ts::Metrics::Counter::AtomicType *agg_buffer_hits = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_misses = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_decompress_failures = nullptr; + ts::Metrics::Counter::AtomicType *all_mem_misses = nullptr; + ts::Metrics::Counter::AtomicType *pread_count = nullptr; + ts::Metrics::Gauge::AtomicType *percent_full = nullptr; + ts::Metrics::Counter::AtomicType *read_seek_fail = nullptr; + ts::Metrics::Counter::AtomicType *read_invalid = nullptr; + ts::Metrics::Counter::AtomicType *write_backlog_failure = nullptr; + ts::Metrics::Counter::AtomicType *directory_collision = nullptr; + ts::Metrics::Counter::AtomicType *read_busy_success = nullptr; + ts::Metrics::Counter::AtomicType *read_busy_failure = nullptr; + ts::Metrics::Counter::AtomicType *gc_bytes_evacuated = nullptr; + ts::Metrics::Counter::AtomicType *gc_frags_evacuated = nullptr; + ts::Metrics::Counter::AtomicType *write_bytes = nullptr; + ts::Metrics::Counter::AtomicType *hdr_vector_marshal = nullptr; + ts::Metrics::Counter::AtomicType *hdr_marshal = nullptr; + ts::Metrics::Counter::AtomicType *hdr_marshal_bytes = nullptr; + ts::Metrics::Counter::AtomicType *directory_wrap = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_count = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_time = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_bytes = nullptr; + ts::Metrics::Counter::AtomicType *span_errors_read = nullptr; + ts::Metrics::Counter::AtomicType *span_errors_write = nullptr; + ts::Metrics::Gauge::AtomicType *span_offline = nullptr; + ts::Metrics::Gauge::AtomicType *span_online = nullptr; + ts::Metrics::Gauge::AtomicType *span_failing = nullptr; + ts::Metrics::Counter::AtomicType *stripe_lock_contention = nullptr; + ts::Metrics::Counter::AtomicType *writer_lock_contention = nullptr; }; diff --git a/src/iocore/cache/RamCacheCLFUS.cc b/src/iocore/cache/RamCacheCLFUS.cc index 639d2faa33c..daf8e00859c 100644 --- a/src/iocore/cache/RamCacheCLFUS.cc +++ b/src/iocore/cache/RamCacheCLFUS.cc @@ -24,6 +24,7 @@ // Clocked Least Frequently Used by Size (CLFUS) replacement policy // See https://cwiki.apache.org/confluence/display/TS/RamCache +#include "RamCacheCLFUS.h" #include "P_RamCache.h" #include "P_CacheInternal.h" #include "StripeSM.h" @@ -31,10 +32,66 @@ #include "iocore/eventsystem/Tasks.h" #include "fastlz/fastlz.h" #include "tscore/CryptoHash.h" +#include "tscore/Throttler.h" + +#include #include #ifdef HAVE_LZMA_H #include #endif +#ifdef HAVE_LZ4_H +#include +#endif +#ifdef HAVE_ZSTD_H +#include +#include +constexpr int CLFUS_ZSTD_LEVEL = 3; + +namespace +{ + +// One-shot ZSTD_compress/ZSTD_decompress allocate and free a context on every +// call, so reuse a per-thread context instead. May return nullptr if zstd +// fails to allocate one; that failure is sticky for the life of the thread, so +// warn when it happens. The compression level is a sticky parameter set once +// here; no explicit ZSTD_CCtx_reset() is needed because ZSTD_compress2() +// starts a new session on every call (resets are only for interrupting the +// streaming API or changing sticky parameters). +ZSTD_CCtx * +zstd_cctx() +{ + thread_local std::unique_ptr ctx = [] { + std::unique_ptr c{ZSTD_createCCtx(), ZSTD_freeCCtx}; + if (c && ZSTD_isError(ZSTD_CCtx_setParameter(c.get(), ZSTD_c_compressionLevel, CLFUS_ZSTD_LEVEL))) { + c.reset(); + } + if (!c) { + Warning("unable to allocate zstd compression context; RAM cache entries will not be compressed on this thread"); + } + return c; + }(); + return ctx.get(); +} + +ZSTD_DCtx * +zstd_dctx() +{ + thread_local std::unique_ptr ctx = [] { + std::unique_ptr c{ZSTD_createDCtx(), ZSTD_freeDCtx}; + if (!c) { + Warning("unable to allocate zstd decompression context; compressed RAM cache entries will miss on this thread"); + } + return c; + }(); + return ctx.get(); +} + +} // end anonymous namespace +#endif + +// The compression type is stored in the 3-bit RamCacheCLFUSEntry +// flag_bits.compressed field; a new codec value must still fit. +static_assert(CACHE_COMPRESSION_ZSTD < (1 << 3)); #define REQUIRED_COMPRESSION 0.9 // must get to this size or declared incompressible #define REQUIRED_SHRINK 0.8 // must get to this size or keep original buffer (with padding) @@ -62,67 +119,6 @@ DbgCtl dbg_ctl_ram_cache_compare{"ram_cache_compare"}; #endif -struct RamCacheCLFUSEntry { - CryptoHash key; - uint64_t auxkey; - uint64_t hits; - uint32_t size; // memory used including padding in buffer - uint32_t len; // actual data length - uint32_t compressed_len; - union { - struct { - uint32_t compressed : 3; // compression type - uint32_t incompressible : 1; - uint32_t lru : 1; - uint32_t copy : 1; // copy-in-copy-out - } flag_bits; - uint32_t flags; - }; - LINK(RamCacheCLFUSEntry, lru_link); - LINK(RamCacheCLFUSEntry, hash_link); - Ptr data; -}; - -class RamCacheCLFUS : public RamCache -{ -public: - RamCacheCLFUS() {} - - // returns 1 on found/stored, 0 on not found/stored, if provided auxkey1 and auxkey2 must match - int get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey = 0) override; - int put(CryptoHash *key, IOBufferData *data, uint32_t len, bool copy = false, uint64_t auxkey = 0) override; - int fixup(const CryptoHash *key, uint64_t old_auxkey, uint64_t new_auxkey) override; - int64_t size() const override; - - void init(int64_t max_bytes, StripeSM *stripe) override; - - void compress_entries(EThread *thread, int do_at_most = INT_MAX); - - // TODO move it to private. - StripeSM *stripe = nullptr; // for stats -private: - int64_t _max_bytes = 0; - int64_t _bytes = 0; - int64_t _objects = 0; - - double _average_value = 0; - int64_t _history = 0; - int _ibuckets = 0; - int _nbuckets = 0; - DList(RamCacheCLFUSEntry, hash_link) *_bucket = nullptr; - Que(RamCacheCLFUSEntry, lru_link) _lru[2]; - uint16_t *_seen = nullptr; - int _ncompressed = 0; - RamCacheCLFUSEntry *_compressed = nullptr; // first uncompressed lru[0] entry - - void _resize_hashtable(); - void _victimize(RamCacheCLFUSEntry *e); - void _move_compressed(RamCacheCLFUSEntry *e); - RamCacheCLFUSEntry *_destroy(RamCacheCLFUSEntry *e); - void _requeue_victims(Que(RamCacheCLFUSEntry, lru_link) & victims); - void _tick(); // move CLOCK on history -}; - int64_t RamCacheCLFUS::size() const { @@ -157,13 +153,21 @@ RamCacheCLFUSCompressor::mainEvent(int /* event ATS_UNUSED */, Event *e) Warning("unknown RAM cache compression type: %d", cache_config_ram_cache_compress); case CACHE_COMPRESSION_NONE: case CACHE_COMPRESSION_FASTLZ: - break; case CACHE_COMPRESSION_LIBZ: - Warning("libz not available for RAM cache compression"); break; case CACHE_COMPRESSION_LIBLZMA: #ifndef HAVE_LZMA_H Warning("lzma not available for RAM cache compression"); +#endif + break; + case CACHE_COMPRESSION_LZ4: +#ifndef HAVE_LZ4_H + Warning("lz4 not available for RAM cache compression"); +#endif + break; + case CACHE_COMPRESSION_ZSTD: +#ifndef HAVE_ZSTD_H + Warning("zstd not available for RAM cache compression"); #endif break; } @@ -298,6 +302,34 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey ram_hit_state = RAM_HIT_COMPRESS_LIBLZMA; break; } +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: { + int l = static_cast(e->len); + if (l != LZ4_decompress_safe(e->data->data(), b, e->compressed_len, l)) { + goto Lfailed; + } + ram_hit_state = RAM_HIT_COMPRESS_LZ4; + break; + } +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: { + size_t l = static_cast(e->len); + ZSTD_DCtx *dctx = zstd_dctx(); + if (dctx == nullptr) { + // This thread can't decompress, but the entry itself is fine: + // miss instead of evicting it. + ats_free(b); + goto Lerror; + } + size_t ll = ZSTD_decompressDCtx(dctx, b, l, e->data->data(), e->compressed_len); + if (ZSTD_isError(ll) || l != ll) { + goto Lfailed; + } + ram_hit_state = RAM_HIT_COMPRESS_ZSTD; + break; + } #endif } IOBufferData *data = new_xmalloc_IOBufferData(b, e->len); @@ -343,6 +375,20 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey return 0; Lfailed: ats_free(b); + { + // A failure here is data corruption or a codec error, not an ordinary + // miss; make it visible beyond the debug-gated trace below. + static Throttler decompress_failure_throttler(std::chrono::seconds(60)); + + uint64_t suppressed = 0; + if (!decompress_failure_throttler.is_throttled(suppressed)) { + Warning("RAM cache decompression failed: type %d len %u compressed_len %u key %X; entry dropped" + " (%" PRIu64 " similar failures suppressed)", + static_cast(e->flag_bits.compressed), e->len, e->compressed_len, key->slice32(3), suppressed); + } + ts::Metrics::Counter::increment(cache_rsb.ram_cache_decompress_failures); + ts::Metrics::Counter::increment(stripe->cache_vol->vol_rsb.ram_cache_decompress_failures); + } this->_destroy(e); DDbg(dbg_ctl_ram_cache, "get %X %" PRId64 " Z_ERR", key->slice32(3), auxkey); goto Lerror; @@ -463,7 +509,17 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) break; #ifdef HAVE_LZMA_H case CACHE_COMPRESSION_LIBLZMA: - l = e->len; + l = static_cast(lzma_stream_buffer_bound(e->len)); + break; +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: + l = static_cast(LZ4_compressBound(e->len)); + break; +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: + l = static_cast(ZSTD_compressBound(e->len)); break; #endif } @@ -504,6 +560,31 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) l = static_cast(pos); break; } +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: { + int ll = l; + if ((l = LZ4_compress_default(edata->data(), b, elen, ll)) == 0) { + failed = true; + } + break; + } +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: { + ZSTD_CCtx *cctx = zstd_cctx(); + if (cctx == nullptr) { + failed = true; + break; + } + size_t zret = ZSTD_compress2(cctx, b, l, edata->data(), elen); + if (ZSTD_isError(zret)) { + failed = true; + } else { + l = static_cast(zret); + } + break; + } #endif } MUTEX_TAKE_LOCK(stripe->mutex, thread); diff --git a/src/iocore/cache/RamCacheCLFUS.h b/src/iocore/cache/RamCacheCLFUS.h new file mode 100644 index 00000000000..037e0cfb159 --- /dev/null +++ b/src/iocore/cache/RamCacheCLFUS.h @@ -0,0 +1,99 @@ +/** @file + + Clocked Least Frequently Used by Size (CLFUS) RAM cache replacement policy. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +// See https://cwiki.apache.org/confluence/display/TS/RamCache + +#include "P_RamCache.h" + +#include "iocore/eventsystem/IOBuffer.h" +#include "tscore/CryptoHash.h" +#include "tscore/List.h" + +#include +#include + +class EThread; +class StripeSM; + +struct RamCacheCLFUSEntry { + CryptoHash key; + uint64_t auxkey; + uint64_t hits; + uint32_t size; // memory used including padding in buffer + uint32_t len; // actual data length + uint32_t compressed_len; + union { + struct { + uint32_t compressed : 3; // compression type + uint32_t incompressible : 1; + uint32_t lru : 1; + uint32_t copy : 1; // copy-in-copy-out + } flag_bits; + uint32_t flags; + }; + LINK(RamCacheCLFUSEntry, lru_link); + LINK(RamCacheCLFUSEntry, hash_link); + Ptr data; +}; + +class RamCacheCLFUS : public RamCache +{ +public: + RamCacheCLFUS() {} + + // returns 1 on found/stored, 0 on not found/stored, if provided auxkey1 and auxkey2 must match + int get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey = 0) override; + int put(CryptoHash *key, IOBufferData *data, uint32_t len, bool copy = false, uint64_t auxkey = 0) override; + int fixup(const CryptoHash *key, uint64_t old_auxkey, uint64_t new_auxkey) override; + int64_t size() const override; + + void init(int64_t max_bytes, StripeSM *stripe) override; + + void compress_entries(EThread *thread, int do_at_most = INT_MAX); + + // TODO move it to private. + StripeSM *stripe = nullptr; // for stats +private: + int64_t _max_bytes = 0; + int64_t _bytes = 0; + int64_t _objects = 0; + + double _average_value = 0; + int64_t _history = 0; + int _ibuckets = 0; + int _nbuckets = 0; + DList(RamCacheCLFUSEntry, hash_link) *_bucket = nullptr; + Que(RamCacheCLFUSEntry, lru_link) _lru[2]; + uint16_t *_seen = nullptr; + int _ncompressed = 0; + RamCacheCLFUSEntry *_compressed = nullptr; // first uncompressed lru[0] entry + + void _resize_hashtable(); + void _victimize(RamCacheCLFUSEntry *e); + void _move_compressed(RamCacheCLFUSEntry *e); + RamCacheCLFUSEntry *_destroy(RamCacheCLFUSEntry *e); + void _requeue_victims(Que(RamCacheCLFUSEntry, lru_link) & victims); + void _tick(); // move CLOCK on history +}; diff --git a/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc new file mode 100644 index 00000000000..cb234e682df --- /dev/null +++ b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc @@ -0,0 +1,265 @@ +/** @file + + Catch-based unit tests for RAM cache (CLFUS) compression roundtrips. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" + +#include "../RamCacheCLFUS.h" +#include "../P_CacheInternal.h" + +#include "iocore/cache/Cache.h" +#include "tscore/ink_config.h" + +#include +#include +#include +#include + +// Required by main.h +int cache_vols = 1; +bool reuse_existing_cache = false; + +namespace +{ + +// A compression backend to exercise, along with the RAM_HIT_* state get() +// should report once a compressible object has been stored compressed. +struct CompressionCase { + int config; // CACHE_COMPRESSION_* + int expected_hit; // RAM_HIT_COMPRESS_* reported by get() for compressible data + const char *name; +}; + +std::vector +compression_cases() +{ + std::vector cases{ + {CACHE_COMPRESSION_NONE, RAM_HIT_COMPRESS_NONE, "none" }, + {CACHE_COMPRESSION_FASTLZ, RAM_HIT_COMPRESS_FASTLZ, "fastlz"}, + {CACHE_COMPRESSION_LIBZ, RAM_HIT_COMPRESS_LIBZ, "libz" }, + }; +#ifdef HAVE_LZMA_H + cases.push_back({CACHE_COMPRESSION_LIBLZMA, RAM_HIT_COMPRESS_LIBLZMA, "liblzma"}); +#endif +#ifdef HAVE_LZ4_H + cases.push_back({CACHE_COMPRESSION_LZ4, RAM_HIT_COMPRESS_LZ4, "lz4"}); +#endif +#ifdef HAVE_ZSTD_H + cases.push_back({CACHE_COMPRESSION_ZSTD, RAM_HIT_COMPRESS_ZSTD, "zstd"}); +#endif + return cases; +} + +// Minimal CacheDisk wiring needed to construct a StripeSM. Mirrors the helper +// in test_Stripe.cc. +void +init_disk(CacheDisk &disk) +{ + disk.path = static_cast(ats_malloc(1)); + disk.path[0] = '\0'; + disk.disk_stripes = static_cast(ats_malloc(sizeof(DiskStripe *))); + disk.disk_stripes[0] = nullptr; + disk.header = static_cast(ats_malloc(sizeof(DiskHeader))); + disk.header->num_volumes = 0; +} + +// The CLFUS get/put/compress paths touch only these metrics and the stripe +// mutex, so that is all the stripe needs for these tests. +void +wire_stripe(StripeSM &stripe, CacheVol &cache_vol) +{ + stripe.cache_vol = &cache_vol; + + cache_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.ram_cache.bytes"); + cache_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.hits"); + cache_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.misses"); + cache_rsb.ram_cache_decompress_failures = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.decompress.failure"); + cache_vol.vol_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.vol.ram_cache.bytes"); + cache_vol.vol_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.hits"); + cache_vol.vol_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.misses"); + cache_vol.vol_rsb.ram_cache_decompress_failures = + ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.decompress.failure"); +} + +Ptr +make_buffer(const std::vector &bytes) +{ + int64_t idx = iobuffer_size_to_index(bytes.size(), MAX_BUFFER_SIZE_INDEX); + Ptr data{make_ptr(new_IOBufferData(idx, MEMALIGNED))}; + std::memcpy(data->data(), bytes.data(), bytes.size()); + return data; +} + +// Highly compressible: a short repeating pattern. +std::vector +compressible_bytes(std::size_t len) +{ + std::vector bytes(len); + for (std::size_t i = 0; i < len; i++) { + bytes[i] = static_cast('A' + (i % 26)); + } + return bytes; +} + +// Effectively incompressible: a deterministic xorshift byte stream. +std::vector +incompressible_bytes(std::size_t len) +{ + std::vector bytes(len); + uint32_t state = 0x9e3779b9; + for (std::size_t i = 0; i < len; i++) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + bytes[i] = static_cast(state & 0xff); + } + return bytes; +} + +struct RoundtripResult { + int hit = 0; + int64_t size_before = 0; // rc.size() after put, before the compression pass + int64_t size_after = 0; // rc.size() after the compression pass + std::vector out; +}; + +// Store payload under a fresh key, force a synchronous compression pass with +// `config`, then read it back. +RoundtripResult +store_compress_get(StripeSM &stripe, int config, const std::vector &payload) +{ + // Initialize with compression disabled so init() does not schedule the + // background compressor (which would retain a pointer to this stack object). + cache_config_ram_cache_compress = CACHE_COMPRESSION_NONE; + cache_config_ram_cache_compress_percent = 100; + cache_config_ram_cache_use_seen_filter = 0; + + RamCacheCLFUS rc; + rc.init(1 << 20, &stripe); + + Ptr in = make_buffer(payload); + uint32_t len = static_cast(payload.size()); + + static uint64_t salt = 0; + ++salt; + CryptoHash key; + key.u64[0] = 0xc0ffee00 + salt; + key.u64[1] = 0xdeadbeef + salt; + + REQUIRE(rc.put(&key, in.get(), len) == 1); + + RoundtripResult r; + + r.size_before = rc.size(); + cache_config_ram_cache_compress = config; + rc.compress_entries(this_ethread()); + r.size_after = rc.size(); + + Ptr ret; + + r.hit = rc.get(&key, &ret); + REQUIRE(ret.get() != nullptr); + r.out.assign(ret->data(), ret->data() + len); + return r; +} + +} // namespace + +TEST_CASE("CLFUS compressible objects roundtrip cleanly", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + // Large enough to exercise the *_compressBound() arithmetic and uint32_t + // casts in compress_entries(), not just small-buffer paths. + auto payload = compressible_bytes(256 * 1024); + const CompressionCase c = GENERATE(from_range(compression_cases())); + INFO("compression backend: " << c.name); + + RoundtripResult r = store_compress_get(stripe, c.config, payload); + + CHECK(r.hit == c.expected_hit); + CHECK(r.out == payload); + if (c.config != CACHE_COMPRESSION_NONE) { + // The feature's contract is that compression saves memory, not merely + // that the entry is tagged compressed. + CHECK(r.size_after < r.size_before); + } +} + +TEST_CASE("CLFUS incompressible objects fall back to uncompressed storage", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + auto payload = incompressible_bytes(256 * 1024); + + // Only the backends that actually attempt compression are interesting here; + // skip the NONE case. + auto cases = compression_cases(); + cases.erase(cases.begin()); + const CompressionCase c = GENERATE_REF(from_range(cases)); + INFO("compression backend: " << c.name); + + RoundtripResult r = store_compress_get(stripe, c.config, payload); + + // Incompressible data is kept verbatim, so a read reports no compression. + CHECK(r.hit == RAM_HIT_COMPRESS_NONE); + CHECK(r.out == payload); +} + +TEST_CASE("CLFUS single-byte payload roundtrips", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + RoundtripResult r = store_compress_get(stripe, CACHE_COMPRESSION_NONE, compressible_bytes(1)); + + CHECK(r.hit == RAM_HIT_COMPRESS_NONE); + CHECK(r.out == compressible_bytes(1)); +} + +// A backend that is not compiled in silently disappears from the parametrized +// cases above; make that visible in the test output rather than shipping an +// untested backend behind a green run. +TEST_CASE("CLFUS compression backends compiled in", "[cache][ramcache][compress]") +{ +#ifndef HAVE_LZMA_H + WARN("liblzma is not compiled in; the liblzma RAM cache compression backend is NOT tested"); +#endif +#ifndef HAVE_LZ4_H + WARN("lz4 is not compiled in; the lz4 RAM cache compression backend is NOT tested"); +#endif +#ifndef HAVE_ZSTD_H + WARN("zstd is not compiled in; the zstd RAM cache compression backend is NOT tested"); +#endif + CHECK(compression_cases().size() >= 3); +} diff --git a/src/traffic_layout/CMakeLists.txt b/src/traffic_layout/CMakeLists.txt index b86d4f2573a..ddc9c2674d8 100644 --- a/src/traffic_layout/CMakeLists.txt +++ b/src/traffic_layout/CMakeLists.txt @@ -35,6 +35,10 @@ if(HAVE_ZSTD_H) target_link_libraries(traffic_layout PRIVATE zstd::zstd) endif() +if(HAVE_LZ4_H) + target_link_libraries(traffic_layout PRIVATE LZ4::LZ4) +endif() + install(TARGETS traffic_layout) clang_tidy_check(traffic_layout) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index e77c2926d3e..30a5c9fd9cc 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -54,6 +54,10 @@ #include #endif +#if HAVE_LZ4_H +#include +#endif + #if HAVE_SSL_CTX_ADD_CERT_COMPRESSION_ALG static constexpr int ts_has_cert_compression_callbacks = 1; #else @@ -133,6 +137,11 @@ produce_features(bool json) print_feature("TS_HAS_ZSTD", 1, json); #else print_feature("TS_HAS_ZSTD", 0, json); +#endif +#ifdef HAVE_LZ4_H + print_feature("TS_HAS_LZ4", 1, json); +#else + print_feature("TS_HAS_LZ4", 0, json); #endif print_feature("TS_HAS_CERT_COMPRESSION", ts_has_cert_compression, json); print_feature("TS_HAS_CERT_COMPRESSION_CALLBACKS", ts_has_cert_compression_callbacks, json); @@ -256,6 +265,12 @@ produce_versions(bool json) #else print_var("zstd", undef, json); #endif +#ifdef HAVE_LZ4_H + print_var("lz4", LBW().print("{}", LZ4_VERSION_STRING).view(), json); + print_var("lz4.run", LBW().print("{}", LZ4_versionString()).view(), json); +#else + print_var("lz4", undef, json); +#endif // This should always be last print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true);