Add lz4 and zstd compression support to the CLFUS RAM cache

CMakeLists.txt

@@ -366,12 +366,17 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR})
 set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS})
 find_package(ZLIB REQUIRED)
 
-find_package(zstd CONFIG QUIET)
-if(zstd_FOUND)
+# 1.4.0 stabilized the advanced one-shot API (ZSTD_compress2 et al.) used by
+# the RAM cache.
+find_package(ZSTD 1.4.0)
+if(ZSTD_FOUND)
 
   # Provide a compatibility target name if the upstream package does not export it
   # Our code links against `zstd::zstd`; upstream zstd usually exports
   # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed.
+  # Normally this will be dead code if we use the packaged FindZSTD.cmake; but
+  # if CMAKE_FIND_PACKAGE_PREFER_CONFIG=1 and zstd-config.cmake is found, this
+  # may be useful.
   if(NOT TARGET zstd::zstd)
     if(TARGET zstd::libzstd_shared)
       set(_zstd_target zstd::libzstd_shared)
@@ -388,11 +393,18 @@ if(zstd_FOUND)
     else()
       set(HAVE_ZSTD_H FALSE)
     endif()
+  else()
+    set(HAVE_ZSTD_H TRUE)
   endif()
 else()
   set(HAVE_ZSTD_H FALSE)
 endif()
 
+find_package(LZ4)
+if(LZ4_FOUND)
+  set(HAVE_LZ4_H TRUE)
+endif()
+
 # ncurses is used in traffic_top
 find_package(Curses)
 set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H})

NOTICE

@@ -118,3 +118,10 @@ LS-HPACK provides functionality to encode and decode HTTP headers using
 HPACK compression mechanism specified in RFC 7541.
 Copyright (c) 2018 - 2023 LiteSpeed Technologies Inc, (MIT License)
 https://github.com/litespeedtech/ls-hpack.git
+
+~~
+
+cmake/FindLZ4.cmake and cmake/FindZSTD.cmake derived from:
+VTK: open-source software system for image processing, 3D graphics, volume rendering and visualization
+Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen (BSD-3-Clause License)
+https://gitlab.kitware.com/vtk/vtk

ci/docker/deb/Dockerfile

@@ -56,7 +56,7 @@ RUN apt-get update; apt-get -y dist-upgrade; \
     libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \
     tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \
     libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \
-    libzstd-dev; \
+    libzstd-dev liblz4-dev; \
     # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed)
     apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \
     groupadd  -g 665 jenkins && \

ci/docker/yum/Dockerfile

@@ -52,7 +52,7 @@ RUN yum -y update; \
     # Devel packages that ATS needs
     yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \
     xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \
-    ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \
+    ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel lz4-devel \
     perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \
     # This is for autest stuff
     yum -y install python3 httpd-tools procps-ng nmap-ncat \

ci/rat-exclude.txt

@@ -79,3 +79,5 @@ tools/http_load/**
 **/clang-tidy.conf
 build*/**
 cmake-build*/**
+cmake/FindLZ4.cmake
+cmake/FindZSTD.cmake

cmake/FindLZ4.cmake

@@ -0,0 +1,78 @@
+#=========================================================================
+#
+# Sourced from the Visualization Toolkit (VTK), CMake/FindLZ4.cmake:
+# https://gitlab.kitware.com/vtk/vtk
+#
+# Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names
+#    of any contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#=========================================================================
+
+find_path(
+  LZ4_INCLUDE_DIR
+  NAMES lz4.h
+  DOC "lz4 include directory"
+)
+mark_as_advanced(LZ4_INCLUDE_DIR)
+find_library(
+  LZ4_LIBRARY
+  NAMES lz4 liblz4
+  DOC "lz4 library"
+)
+mark_as_advanced(LZ4_LIBRARY)
+
+if(LZ4_INCLUDE_DIR)
+  file(STRINGS "${LZ4_INCLUDE_DIR}/lz4.h" _lz4_version_lines REGEX "#define[ \t]+LZ4_VERSION_(MAJOR|MINOR|RELEASE)")
+  string(REGEX REPLACE ".*LZ4_VERSION_MAJOR *\([0-9]*\).*" "\\1" _lz4_version_major "${_lz4_version_lines}")
+  string(REGEX REPLACE ".*LZ4_VERSION_MINOR *\([0-9]*\).*" "\\1" _lz4_version_minor "${_lz4_version_lines}")
+  string(REGEX REPLACE ".*LZ4_VERSION_RELEASE *\([0-9]*\).*" "\\1" _lz4_version_release "${_lz4_version_lines}")
+  set(LZ4_VERSION "${_lz4_version_major}.${_lz4_version_minor}.${_lz4_version_release}")
+  unset(_lz4_version_major)
+  unset(_lz4_version_minor)
+  unset(_lz4_version_release)
+  unset(_lz4_version_lines)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+  LZ4
+  REQUIRED_VARS LZ4_LIBRARY LZ4_INCLUDE_DIR
+  VERSION_VAR LZ4_VERSION
+)
+
+if(LZ4_FOUND)
+  set(LZ4_INCLUDE_DIRS "${LZ4_INCLUDE_DIR}")
+  set(LZ4_LIBRARIES "${LZ4_LIBRARY}")
+
+  if(NOT TARGET LZ4::LZ4)
+    add_library(LZ4::LZ4 UNKNOWN IMPORTED)
+    set_target_properties(
+      LZ4::LZ4 PROPERTIES IMPORTED_LOCATION "${LZ4_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}"
+    )
+  endif()
+endif()

cmake/FindZSTD.cmake

@@ -0,0 +1,78 @@
+#=========================================================================
+#
+# Derived from the Visualization Toolkit (VTK), CMake/FindLZ4.cmake:
+# https://gitlab.kitware.com/vtk/vtk
+#
+# Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names
+#    of any contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#=========================================================================
+
+find_path(
+  ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  DOC "zstd include directory"
+)
+mark_as_advanced(ZSTD_INCLUDE_DIR)
+find_library(
+  ZSTD_LIBRARY
+  NAMES zstd libzstd
+  DOC "zstd library"
+)
+mark_as_advanced(ZSTD_LIBRARY)
+
+if(ZSTD_INCLUDE_DIR)
+  file(STRINGS "${ZSTD_INCLUDE_DIR}/zstd.h" _zstd_version_lines REGEX "#define[ \t]+ZSTD_VERSION_(MAJOR|MINOR|RELEASE)")
+  string(REGEX REPLACE ".*ZSTD_VERSION_MAJOR *\([0-9]*\).*" "\\1" _zstd_version_major "${_zstd_version_lines}")
+  string(REGEX REPLACE ".*ZSTD_VERSION_MINOR *\([0-9]*\).*" "\\1" _zstd_version_minor "${_zstd_version_lines}")
+  string(REGEX REPLACE ".*ZSTD_VERSION_RELEASE *\([0-9]*\).*" "\\1" _zstd_version_release "${_zstd_version_lines}")
+  set(ZSTD_VERSION "${_zstd_version_major}.${_zstd_version_minor}.${_zstd_version_release}")
+  unset(_zstd_version_major)
+  unset(_zstd_version_minor)
+  unset(_zstd_version_release)
+  unset(_zstd_version_lines)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+  ZSTD
+  REQUIRED_VARS ZSTD_LIBRARY ZSTD_INCLUDE_DIR
+  VERSION_VAR ZSTD_VERSION
+)
+
+if(ZSTD_FOUND)
+  set(ZSTD_INCLUDE_DIRS "${ZSTD_INCLUDE_DIR}")
+  set(ZSTD_LIBRARIES "${ZSTD_LIBRARY}")
+
+  if(NOT TARGET zstd::zstd)
+    add_library(zstd::zstd UNKNOWN IMPORTED)
+    set_target_properties(
+      zstd::zstd PROPERTIES IMPORTED_LOCATION "${ZSTD_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}"
+    )
+  endif()
+endif()

doc/admin-guide/files/records.yaml.en.rst

@@ -2927,9 +2927,11 @@ RAM Cache
    Value    Description
    ======== ===================================================================
    ``0``    No compression
-   ``1``    Fastlz (extremely fast, relatively low compression)
-   ``2``    Libz (moderate speed, reasonable compression)
+   ``1``    Fastlz (extremely fast, relatively low compression) - prefer lz4
+   ``2``    Libz (moderate speed, reasonable compression) - prefer zstd
    ``3``    Liblzma (very slow, high compression)
+   ``4``    lz4 (extremely fast, relatively low compression)
+   ``5``    zstd (fast speed, reasonable compression)
    ======== ===================================================================
 
    Compression runs on task threads. To use more cores for RAM cache

doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst

@@ -132,6 +132,11 @@ a configuration with only one cache volume: :literal:`0`.
 
    Accumulates the number of misses to the LRU RAM cache for this volume.  Note that this count includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk.
 
+.. ts:stat:: global proxy.process.cache.volume_0.ram_cache.decompress.failure integer
+   :type: counter
+
+   Accumulates the number of RAM cache entries that failed to decompress on read, for this volume.  A failed entry is dropped from the RAM cache and the read is treated as a miss.  A nonzero value indicates data corruption or a compression library error, not ordinary cache churn.
+
 .. ts:stat:: global proxy.process.cache.volume_0.last_open_read.hits integer
    :type: counter
 

doc/admin-guide/monitoring/statistics/core/cache.en.rst

@@ -95,6 +95,11 @@ Cache
 
    Accumulates the number of misses to the LRU RAM cache for all volumes.  Note that this includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk.
 
+.. ts:stat:: global proxy.process.cache.ram_cache.decompress.failure integer
+   :type: counter
+
+   Accumulates the number of RAM cache entries that failed to decompress on read, for all volumes.  A failed entry is dropped from the RAM cache and the read is treated as a miss.  A nonzero value indicates data corruption or a compression library error, not ordinary cache churn.
+
 .. ts:stat:: global proxy.process.cache.last_open_read.hits integer
    :type: counter
 

doc/admin-guide/storage/index.en.rst

@@ -105,6 +105,8 @@ Value   Meaning
 1       *fastlz* compression
 2       *libz* compression
 3       *liblzma* compression
+4       *lz4* compression
+5       *zstd* compression
 ======= =============================
 
 .. _changing-the-size-of-the-ram-cache:

doc/developer-guide/cache-architecture/ram-cache.en.rst

@@ -37,7 +37,7 @@ following features:
 * Is Scan Resistant and extracts robust hit rates even when the working set does
   not fit in the RAM Cache.
 
-* Supports compression at 3 levels: fastlz, gzip (libz), and xz (liblzma).
+* Supports compression at 5 levels: fastlz, gzip (libz), xz (liblzma), lz4 and zstd.
   Compression can be moved to another thread.
 
 * Has very low CPU overhead, only slightly more than a basic LRU. Rather than
@@ -72,8 +72,8 @@ len            Length of the object, which differs from *size* because of
                compression and padding).
 compressed_len Compressed length of the object.
 compressed     Compression type, or ``none`` if no compression. Possible types
-               are: *fastlz*, *libz*, and *liblzma*.
-uncompressible Flag indicating that content cannot be compressed (true), or that
+               are: *fastlz*, *libz*, *liblzma*, *lz4* and *zstd*.
+incompressible Flag indicating that content cannot be compressed (true), or that
                it mat be compressed (false).
 copy           Whether or not this object should be copied in and copied out
                (e.g. HTTP HDR).
@@ -147,18 +147,24 @@ since we need to make a copy anyway. Those not tagged ``copy`` are inserted
 uncompressed in the hope that they can be reused in uncompressed form. This is
 a compile time option and may be something we want to change.
 
-There are 3 algorithms and levels of compression (speed on an Intel i7 920
-series processor using one thread):
-
-======= ================ ================== ====================================
-Method  Compression Rate Decompression Rate Notes
-======= ================ ================== ====================================
-fastlz  173 MB/sec       442 MB/sec         Basically free since disk or network
-                                            will limit first; ~53% final size.
-libz    55 MB/sec        234 MB/sec         Almost free, particularly
-                                            decompression; ~37% final size.
-liblzma 3 MB/sec         50 MB/sec          Expensive; ~27% final size.
-======= ================ ================== ====================================
+There are 5 algorithms and levels of compression (speed on an Intel Xeon Gold
+6338 processor using lzbench and the silesia XML corpus):
+
+======= ===== ================= ================== ====================================
+Method  Level Compression Rate  Decompression Rate Notes
+======= ===== ================= ================== ====================================
+fastlz    1   452 MB/sec        913 MB/sec         Effectively obsolete; prefer lz4.
+                                                   Basically free since disk or network
+                                                   will limit first; ~26% final size.
+libz      6   54 MB/sec         536 MB/sec         Effectively obsolete; prefer zstd.
+                                                   Almost free, particularly
+                                                   decompression; ~13% final size.
+liblzma   6   5 MB/sec          291 MB/sec         Expensive; ~8% final size.
+lz4       1   727 MB/sec        3458 MB/sec        Basically free since disk or network
+                                                   will limit first; ~23% final size.
+zstd      3   508 MB/sec        1690 MB/sec        Basically free since disk or network
+                                                   will limit first; ~12% final size.
+======= ===== ================= ================== ====================================
 
 These are ballpark numbers, and your millage will vary enormously. JPEG, for
 example, will not compress with any of these (or at least will only do so at

include/iocore/cache/Cache.h

@@ -43,8 +43,27 @@ static constexpr ts::ModuleVersion CACHE_MODULE_VERSION(1, 0);
 #define CACHE_COMPRESSION_FASTLZ  1
 #define CACHE_COMPRESSION_LIBZ    2
 #define CACHE_COMPRESSION_LIBLZMA 3
+#define CACHE_COMPRESSION_LZ4     4
+#define CACHE_COMPRESSION_ZSTD    5
+
+enum {
+  RAM_HIT_COMPRESS_NONE = 1,
+  RAM_HIT_COMPRESS_FASTLZ,
+  RAM_HIT_COMPRESS_LIBZ,
+  RAM_HIT_COMPRESS_LIBLZMA,
+  RAM_HIT_COMPRESS_LZ4,
+  RAM_HIT_COMPRESS_ZSTD,
+  RAM_HIT_LAST_ENTRY
+};
 
-enum { RAM_HIT_COMPRESS_NONE = 1, RAM_HIT_COMPRESS_FASTLZ, RAM_HIT_COMPRESS_LIBZ, RAM_HIT_COMPRESS_LIBLZMA, RAM_HIT_LAST_ENTRY };
+// The RAM_HIT_COMPRESS_* values are the CACHE_COMPRESSION_* values offset by
+// one; keep the two sequences from silently desyncing when a codec is added.
+static_assert(RAM_HIT_COMPRESS_NONE == CACHE_COMPRESSION_NONE + 1);
+static_assert(RAM_HIT_COMPRESS_FASTLZ == CACHE_COMPRESSION_FASTLZ + 1);
+static_assert(RAM_HIT_COMPRESS_LIBZ == CACHE_COMPRESSION_LIBZ + 1);
+static_assert(RAM_HIT_COMPRESS_LIBLZMA == CACHE_COMPRESSION_LIBLZMA + 1);
+static_assert(RAM_HIT_COMPRESS_LZ4 == CACHE_COMPRESSION_LZ4 + 1);
+static_assert(RAM_HIT_COMPRESS_ZSTD == CACHE_COMPRESSION_ZSTD + 1);
 
 struct CacheVC;
 class CacheEvacuateDocVC;

include/tscore/ink_config.h.cmake.in

@@ -47,6 +47,7 @@
 #cmakedefine HAVE_POSIX_FALLOCATE 1
 #cmakedefine HAVE_POSIX_MADVISE 1
 #cmakedefine HAVE_ZSTD_H 1
+#cmakedefine HAVE_LZ4_H 1
 
 #cmakedefine HAVE_PTHREAD_GETNAME_NP 1
 #cmakedefine HAVE_PTHREAD_GET_NAME_NP 1

src/iocore/cache/CMakeLists.txt

@@ -56,6 +56,14 @@ if(HAVE_LZMA_H)
   target_link_libraries(inkcache PRIVATE LibLZMA::LibLZMA)
 endif()
 
+if(HAVE_LZ4_H)
+  target_link_libraries(inkcache PRIVATE LZ4::LZ4)
+endif()
+
+if(HAVE_ZSTD_H)
+  target_link_libraries(inkcache PRIVATE zstd::zstd)
+endif()
+
 if(BUILD_TESTING)
   # Unit Tests with unit_tests/main.cc
   macro(add_cache_test name)
@@ -91,6 +99,7 @@ if(BUILD_TESTING)
   add_cache_test(Update_Header unit_tests/test_Update_header.cc)
   add_cache_test(CacheStripe unit_tests/test_Stripe.cc)
   add_cache_test(CacheAggregateWriteBuffer unit_tests/test_AggregateWriteBuffer.cc)
+  add_cache_test(RamCacheCLFUS unit_tests/test_RamCacheCLFUS.cc)
 
   # Unit Tests without unit_tests/main.cc
   add_executable(test_ConfigVolumes unit_tests/test_ConfigVolumes.cc)