diff -Nadur a/cmake/lzma.cmake b/cmake/lzma.cmake --- a/cmake/lzma.cmake 1970-01-01 01:00:00.000000000 +0100 +++ b/cmake/lzma.cmake 2022-08-31 12:05:20.629322237 +0200 @@ -0,0 +1,61 @@ +# cmake -DWITH_LZMA=system + +SET(MIN_LZMA_VERSION_REQUIRED "5.2.0") + +MACRO (FIND_LZMA_VERSION) + FOREACH(version_part + LZMA_VERSION_MAJOR + LZMA_VERSION_MINOR + LZMA_VERSION_PATCH + ) + FILE(STRINGS "${LZMA_INCLUDE_DIR}/lzma/version.h" ${version_part} + REGEX "^#[\t ]*define[\t ]+${version_part}[\t ]+([0-9]+).*") + STRING(REGEX REPLACE + "^.*${version_part}[\t ]+([0-9]+).*" "\\1" + ${version_part} "${${version_part}}") + ENDFOREACH() + SET(LZMA_VERSION + "${LZMA_VERSION_MAJOR}.${LZMA_VERSION_MINOR}.${LZMA_VERSION_PATCH}") + SET(LZMA_VERSION "${LZMA_VERSION}" CACHE INTERNAL "LZMA major.minor.step") + MESSAGE(STATUS "LZMA_VERSION (${WITH_LZMA}) is ${LZMA_VERSION}") + MESSAGE(STATUS "LZMA_INCLUDE_DIR ${LZMA_INCLUDE_DIR}") + MESSAGE(STATUS "LZMA_LIBRARY ${LZMA_LIBRARY}") +ENDMACRO() + +MACRO (FIND_SYSTEM_LZMA) + FIND_PATH(LZMA_INCLUDE_DIR + NAMES lzma.h + PATH_SUFFIXES include) + FIND_LIBRARY(LZMA_SYSTEM_LIBRARY + NAMES lzma + PATH_SUFFIXES lib) + IF (LZMA_INCLUDE_DIR AND LZMA_SYSTEM_LIBRARY) + SET(SYSTEM_LZMA_FOUND 1) + SET(LZMA_LIBRARY ${LZMA_SYSTEM_LIBRARY}) + IF(NOT LZMA_INCLUDE_DIR STREQUAL "/usr/include") + # In case of -DCMAKE_PREFIX_PATH= + INCLUDE_DIRECTORIES(BEFORE SYSTEM ${LZMA_INCLUDE_DIR}) + ENDIF() + ENDIF() +ENDMACRO() + +MACRO (MYSQL_CHECK_LZMA) + IF(NOT WITH_LZMA) + SET(WITH_LZMA "system" CACHE STRING "By default use system lzma library") + ENDIF() + + IF(WITH_LZMA STREQUAL "system") + FIND_SYSTEM_LZMA() + IF (NOT SYSTEM_LZMA_FOUND) + MESSAGE(FATAL_ERROR "Cannot find system lzma libraries.") + ENDIF() + ELSE() + MESSAGE(FATAL_ERROR "WITH_LZMA must be system") + ENDIF() + FIND_LZMA_VERSION() + IF(LZMA_VERSION VERSION_LESS MIN_LZMA_VERSION_REQUIRED) + MESSAGE(FATAL_ERROR + "LZMA version must be at least ${MIN_LZMA_VERSION_REQUIRED}, " + "found ${LZMA_VERSION}.\n") + ENDIF() +ENDMACRO() diff -Nadur a/CMakeLists.txt b/CMakeLists.txt --- a/CMakeLists.txt 2022-07-13 20:42:43.000000000 +0200 +++ b/CMakeLists.txt 2022-08-31 12:06:33.527676629 +0200 @@ -766,6 +766,7 @@ INCLUDE(fileutils) INCLUDE(zlib) INCLUDE(zstd) +INCLUDE(lzma) INCLUDE(lz4) INCLUDE(icu) INCLUDE(libevent) @@ -1680,6 +1681,7 @@ PROTOBUF # Homebrew protobuf SSL # Homebrew openssl@1.1 ZSTD # Homebrew zstd + LZMA # Homebrew lzma FIDO # Homebrew libfido2 ) @@ -1741,6 +1743,9 @@ # Add bundled or system zstd. MYSQL_CHECK_ZSTD() +# Add system lzma. +MYSQL_CHECK_LZMA() + IF(LINUX) # Look up patchelf(1) here, before doing SSL/KERBEROS/SASL/LDAP config. FIND_PROGRAM(PATCHELF_EXECUTABLE patchelf) @@ -1930,7 +1935,7 @@ IF(APPLE) GET_FILENAME_COMPONENT(HOMEBREW_BASE ${HOMEBREW_HOME} DIRECTORY) IF(EXISTS ${HOMEBREW_BASE}/include/boost) - FOREACH(SYSTEM_LIB ICU LIBEVENT LZ4 PROTOBUF ZSTD FIDO) + FOREACH(SYSTEM_LIB ICU LIBEVENT LZ4 PROTOBUF ZSTD LZMA FIDO) IF(WITH_${SYSTEM_LIB} STREQUAL "system") MESSAGE(FATAL_ERROR "WITH_${SYSTEM_LIB}=system is not compatible with Homebrew boost\n" @@ -1944,7 +1949,7 @@ ENDFOREACH() ENDIF() # Ensure that we look in /usr/local/include or /opt/homebrew/include - FOREACH(SYSTEM_LIB ICU LIBEVENT LZ4 PROTOBUF ZSTD FIDO) + FOREACH(SYSTEM_LIB ICU LIBEVENT LZ4 PROTOBUF ZSTD LZMA FIDO) IF(WITH_${SYSTEM_LIB} STREQUAL "system") INCLUDE_DIRECTORIES(SYSTEM ${HOMEBREW_BASE}/include) BREAK() diff -Nadur a/sql/handler.h b/sql/handler.h --- a/sql/handler.h 2022-08-31 14:09:16.027462657 +0200 +++ b/sql/handler.h 2022-08-31 12:23:39.994828396 +0200 @@ -755,7 +755,7 @@ */ #define HA_CREATE_USED_TABLESPACE (1L << 25) -/** COMPRESSION="zlib|lz4|zstd|none" used during table create. */ +/** COMPRESSION="zlib|lz4|zstd|lzma|none" used during table create. */ #define HA_CREATE_USED_COMPRESS (1L << 26) /** ENCRYPTION="Y" used during table create. */ diff -Nadur a/storage/innobase/clone/clone0desc.cc b/storage/innobase/clone/clone0desc.cc --- a/storage/innobase/clone/clone0desc.cc 2022-08-31 14:09:16.034462883 +0200 +++ b/storage/innobase/clone/clone0desc.cc 2022-08-31 12:24:24.100252862 +0200 @@ -633,6 +633,8 @@ static const uint CLONE_DESC_FILE_HAS_KEY = 6; /** Clone File Flag: Compression type ZSTD*/ static const uint CLONE_DESC_FILE_FLAG_ZSTD = 7; +/** Clone File Flag: Compression type LZMA*/ +static const uint CLONE_DESC_FILE_FLAG_LZMA = 8; /** File Metadata: Tablespace ID in 4 bytes */ static const uint CLONE_FILE_SPACE_ID_OFFSET = CLONE_FILE_FLAGS_OFFSET + 2; @@ -701,6 +703,8 @@ DESC_SET_FLAG(file_flags, CLONE_DESC_FILE_FLAG_LZ4); } else if (m_file_meta.m_compress_type == Compression::ZSTD) { DESC_SET_FLAG(file_flags, CLONE_DESC_FILE_FLAG_ZSTD); + } else if (m_file_meta.m_compress_type == Compression::LZMA) { + DESC_SET_FLAG(file_flags, CLONE_DESC_FILE_FLAG_LZMA); } /* Set file encryption type */ if (m_file_meta.m_encryption_metadata.m_type == Encryption::AES) { @@ -791,6 +795,8 @@ m_file_meta.m_compress_type = Compression::LZ4; } else if (DESC_CHECK_FLAG(file_flags, CLONE_DESC_FILE_FLAG_ZSTD)) { m_file_meta.m_compress_type = Compression::ZSTD; + } else if (DESC_CHECK_FLAG(file_flags, CLONE_DESC_FILE_FLAG_LZMA)) { + m_file_meta.m_compress_type = Compression::LZMA; } /* Get file encryption information */ diff -Nadur a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt --- a/storage/innobase/CMakeLists.txt 2022-08-31 14:09:16.040463077 +0200 +++ b/storage/innobase/CMakeLists.txt 2022-08-31 12:24:51.385134083 +0200 @@ -317,7 +317,7 @@ ${INNOBASE_SOURCES} ${INNOBASE_ZIP_DECOMPRESS_SOURCES} STORAGE_ENGINE MANDATORY MODULE_OUTPUT_NAME ha_innodb - LINK_LIBRARIES sql_dd sql_gis ext::zlib ext::lz4 ${ZSTD_LIBRARY} ${NUMA_LIBRARY} + LINK_LIBRARIES sql_dd sql_gis ext::zlib ext::lz4 ${ZSTD_LIBRARY} ${LZMA_LIBRARY} ${NUMA_LIBRARY} extra::rapidjson) # On linux: /usr/include/stdio.h:#define BUFSIZ 8192 diff -Nadur a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc --- a/storage/innobase/fil/fil0fil.cc 2022-08-31 14:09:16.048463335 +0200 +++ b/storage/innobase/fil/fil0fil.cc 2022-08-31 12:25:12.674821674 +0200 @@ -8865,6 +8865,7 @@ case Compression::LZ4: case Compression::ZLIB: case Compression::ZSTD: + case Compression::LZMA: case Compression::NONE: compression.m_type = static_cast(srv_debug_compress); diff -Nadur a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc --- a/storage/innobase/handler/ha_innodb.cc 2022-08-31 14:09:16.056463593 +0200 +++ b/storage/innobase/handler/ha_innodb.cc 2022-08-31 12:33:19.910557885 +0200 @@ -444,7 +444,7 @@ #ifdef UNIV_DEBUG /** Values for --innodb-debug-compress names. */ -static const char *innodb_debug_compress_names[] = {"none", "zlib", "lz4", "zstd", +static const char *innodb_debug_compress_names[] = {"none", "zlib", "lz4", "zstd", "lzma", "lz4hc", NullS}; /** Enumeration of --innodb-debug-compress */ @@ -2604,6 +2604,9 @@ } else if (innobase_strcasecmp(algorithm, "zstd") == 0) { compression->m_type = ZSTD; + } else if (innobase_strcasecmp(algorithm, "lzma") == 0) { + compression->m_type = LZMA; + } else { return (DB_UNSUPPORTED); } @@ -2628,6 +2631,7 @@ case ZLIB: case LZ4: case ZSTD: + case LZMA: break; default: ret = false; @@ -22169,6 +22173,12 @@ "1 is fastest, 20 is best compression, 22 is max (ultra) compression and default is 3.", nullptr, nullptr, DEFAULT_ZSTD_COMPRESSION_LEVEL, 1, 22, 0); +static MYSQL_SYSVAR_UINT( + lzma_compression_level, page_lzma_level, PLUGIN_VAR_RQCMDARG, + "Compression level used for LZMA page compression. " + "0 is fastest, 9 is best compression and default is 6.", + nullptr, nullptr, DEFAULT_LZMA_COMPRESSION_LEVEL, 0, 9, 0); + static MYSQL_SYSVAR_BOOL( log_compressed_pages, page_zip_log_pages, PLUGIN_VAR_OPCMDARG, "Enables/disables the logging of entire compressed page images." @@ -23144,6 +23154,7 @@ MYSQL_SYSVAR(concurrency_tickets), MYSQL_SYSVAR(compression_level), MYSQL_SYSVAR(zstd_compression_level), + MYSQL_SYSVAR(lzma_compression_level), MYSQL_SYSVAR(ddl_buffer_size), MYSQL_SYSVAR(ddl_threads), MYSQL_SYSVAR(data_file_path), diff -Nadur a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h --- a/storage/innobase/include/page0zip.h 2022-08-31 14:09:16.068463981 +0200 +++ b/storage/innobase/include/page0zip.h 2022-08-31 12:33:57.937786048 +0200 @@ -55,10 +55,12 @@ /* Compression level to be used by zlib. Settable by user. */ extern uint page_zip_level; extern uint page_zstd_level; +extern uint page_lzma_level; /* Default compression level. */ constexpr uint32_t DEFAULT_COMPRESSION_LEVEL = 6; constexpr uint32_t DEFAULT_ZSTD_COMPRESSION_LEVEL = 3; +constexpr uint32_t DEFAULT_LZMA_COMPRESSION_LEVEL = 6; /** Start offset of the area that will be compressed */ #define PAGE_ZIP_START PAGE_NEW_SUPREMUM_END /** Predefine the sum of DIR_SLOT, TRX_ID & ROLL_PTR */ diff -Nadur a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i --- a/storage/innobase/include/univ.i 2022-08-31 14:09:16.075464207 +0200 +++ b/storage/innobase/include/univ.i 2022-08-31 12:34:23.722618820 +0200 @@ -371,7 +371,7 @@ MAX_TABLE_NAME_LEN + MAX_DATABASE_NAME_LEN + 14; /** Maximum length of the compression algorithm string. Currently we support -only (NONE | ZLIB | LZ4 | ZSTD). */ +only (NONE | ZLIB | LZ4 | ZSTD | LZMA). */ constexpr uint32_t MAX_COMPRESSION_LEN = 4; /* diff -Nadur a/storage/innobase/innodb.cmake b/storage/innobase/innodb.cmake --- a/storage/innobase/innodb.cmake 2022-08-31 14:09:16.081464401 +0200 +++ b/storage/innobase/innodb.cmake 2022-08-31 12:34:43.515258059 +0200 @@ -34,6 +34,10 @@ ADD_DEFINITIONS(-DHAVE_ZSTD=1) ENDIF() +IF(LZMA_LIBRARY) + ADD_DEFINITIONS(-DHAVE_LZMA=1) +ENDIF() + #ADD_DEFINITIONS("-O0") # OS tests diff -Nadur a/storage/innobase/os/file.cc b/storage/innobase/os/file.cc --- a/storage/innobase/os/file.cc 2022-08-31 14:09:16.087464595 +0200 +++ b/storage/innobase/os/file.cc 2022-08-31 13:40:52.265436424 +0200 @@ -51,6 +51,7 @@ #include #include #include +#include /** Convert to a "string". @param[in] type The compression type @@ -65,6 +66,8 @@ return ("LZ4"); case ZSTD: return ("ZStd"); + case LZMA: + return ("LZMA"); } ut_d(ut_error); @@ -242,6 +245,17 @@ if (allocated) ut::free(dst); return (DB_IO_DECOMPRESS_FAIL); } + break; + } + + case Compression::LZMA: { + size_t src_pos = 0; + size_t dst_pos = 0; + uint64_t memlimit = UINT64_MAX; + if ((LZMA_OK != lzma_stream_buffer_decode(&memlimit, 0, NULL, ptr, &src_pos, header.m_compressed_size, dst, &dst_pos, header.m_original_size)) || (dst_pos != header.m_original_size)) { + if (allocated) ut::free(dst); + return (DB_IO_DECOMPRESS_FAIL); + } break; } diff -Nadur a/storage/innobase/os/file.h b/storage/innobase/os/file.h --- a/storage/innobase/os/file.h 2022-08-31 14:09:16.093464788 +0200 +++ b/storage/innobase/os/file.h 2022-08-31 12:36:10.253059424 +0200 @@ -65,6 +65,9 @@ /** Use ZStd */ ZSTD = 3, + + /** Use LZMA */ + LZMA = 4, }; /** Compressed page meta-data */ @@ -97,6 +100,7 @@ case ZLIB: case LZ4: case ZSTD: + case LZMA: break; default: ut_error; @@ -122,6 +126,9 @@ case ZSTD: os << "ZSTD"; break; + case LZMA: + os << "LZMA"; + break; default: os << ""; break; diff -Nadur a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc --- a/storage/innobase/os/os0file.cc 2022-08-31 14:09:16.100465015 +0200 +++ b/storage/innobase/os/os0file.cc 2022-08-31 13:40:44.872197646 +0200 @@ -97,6 +97,7 @@ #include #include +#include #ifdef UNIV_HOTBACKUP #include @@ -810,6 +811,7 @@ /** Default Zip compression level */ extern uint page_zip_level; extern uint page_zstd_level; +extern uint page_lzma_level; static_assert(DATA_TRX_ID_LEN <= 6, "COMPRESSION_ALGORITHM will not fit!"); @@ -1333,6 +1335,7 @@ ulint len = 0; ulint compression_level = page_zip_level; ulint zstd_compression_level = page_zstd_level; + ulint lzma_compression_level = page_lzma_level; ulint page_type = mach_read_from_2(src + FIL_PAGE_TYPE); /* The page size must be a multiple of the OS punch hole size. */ @@ -1419,6 +1422,16 @@ break; } + case Compression::LZMA: { + size_t out_pos = 0; + if ((LZMA_OK != lzma_easy_buffer_encode(lzma_compression_level, LZMA_CHECK_NONE, NULL, src + FIL_PAGE_DATA, content_len, dst + FIL_PAGE_DATA, &out_pos, out_len)) || (out_pos == 0)) { + *dst_len = src_len; + return (src); + } + len = out_pos; + break; + } + default: *dst_len = src_len; return (src); diff -Nadur a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc --- a/storage/innobase/page/page0zip.cc 2022-08-31 14:09:16.107465241 +0200 +++ b/storage/innobase/page/page0zip.cc 2022-08-31 12:35:33.937886558 +0200 @@ -85,6 +85,7 @@ /* Compression level to be used by zlib. Settable by user. */ uint page_zip_level = DEFAULT_COMPRESSION_LEVEL; uint page_zstd_level = DEFAULT_ZSTD_COMPRESSION_LEVEL; +uint page_lzma_level = DEFAULT_LZMA_COMPRESSION_LEVEL; /* Whether or not to log compressed page images to avoid possible compression algorithm changes in zlib. */ diff -Nadur a/utilities/CMakeLists.txt b/utilities/CMakeLists.txt --- a/utilities/CMakeLists.txt 2022-07-13 20:42:43.000000000 +0200 +++ b/utilities/CMakeLists.txt 2022-08-31 14:04:51.302912878 +0200 @@ -156,7 +156,7 @@ innochecksum.cc COMPONENT Server DEPENDENCIES GenError - LINK_LIBRARIES mysys innodb_zipdecompress ext::lz4 extra::rapidjson + LINK_LIBRARIES mysys innodb_zipdecompress ext::lz4 ${LZMA_LIBRARY} extra::rapidjson ) TARGET_COMPILE_DEFINITIONS(innochecksum PRIVATE UNIV_NO_ERR_MSGS UNIV_LIBRARY) IF(MY_COMPILER_IS_GNU_OR_CLANG)