From e19202f710814ac4fbaae946575f93a2842bc7e9 Mon Sep 17 00:00:00 2001 From: Davi Arnaut Date: Tue, 20 Oct 2015 16:11:30 -0700 Subject: [PATCH] Compression of row-based events In row-based-replication, row events usually contain full copies of the rows being modified, and in some cases even semi-duplicated copies if the rows are being updated. Although it is possible to omit columns which are not being changed, the full image of changed columns is still logged no matter how much of the data changed. If only a small portion of the data in a column is changed, much space might be wasted. In order to reduce the size of such row events, this change introduces the ability to compress the row data part of row events. Row events have the same format as before, only that the row images are stored in a compressed format. An entry is added to the extra information area to note when a row event is compressed. The binlog_row_compression option can be used to control row compression. The option is of type enumeration and the permitted values are NONE and SNAPPY. NONE indicates no compression while SNAPPY enables compression using the Snappy algorithm. The binlog_row_compression_threshold option can be used to control the size threshold in bytes beyond which compression is attempted. That is, if the size of the rows in an event is below the specified threshold, compression is not attempted. --- CMakeLists.txt | 2 + client/CMakeLists.txt | 2 +- cmake/FindSnappy.cmake | 54 +++ libmysqld/CMakeLists.txt | 1 + mysql-test/extra/rpl_tests/rpl_row_img_blobs.test | 9 +- .../suite/rpl/r/rpl_row_compression_stress.result | 475 +++++++++++++++++++++ .../suite/rpl/t/rpl_row_compression_stress.cnf | 1 + .../suite/rpl/t/rpl_row_compression_stress.test | 42 ++ sql/CMakeLists.txt | 2 +- sql/log_event.cc | 153 +++++++ sql/log_event.h | 20 + sql/mysqld.cc | 6 + sql/rpl_constants.h | 2 + sql/sys_vars.cc | 17 +- 14 files changed, 780 insertions(+), 6 deletions(-) create mode 100644 cmake/FindSnappy.cmake create mode 100644 mysql-test/suite/rpl/r/rpl_row_compression_stress.result create mode 100644 mysql-test/suite/rpl/t/rpl_row_compression_stress.cnf create mode 100644 mysql-test/suite/rpl/t/rpl_row_compression_stress.test diff --git a/CMakeLists.txt b/CMakeLists.txt index e5f851b..1f09ccc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,6 +195,8 @@ INCLUDE(install_macros) INCLUDE(install_layout) INCLUDE(mysql_add_executable) +FIND_PACKAGE(Snappy) + # Handle options OPTION(DISABLE_SHARED "Don't build shared libraries, compile code as position-dependent" OFF) diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index d24555a..ababd79 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -63,7 +63,7 @@ MYSQL_ADD_EXECUTABLE(mysql_plugin mysql_plugin.c) TARGET_LINK_LIBRARIES(mysql_plugin mysqlclient) MYSQL_ADD_EXECUTABLE(mysqlbinlog mysqlbinlog.cc) -TARGET_LINK_LIBRARIES(mysqlbinlog mysqlclient) +TARGET_LINK_LIBRARIES(mysqlbinlog mysqlclient ${SNAPPY_LIBRARIES}) MYSQL_ADD_EXECUTABLE(mysqladmin mysqladmin.cc) TARGET_LINK_LIBRARIES(mysqladmin mysqlclient) diff --git a/cmake/FindSnappy.cmake b/cmake/FindSnappy.cmake new file mode 100644 index 0000000..e9053af --- /dev/null +++ b/cmake/FindSnappy.cmake @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Tries to find Snappy headers and libraries. +# +# Usage of this module as follows: +# +# find_package(Snappy) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# SNAPPY_ROOT_DIR Set this variable to the root installation of +# Snappy if the module has problems finding +# the proper installation path. +# +# Variables defined by this module: +# +# SNAPPY_FOUND System has Snappy libs/headers +# SNAPPY_LIBRARIES The Snappy libraries +# SNAPPY_INCLUDE_DIR The location of Snappy headers + +find_path(SNAPPY_INCLUDE_DIR + NAMES snappy.h + HINTS ${SNAPPY_ROOT_DIR}/include) + +find_library(SNAPPY_LIBRARIES + NAMES snappy + HINTS ${SNAPPY_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Snappy DEFAULT_MSG + SNAPPY_LIBRARIES + SNAPPY_INCLUDE_DIR) + +mark_as_advanced( + SNAPPY_ROOT_DIR + SNAPPY_LIBRARIES + SNAPPY_INCLUDE_DIR) diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 150ffe1..3f80e30 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -98,6 +98,7 @@ SET(LIBS dbug strings regex mysys mysys_ssl vio ${ZLIB_LIBRARY} ${SSL_LIBRARIES} ${LIBWRAP} ${LIBCRYPT} ${LIBDL} + ${SNAPPY_LIBRARIES} ${MYSQLD_STATIC_PLUGIN_LIBS} sql_embedded ) diff --git a/mysql-test/extra/rpl_tests/rpl_row_img_blobs.test b/mysql-test/extra/rpl_tests/rpl_row_img_blobs.test index 9d3a0a5..6d0fdab 100644 --- a/mysql-test/extra/rpl_tests/rpl_row_img_blobs.test +++ b/mysql-test/extra/rpl_tests/rpl_row_img_blobs.test @@ -138,9 +138,12 @@ while($i) -- connection server_1 - -- let $blob1= "a" - -- let $blob2= "b" - -- let $blob3= "c" + if (!$blob1) + { + -- let $blob1= "a" + -- let $blob2= "b" + -- let $blob3= "c" + } -- eval INSERT INTO t VALUES (1, $blob1, 10) -- eval INSERT INTO t VALUES (2, $blob2, 20) diff --git a/mysql-test/suite/rpl/r/rpl_row_compression_stress.result b/mysql-test/suite/rpl/r/rpl_row_compression_stress.result new file mode 100644 index 0000000..6bb0405 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_row_compression_stress.result @@ -0,0 +1,475 @@ +include/rpl_init.inc [topology=1->2->3] +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +SET GLOBAL binlog_row_compression='snappy'; +SET GLOBAL binlog_row_compression='snappy'; +SET GLOBAL binlog_row_compression='snappy'; +CON: 'server_1', IMG: 'FULL', RESTART SLAVE: 'N' +SET SESSION binlog_row_image= 'FULL'; +SET GLOBAL binlog_row_image= 'FULL'; +FLUSH TABLES; +SHOW VARIABLES LIKE 'binlog_row_image'; +Variable_name Value +binlog_row_image FULL +CON: 'server_2', IMG: 'FULL', RESTART SLAVE: 'Y' +SET SESSION binlog_row_image= 'FULL'; +SET GLOBAL binlog_row_image= 'FULL'; +include/stop_slave.inc +include/start_slave.inc +FLUSH TABLES; +SHOW VARIABLES LIKE 'binlog_row_image'; +Variable_name Value +binlog_row_image FULL +CON: 'server_3', IMG: 'FULL', RESTART SLAVE: 'Y' +SET SESSION binlog_row_image= 'FULL'; +SET GLOBAL binlog_row_image= 'FULL'; +include/stop_slave.inc +include/start_slave.inc +FLUSH TABLES; +SHOW VARIABLES LIKE 'binlog_row_image'; +Variable_name Value +binlog_row_image FULL +### engines: MyISAM, MyISAM, MyISAM +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check when there is no key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check even if there is a key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (UK NOT NULL exists in the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (PK exists int the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob in a key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a unique (not null) key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a primary key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### engines: MyISAM, MyISAM, InnoDB +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check when there is no key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check even if there is a key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (UK NOT NULL exists in the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (PK exists int the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob in a key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a unique (not null) key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a primary key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### engines: MyISAM, InnoDB, MyISAM +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check when there is no key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check even if there is a key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (UK NOT NULL exists in the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (PK exists int the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob in a key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a unique (not null) key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a primary key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### engines: MyISAM, InnoDB, InnoDB +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check when there is no key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check even if there is a key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (UK NOT NULL exists in the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (PK exists int the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob in a key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a unique (not null) key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a primary key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### engines: InnoDB, MyISAM, MyISAM +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check when there is no key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check even if there is a key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (UK NOT NULL exists in the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (PK exists int the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob in a key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a unique (not null) key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a primary key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### engines: InnoDB, MyISAM, InnoDB +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check when there is no key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check even if there is a key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (UK NOT NULL exists in the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (PK exists int the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob in a key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a unique (not null) key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a primary key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### engines: InnoDB, InnoDB, MyISAM +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check when there is no key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check even if there is a key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (UK NOT NULL exists in the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (PK exists int the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob in a key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a unique (not null) key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a primary key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### engines: InnoDB, InnoDB, InnoDB +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check when there is no key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the AI (they are not updated) +### will not break replication (check even if there is a key in the table) +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (UK NOT NULL exists in the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that updates without blobs in the BI (PK exists int the table) +### will not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob in a key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a unique (not null) key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +### Asserts that declaring a blob as part of a primary key does not break replication +include/rpl_sync.inc +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +include/diff_tables.inc [server_1:test.t, server_2:test.t, server_3:test.t] +include/rpl_sync.inc +SET GLOBAL binlog_row_compression='none'; +SET GLOBAL binlog_row_compression='none'; +SET GLOBAL binlog_row_compression='none'; +SELECT (VARIABLE_VALUE+0) > 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE +VARIABLE_NAME = 'Binlog_row_compression_succeed'; +(VARIABLE_VALUE+0) > 1 +1 +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_row_compression_stress.cnf b/mysql-test/suite/rpl/t/rpl_row_compression_stress.cnf new file mode 100644 index 0000000..d758d29 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_compression_stress.cnf @@ -0,0 +1 @@ +!include suite/rpl/t/rpl_row_img.cnf diff --git a/mysql-test/suite/rpl/t/rpl_row_compression_stress.test b/mysql-test/suite/rpl/t/rpl_row_compression_stress.test new file mode 100644 index 0000000..0e5017f --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_compression_stress.test @@ -0,0 +1,42 @@ +--let $rpl_topology= 1->2->3 +--source include/rpl_init.inc +--source include/have_binlog_format_row.inc + +--connection server_1 +--source include/have_innodb.inc +SET GLOBAL binlog_row_compression='snappy'; +--connection server_2 +--source include/have_innodb.inc +SET GLOBAL binlog_row_compression='snappy'; +--connection server_3 +--source include/have_innodb.inc +SET GLOBAL binlog_row_compression='snappy'; +--connection server_1 + +--let $blob1= "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +--let $blob2= "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +--let $blob3= "cccccccccccccccccccccccccccccccccccccccccccccccc" + +--let $row_img_set=server_1:FULL:N,server_2:FULL:Y,server_3:FULL:Y +--source include/rpl_row_img_set.inc + +--disable_query_log +--let $row_img_test_script= extra/rpl_tests/rpl_row_img_blobs.test +--source include/rpl_row_img_general_loop.inc +--enable_query_log + +--connection server_1 +--source include/have_innodb.inc +SET GLOBAL binlog_row_compression='none'; +--connection server_2 +--source include/have_innodb.inc +SET GLOBAL binlog_row_compression='none'; +--connection server_3 +--source include/have_innodb.inc +SET GLOBAL binlog_row_compression='none'; + +--connection server_1 +SELECT (VARIABLE_VALUE+0) > 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE + VARIABLE_NAME = 'Binlog_row_compression_succeed'; + +--source include/rpl_end.inc diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index a19174d..c037240 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -227,7 +227,7 @@ DTRACE_INSTRUMENT(sql) TARGET_LINK_LIBRARIES(sql ${MYSQLD_STATIC_PLUGIN_LIBS} mysys mysys_ssl dbug strings vio regex ${LIBWRAP} ${LIBCRYPT} ${LIBDL} - ${SSL_LIBRARIES}) + ${SSL_LIBRARIES} ${SNAPPY_LIBRARIES}) # # On Windows platform we compile in the clinet-side Windows Native Authentication diff --git a/sql/log_event.cc b/sql/log_event.cc index 4cbaa3d..fd2164b 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -60,6 +60,8 @@ slave_ignored_err_throttle(window_size, #include "sql_digest.h" +#include + using std::min; using std::max; @@ -9558,6 +9560,14 @@ Rows_log_event::Rows_log_event(const char *buf, uint event_len, DBUG_PRINT("info",("m_table_id: %llu m_flags: %d m_width: %lu data_size: %lu", m_table_id.id(), m_flags, m_width, (ulong) data_size)); + // Uncompress row data if compressed + uchar comp_format= COMPRESSION_LAST; + if (row_data_is_compressed(&comp_format)) + { + uncompress_row_data(comp_format, ptr_rows_data, data_size); + DBUG_VOID_RETURN; + } + // Allocate one extra byte, in case we have to do uint3korr! m_rows_buf= (uchar*) my_malloc(data_size + 1, MYF(MY_WME)); // didrik if (likely((bool)m_rows_buf)) @@ -11657,6 +11667,149 @@ Rows_log_event::do_update_pos(Relay_log_info *rli) #endif /* !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) */ +#if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) + +bool Rows_log_event::write(IO_CACHE *file) +{ + enum enum_row_compression compression; + if (row_data_is_compressible(&compression)) + compress_row_data(compression); + return Log_event::write(file); +} + +bool Rows_log_event::row_data_is_compressible(enum enum_row_compression *compression) const +{ + // Extra info area is used for tagging whether the row data is compressed + if (m_extra_row_data != NULL || log_bin_use_v1_row_events != 0) + return false; + + const size_t rows_buf_len= m_rows_cur - m_rows_buf; + extern ulong opt_binlog_row_compression_threshold; + if (rows_buf_len < opt_binlog_row_compression_threshold) + return false; + + extern ulong opt_binlog_row_compression; + *compression= (enum enum_row_compression) opt_binlog_row_compression; + return *compression != COMPRESSION_NONE; +} + +extern ulong binlog_row_compression_failed, binlog_row_compression_succeed; + +void Rows_log_event::compress_row_data(enum enum_row_compression format) +{ + bool status= false; + if (format == COMPRESSION_SNAPPY) + status= snappy_compress_row_data(); + if (status) + binlog_row_compression_succeed++; + else + binlog_row_compression_failed++; +} + +bool Rows_log_event::snappy_compress_row_data(void) +{ + const size_t rows_buf_len= m_rows_cur - m_rows_buf; + size_t compressed_length= snappy_max_compressed_length(rows_buf_len); + char *compressed_buffer= (char*) my_malloc(compressed_length, MYF(MY_WME)); + if (compressed_buffer == NULL) + return false; + + snappy_status status= snappy_compress((char*) m_rows_buf, + rows_buf_len, compressed_buffer, &compressed_length); + if (status != SNAPPY_OK || compressed_length >= rows_buf_len) + { + my_free(compressed_buffer); + return false; + } + + uint8 extra_data_len= EXTRA_ROW_INFO_HDR_BYTES + 1; + m_extra_row_data= (uchar*) my_malloc(extra_data_len, MYF(MY_WME)); + if (m_extra_row_data == NULL) + { + my_free(compressed_buffer); + return false; + } + + my_free(m_rows_buf); + m_rows_buf= (uchar *) compressed_buffer; + m_rows_cur= m_rows_end= m_rows_buf + compressed_length; + + // Note the compression method + m_extra_row_data[EXTRA_ROW_INFO_LEN_OFFSET]= extra_data_len; + m_extra_row_data[EXTRA_ROW_INFO_FORMAT_OFFSET]= ERIF_COMPRESSION; + m_extra_row_data[EXTRA_ROW_INFO_HDR_BYTES]= COMPRESSION_SNAPPY; + + extern ulonglong binlog_row_compression_comp, binlog_row_compression_uncomp; + binlog_row_compression_uncomp+= rows_buf_len; + binlog_row_compression_comp+= (extra_data_len + compressed_length); + + return true; +} + +#endif + +bool Rows_log_event::row_data_is_compressed(uchar *compression) const +{ + uchar *extra= m_extra_row_data; + DBUG_EXECUTE_IF("extra_row_data_check", extra= NULL;); + if (extra == NULL) + return false; + else if (extra[EXTRA_ROW_INFO_FORMAT_OFFSET] != ERIF_COMPRESSION) + return false; + else if (extra[EXTRA_ROW_INFO_LEN_OFFSET] != (EXTRA_ROW_INFO_HDR_BYTES + 1)) + return true; + *compression= extra[EXTRA_ROW_INFO_HDR_BYTES]; + return true; +} + +void Rows_log_event::uncompress_row_data(uchar compressed_format, + const uchar *compressed_buf, size_t compressed_length) +{ + bool status= false; + + if (compressed_format == COMPRESSION_SNAPPY) + status= snappy_uncompress_row_data(compressed_buf, compressed_length); + + if (compressed_format >= COMPRESSION_LAST) + sql_print_error("Invalid compression format %u for %s event at position %llu", + compressed_format, get_type_str(), (ulonglong) log_pos); + else if (!status) + sql_print_error("Failed to decompress %s event at position %llu", + get_type_str(), (ulonglong) log_pos); +} + +bool Rows_log_event::snappy_uncompress_row_data(const uchar *compressed_buf, + size_t compressed_length) +{ + size_t uncompressed_length; + const char *compressed= (const char *) compressed_buf; + snappy_status status= snappy_uncompressed_length(compressed, + compressed_length, &uncompressed_length); + if (status != SNAPPY_OK) + return false; + + char *uncompressed= (char*) my_malloc(uncompressed_length + 1, MYF(MY_WME)); + if (uncompressed == NULL) + return false; + + status= snappy_uncompress(compressed, compressed_length, + uncompressed, &uncompressed_length); + if (status != SNAPPY_OK) + { + my_free(uncompressed); + return false; + } + + m_rows_buf= reinterpret_cast(uncompressed); +#if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) + m_curr_row= m_rows_buf; +#endif + m_rows_end= m_rows_buf + uncompressed_length; + m_rows_cur= m_rows_end; + + return true; +} + #ifndef MYSQL_CLIENT bool Rows_log_event::write_data_header(IO_CACHE *file) { diff --git a/sql/log_event.h b/sql/log_event.h index e179282..86dd382 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -4144,6 +4144,26 @@ public: } #endif + /** + Enumeration of the row compression methods. + */ + enum enum_row_compression + { + COMPRESSION_NONE= 0, + COMPRESSION_SNAPPY= 1, + COMPRESSION_LAST= 2, + }; + +#if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) + virtual bool write(IO_CACHE *file); + bool row_data_is_compressible(enum enum_row_compression *) const; + void compress_row_data(enum enum_row_compression); + bool snappy_compress_row_data(void); +#endif + bool row_data_is_compressed(uchar *) const; + void uncompress_row_data(uchar, const uchar *, size_t); + bool snappy_uncompress_row_data(const uchar *, size_t); + #ifdef MYSQL_SERVER virtual bool write_data_header(IO_CACHE *file); virtual bool write_data_body(IO_CACHE *file); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index d401f83..76665a8 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7899,6 +7899,8 @@ show_ssl_get_server_not_after(THD *thd, SHOW_VAR *var, char *buff) #endif /* HAVE_OPENSSL && !EMBEDDED_LIBRARY */ +ulong binlog_row_compression_failed= 0, binlog_row_compression_succeed= 0; +ulonglong binlog_row_compression_comp= 0, binlog_row_compression_uncomp= 0; /* Variables shown by SHOW STATUS in alphabetical order @@ -7909,6 +7911,10 @@ SHOW_VAR status_vars[]= { {"Aborted_connects", (char*) &aborted_connects, SHOW_LONG}, {"Binlog_cache_disk_use", (char*) &binlog_cache_disk_use, SHOW_LONG}, {"Binlog_cache_use", (char*) &binlog_cache_use, SHOW_LONG}, + {"Binlog_row_compression_failed", (char*) &binlog_row_compression_failed, SHOW_LONG}, + {"Binlog_row_compression_succeed", (char*) &binlog_row_compression_succeed, SHOW_LONG}, + {"Binlog_row_compressed_bytes", (char*) &binlog_row_compression_comp, SHOW_LONGLONG}, + {"Binlog_row_uncompressed_bytes", (char*) &binlog_row_compression_uncomp, SHOW_LONGLONG}, {"Binlog_stmt_cache_disk_use",(char*) &binlog_stmt_cache_disk_use, SHOW_LONG}, {"Binlog_stmt_cache_use", (char*) &binlog_stmt_cache_use, SHOW_LONG}, {"Bytes_received", (char*) offsetof(STATUS_VAR, bytes_received), SHOW_LONGLONG_STATUS}, diff --git a/sql/rpl_constants.h b/sql/rpl_constants.h index e963388..2c97d4b 100644 --- a/sql/rpl_constants.h +++ b/sql/rpl_constants.h @@ -64,6 +64,8 @@ enum ExtraRowInfoFormat { ERIF_OPEN1 = 64, ERIF_OPEN2 = 65, + ERIF_COMPRESSION = 100, + ERIF_LASTOPEN = 254, /** diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 1fe601e..92c9eb4 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -4707,8 +4707,23 @@ static Sys_var_gtid_last Sys_gtid_last( "gtid_last", "GTID of the last transaction committed in the current connection."); -#endif // HAVE_REPLICATION +static const char *binlog_row_compression_names[]= {"NONE", "SNAPPY", 0}; +ulong opt_binlog_row_compression= Rows_log_event::COMPRESSION_NONE; +static Sys_var_enum Sys_binlog_row_compression( + "binlog_row_compression", + "Whether binary logged rows are compressed. NONE or SNAPPY.", + GLOBAL_VAR(opt_binlog_row_compression), CMD_LINE(OPT_ARG), + binlog_row_compression_names, + DEFAULT(Rows_log_event::COMPRESSION_NONE)); + +ulong opt_binlog_row_compression_threshold= 64; +static Sys_var_ulong Sys_binlog_row_compression_threshold( + "binlog_row_compression_threshold", + "Size threshold in bytes beyond which compression is attempted.", + GLOBAL_VAR(opt_binlog_row_compression_threshold), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(1, ULONG_MAX), DEFAULT(64), BLOCK_SIZE(1)); +#endif // HAVE_REPLICATION static Sys_var_mybool Sys_disconnect_on_expired_password( "disconnect_on_expired_password", -- 1.8.5.6