diff --git a/include/mysql/components/services/clone_protocol_service.h b/include/mysql/components/services/clone_protocol_service.h index 827f8f66fdf..b6bc691c58e 100644 --- a/include/mysql/components/services/clone_protocol_service.h +++ b/include/mysql/components/services/clone_protocol_service.h @@ -59,9 +59,11 @@ struct mysql_clone_ssl_context { const char *m_ssl_cert; /** Clone ssl certificate authority. Same as mysql client --ssl-ca */ const char *m_ssl_ca; - - /** Enable network compression. */ + /** Enable legacy network compression. */ bool m_enable_compression; + /** Network compression. */ + const char *m_compression_algorithms; + uint m_zstd_compression_level; NET_SERVER *m_server_extn; }; diff --git a/mysql-test/suite/clone/r/local_patch.result b/mysql-test/suite/clone/r/local_patch.result new file mode 100644 index 00000000000..8e456f89151 --- /dev/null +++ b/mysql-test/suite/clone/r/local_patch.result @@ -0,0 +1,54 @@ +CREATE TABLE t1(col1 INT AUTO_INCREMENT PRIMARY KEY, col2 char(64), FULLTEXT KEY fts_index(col2)) Engine=InnoDB; +INSERT INTO t1(col2) VALUES('clone row 1'); +INSERT INTO t1(col2) VALUES('clone row 2'); +INSERT INTO t1(col2) VALUES('clone row 3'); +SELECT * from t1 ORDER BY col1; +col1 col2 +1 clone row 1 +2 clone row 2 +3 clone row 3 +SELECT count(*) FROM mysql.general_log; +count(*) +0 +SELECT count(*) FROM mysql.slow_log; +count(*) +0 +show variables like 'protocol_compression_algorithms'; +Variable_name Value +protocol_compression_algorithms zlib,zstd,uncompressed +INSTALL PLUGIN clone SONAME 'CLONE_PLUGIN'; +SET GLOBAL clone_compression_algorithms='zstd'; +SET GLOBAL clone_zstd_compression_level=8; +SET GLOBAL clone_autotune_concurrency = OFF; +SET GLOBAL clone_max_concurrency = 8; +CLONE LOCAL DATA DIRECTORY = 'CLONE_DATADIR'; +select ID, STATE, ERROR_NO from performance_schema.clone_status; +ID STATE ERROR_NO +1 Completed 0 +select ID, STAGE, STATE from performance_schema.clone_progress; +ID STAGE STATE +1 DROP DATA Completed +1 FILE COPY Completed +1 PAGE COPY Completed +1 REDO COPY Completed +1 FILE SYNC Completed +1 RESTART Not Started +1 RECOVERY Not Started +# restart: --datadir=CLONE_DATADIR +INSERT INTO t1(col2) VALUES('clone row 4'); +SELECT * from t1 ORDER BY col1; +col1 col2 +1 clone row 1 +2 clone row 2 +3 clone row 3 +4 clone row 4 +SET GLOBAL general_log = ON; +SET GLOBAL slow_query_log = ON; +# restart: +SELECT * from t1 ORDER BY col1; +col1 col2 +1 clone row 1 +2 clone row 2 +3 clone row 3 +DROP TABLE t1; +UNINSTALL PLUGIN clone; diff --git a/mysql-test/suite/clone/r/remote_patch.result b/mysql-test/suite/clone/r/remote_patch.result new file mode 100644 index 00000000000..aa1393977f8 --- /dev/null +++ b/mysql-test/suite/clone/r/remote_patch.result @@ -0,0 +1,55 @@ +CREATE TABLE t1(col1 INT AUTO_INCREMENT PRIMARY KEY, col2 char(64), FULLTEXT KEY fts_index(col2)) Engine=InnoDB; +INSERT INTO t1(col2) VALUES('clone row 1'); +INSERT INTO t1(col2) VALUES('clone row 2'); +INSERT INTO t1(col2) VALUES('clone row 3'); +SELECT * from t1 ORDER BY col1; +col1 col2 +1 clone row 1 +2 clone row 2 +3 clone row 3 +SELECT count(*) FROM mysql.general_log; +count(*) +0 +SELECT count(*) FROM mysql.slow_log; +count(*) +0 +show variables like 'protocol_compression_algorithms'; +Variable_name Value +protocol_compression_algorithms zlib,zstd,uncompressed +INSTALL PLUGIN clone SONAME 'CLONE_PLUGIN'; +SET GLOBAL clone_compression_algorithms='zstd'; +SET GLOBAL clone_zstd_compression_level=8; +SET GLOBAL clone_autotune_concurrency = OFF; +SET GLOBAL clone_max_concurrency = 8; +SET GLOBAL clone_valid_donor_list = 'HOST:PORT'; +CLONE INSTANCE FROM USER@HOST:PORT IDENTIFIED BY '' DATA DIRECTORY = 'CLONE_DATADIR'; +select ID, STATE, ERROR_NO from performance_schema.clone_status; +ID STATE ERROR_NO +1 Completed 0 +select ID, STAGE, STATE from performance_schema.clone_progress; +ID STAGE STATE +1 DROP DATA Completed +1 FILE COPY Completed +1 PAGE COPY Completed +1 REDO COPY Completed +1 FILE SYNC Completed +1 RESTART Not Started +1 RECOVERY Not Started +# restart: --datadir=CLONE_DATADIR +INSERT INTO t1(col2) VALUES('clone row 4'); +SELECT * from t1 ORDER BY col1; +col1 col2 +1 clone row 1 +2 clone row 2 +3 clone row 3 +4 clone row 4 +SET GLOBAL general_log = ON; +SET GLOBAL slow_query_log = ON; +# restart: +SELECT * from t1 ORDER BY col1; +col1 col2 +1 clone row 1 +2 clone row 2 +3 clone row 3 +DROP TABLE t1; +UNINSTALL PLUGIN clone; diff --git a/mysql-test/suite/clone/r/sys_var.result b/mysql-test/suite/clone/r/sys_var.result new file mode 100644 index 00000000000..ac2434f793b --- /dev/null +++ b/mysql-test/suite/clone/r/sys_var.result @@ -0,0 +1,45 @@ +INSTALL PLUGIN clone SONAME 'CLONE_PLUGIN'; +SELECT @@clone_zstd_compression_level; +@@clone_zstd_compression_level +3 +SET GLOBAL clone_zstd_compression_level=5; +SELECT @@clone_zstd_compression_level; +@@clone_zstd_compression_level +5 +SET GLOBAL clone_zstd_compression_level=30; +ERROR 42000: Variable 'clone_zstd_compression_level' can't be set to the value of '30' +SELECT @@clone_zstd_compression_level; +@@clone_zstd_compression_level +5 +SET GLOBAL clone_zstd_compression_level=-1; +ERROR 42000: Variable 'clone_zstd_compression_level' can't be set to the value of '-1' +SELECT @@clone_zstd_compression_level; +@@clone_zstd_compression_level +5 +SELECT @@clone_compression_algorithms; +@@clone_compression_algorithms +uncompressed +SET GLOBAL clone_compression_algorithms='zstd'; +SELECT @@clone_compression_algorithms; +@@clone_compression_algorithms +zstd +SET GLOBAL clone_compression_algorithms='zstd,zlib'; +SELECT @@clone_compression_algorithms; +@@clone_compression_algorithms +zstd,zlib +SET GLOBAL clone_compression_algorithms='unknown'; +ERROR 42000: Variable 'clone_compression_algorithms' can't be set to the value of 'unknown' +SELECT @@clone_compression_algorithms; +@@clone_compression_algorithms +zstd,zlib +SET GLOBAL clone_compression_algorithms='unknown,zstd'; +ERROR 42000: Variable 'clone_compression_algorithms' can't be set to the value of 'unknown,zstd' +SELECT @@clone_compression_algorithms; +@@clone_compression_algorithms +zstd,zlib +SET GLOBAL clone_compression_algorithms='zstd,unknown'; +ERROR 42000: Variable 'clone_compression_algorithms' can't be set to the value of 'zstd,unknown' +SELECT @@clone_compression_algorithms; +@@clone_compression_algorithms +zstd,zlib +UNINSTALL PLUGIN clone; diff --git a/mysql-test/suite/clone/t/local_patch.test b/mysql-test/suite/clone/t/local_patch.test new file mode 100644 index 00000000000..f02fd10455a --- /dev/null +++ b/mysql-test/suite/clone/t/local_patch.test @@ -0,0 +1,79 @@ +# Test clone command + +--source include/have_example_plugin.inc +--source include/have_log_bin.inc +--source ../include/clone_connection_begin.inc +--let $CLONE_DATADIR = $MYSQL_TMP_DIR/data_new + +--disable_query_log +call mtr.add_suppression("\\[Warning\\] .*MY-\\d.* Non innodb table: .* is not cloned and is empty."); +call mtr.add_suppression("\\[Warning\\] .*MY-\\d.* Clone removing all user data for provisioning: .*"); +--enable_query_log + +CREATE TABLE t1(col1 INT AUTO_INCREMENT PRIMARY KEY, col2 char(64), FULLTEXT KEY fts_index(col2)) Engine=InnoDB; + +INSERT INTO t1(col2) VALUES('clone row 1'); +INSERT INTO t1(col2) VALUES('clone row 2'); +INSERT INTO t1(col2) VALUES('clone row 3'); + +SELECT * from t1 ORDER BY col1; + +SELECT count(*) FROM mysql.general_log; +SELECT count(*) FROM mysql.slow_log; + +# Check Server Compression Algorithm +show variables like 'protocol_compression_algorithms'; + +# Install Clone Plugin +--replace_result $CLONE_PLUGIN CLONE_PLUGIN +--eval INSTALL PLUGIN clone SONAME '$CLONE_PLUGIN' + +# Clone data +--connection clone_conn_1 +SET GLOBAL clone_compression_algorithms='zstd'; +SET GLOBAL clone_zstd_compression_level=8; +--source ../include/clone_command.inc + +if (!$clone_remote_replace) { + --connection default + # Restart server on cloned data directory + --replace_result $CLONE_DATADIR CLONE_DATADIR + --let restart_parameters="restart: --datadir=$CLONE_DATADIR" + --source include/restart_mysqld.inc +} + +# Validate data +INSERT INTO t1(col2) VALUES('clone row 4'); +SELECT * from t1 ORDER BY col1; + +SET GLOBAL general_log = ON; +SET GLOBAL slow_query_log = ON; + +#Cleanup +if (!$clone_remote_replace) { + --let restart_parameters="restart:" + --source include/restart_mysqld.inc +} +--connection default + +SELECT * from t1 ORDER BY col1; + +DROP TABLE t1; + +if (!$clone_remote_replace) { + --force-rmdir $CLONE_DATADIR +} + +# Clean recipient by cloning the cleaned up donor +if ($clone_remote_replace) { + --connection clone_conn_1 + --let $clone_err = ER_SCHEMA_READ_ONLY + --source ../include/clone_command.inc + --let $clone_err = 0 + + ALTER SCHEMA schema_read_only READ ONLY=0; + --source ../include/clone_command.inc +} +--source ../include/clone_connection_end.inc + +UNINSTALL PLUGIN clone; diff --git a/mysql-test/suite/clone/t/remote_patch.test b/mysql-test/suite/clone/t/remote_patch.test new file mode 100644 index 00000000000..43d3d112262 --- /dev/null +++ b/mysql-test/suite/clone/t/remote_patch.test @@ -0,0 +1,8 @@ +# Test clone remote command + +--let $HOST = 127.0.0.1 +--let $PORT =`select @@port` +--let $USER = root +--let remote_clone = 1 + +--source local_patch.test diff --git a/mysql-test/suite/clone/t/sys_var.test b/mysql-test/suite/clone/t/sys_var.test new file mode 100644 index 00000000000..10a64eda34b --- /dev/null +++ b/mysql-test/suite/clone/t/sys_var.test @@ -0,0 +1,52 @@ +# Test clone plugin system variables + +--source include/have_example_plugin.inc +--source include/have_log_bin.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc + +--disable_query_log +call mtr.add_suppression("\\[Warning\\] .*MY-\\d.* Non innodb table: .* is not cloned and is empty."); +call mtr.add_suppression("\\[Warning\\] .*MY-\\d.* Clone removing all user data for provisioning: .*"); +--enable_query_log + +# Install Clone Plugin +--replace_result $CLONE_PLUGIN CLONE_PLUGIN +--eval INSTALL PLUGIN clone SONAME '$CLONE_PLUGIN' + +SELECT @@clone_zstd_compression_level; + +SET GLOBAL clone_zstd_compression_level=5; +SELECT @@clone_zstd_compression_level; + +--error 1231 +SET GLOBAL clone_zstd_compression_level=30; +SELECT @@clone_zstd_compression_level; + +--error 1231 +SET GLOBAL clone_zstd_compression_level=-1; +SELECT @@clone_zstd_compression_level; + +SELECT @@clone_compression_algorithms; + +SET GLOBAL clone_compression_algorithms='zstd'; +SELECT @@clone_compression_algorithms; + +SET GLOBAL clone_compression_algorithms='zstd,zlib'; +SELECT @@clone_compression_algorithms; + +--error 1231 +SET GLOBAL clone_compression_algorithms='unknown'; +SELECT @@clone_compression_algorithms; + +--error 1231 +SET GLOBAL clone_compression_algorithms='unknown,zstd'; +SELECT @@clone_compression_algorithms; + +--error 1231 +SET GLOBAL clone_compression_algorithms='zstd,unknown'; +SELECT @@clone_compression_algorithms; + +# Clean up +UNINSTALL PLUGIN clone; + diff --git a/plugin/clone/include/clone.h b/plugin/clone/include/clone.h index c39746d197c..990e6f5be3f 100644 --- a/plugin/clone/include/clone.h +++ b/plugin/clone/include/clone.h @@ -88,9 +88,15 @@ extern uint clone_max_network_bandwidth; /** Clone system variable: Maximum IO bandwidth in MiB/sec */ extern uint clone_max_io_bandwidth; -/** Clone system variable: If network compression is enabled */ +/** Clone system variable: If legacy network compression is enabled */ extern bool clone_enable_compression; +/** Clone system variable: If network compression is enabled */ +extern char *clone_compression_algorithms; + +/** Clone system variable: If network compression with zstd is enabled */ +extern uint clone_zstd_compression_level; + /** Clone system variable: SSL private key */ extern char *clone_client_ssl_private_key; diff --git a/plugin/clone/src/clone_client.cc b/plugin/clone/src/clone_client.cc index 021a3c9478c..13a49950ce1 100644 --- a/plugin/clone/src/clone_client.cc +++ b/plugin/clone/src/clone_client.cc @@ -891,8 +891,9 @@ int Client::connect_remote(bool is_restart, bool use_aux) { mysql_clone_ssl_context ssl_context; ssl_context.m_enable_compression = clone_enable_compression; - ssl_context.m_server_extn = - ssl_context.m_enable_compression ? &m_conn_server_extn : nullptr; + ssl_context.m_compression_algorithms = clone_compression_algorithms; + ssl_context.m_zstd_compression_level = clone_zstd_compression_level; + ssl_context.m_server_extn = &m_conn_server_extn; ssl_context.m_ssl_mode = m_share->m_ssl_mode; /* Get Clone SSL configuration parameter value safely. */ diff --git a/plugin/clone/src/clone_plugin.cc b/plugin/clone/src/clone_plugin.cc index 6b230c24e54..66327567d55 100644 --- a/plugin/clone/src/clone_plugin.cc +++ b/plugin/clone/src/clone_plugin.cc @@ -68,6 +68,12 @@ uint clone_max_io_bandwidth; /** Clone system variable: If network compression is enabled */ bool clone_enable_compression; +/** Clone system variable: If network compression is enabled */ +char *clone_compression_algorithms; + +/** Clone system variable: If network compression with zstd is enabled */ +uint clone_zstd_compression_level; + /** Clone system variable: valid list of donor addresses. */ static char *clone_valid_donor_list; @@ -332,6 +338,66 @@ static int check_donor_addr_format(MYSQL_THD thd, SYS_VAR *var [[maybe_unused]], return (0); } +/** Check clone compression algorithm +@param[in] thd user session THD +@param[in] var system variable +@param[out] save possibly updated variable value +@param[in] value current variable value +@return error code + */ +static int check_compression_algorithms(MYSQL_THD thd, + SYS_VAR *var [[maybe_unused]], + void *save, + struct st_mysql_value *value) { + char temp_buffer[STRING_BUFFER_USUAL_SIZE]; + auto buf_len = static_cast(sizeof(temp_buffer)); + + auto compression_algorithms = value->val_str(value, temp_buffer, &buf_len); + + if (compression_algorithms && (compression_algorithms == temp_buffer)) { + compression_algorithms = thd_strmake(thd, compression_algorithms, buf_len); + } + + if (compression_algorithms == nullptr) return 1; + + std::string compress_option(static_cast(compression_algorithms)); + std::vector list; + parse_compression_algorithms_list(compress_option, list); + auto it = list.begin(); + while (it != list.end()) { + std::string value = *it; + switch (get_compression_algorithm(value)) { + case enum_compression_algorithm::MYSQL_INVALID: + return 1; + default: + break; + } + it++; + } + *(const char **)save = compression_algorithms; + return 0; +} + +/** Check clone zstd compression level +@param[in] thd user session THD +@param[in] var system variable +@param[out] save possibly updated variable value +@param[in] value current variable value +@return error code + */ +static int check_zstd_compression_level(MYSQL_THD thd [[maybe_unused]], + SYS_VAR *var [[maybe_unused]], + void *save [[maybe_unused]], + struct st_mysql_value *value) { + long long new_value; + if (value->val_int(value, &new_value)) return 1; /* NULL value */ + if (is_zstd_compression_level_valid(static_cast(new_value))) { + *((uint *)save) = new_value; + return 0; + } + return 1; +} + /** Check if it is safe to uninstall plugin. @param[in] plugin_info server plugin handle @return error code */ @@ -576,6 +642,21 @@ static MYSQL_SYSVAR_BOOL(enable_compression, clone_enable_compression, "If compression is done at network", nullptr, nullptr, false); /* Disable compression by default */ +static MYSQL_SYSVAR_STR(compression_algorithms, clone_compression_algorithms, + PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_MEMALLOC, + "If compression is done at network", check_compression_algorithms, nullptr, + COMPRESSION_ALGORITHM_UNCOMPRESSED); /* Default = "uncompressed" */ + +/** If data is compressed in network layer by zstd */ +static MYSQL_SYSVAR_UINT(zstd_compression_level, + clone_zstd_compression_level, PLUGIN_VAR_OPCMDARG, + "If compression is done at network and use zstd compression", + check_zstd_compression_level, nullptr, + default_zstd_compression_level, /* Default */ + 1, /* Minimum */ + 22, /* Maximum */ + 1); /* Step */ + /** List of valid donor addresses allowed to clone from. */ static MYSQL_SYSVAR_STR(valid_donor_list, clone_valid_donor_list, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_MEMALLOC, @@ -638,6 +719,8 @@ static SYS_VAR *clone_system_variables[] = { MYSQL_SYSVAR(max_network_bandwidth), MYSQL_SYSVAR(max_data_bandwidth), MYSQL_SYSVAR(enable_compression), + MYSQL_SYSVAR(compression_algorithms), + MYSQL_SYSVAR(zstd_compression_level), MYSQL_SYSVAR(autotune_concurrency), MYSQL_SYSVAR(valid_donor_list), MYSQL_SYSVAR(ssl_key), diff --git a/sql/server_component/clone_protocol_service.cc b/sql/server_component/clone_protocol_service.cc index 1b626b072eb..37a4a20ad83 100644 --- a/sql/server_component/clone_protocol_service.cc +++ b/sql/server_component/clone_protocol_service.cc @@ -408,11 +408,13 @@ DEFINE_METHOD(MYSQL *, mysql_clone_connect, mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, reinterpret_cast(&timeout)); - /* Enable compression. */ if (ssl_ctx->m_enable_compression) { mysql_options(mysql, MYSQL_OPT_COMPRESS, nullptr); - mysql_extension_set_server_extn(mysql, ssl_ctx->m_server_extn); + } else { + mysql_options(mysql, MYSQL_OPT_COMPRESSION_ALGORITHMS, ssl_ctx->m_compression_algorithms); + mysql_options(mysql, MYSQL_OPT_ZSTD_COMPRESSION_LEVEL, &ssl_ctx->m_zstd_compression_level); } + mysql_extension_set_server_extn(mysql, ssl_ctx->m_server_extn); ret_mysql = mysql_real_connect(mysql, host, user, passwd, nullptr, port, nullptr, 0); @@ -557,6 +559,18 @@ DEFINE_METHOD(int, mysql_clone_get_response, *net_length = 0; *length = my_net_read(net); + if (saved_extn != nullptr && net->compress) { + NET_SERVER *saved = static_cast(saved_extn); + mysql_compress_context old_saved_ctx = saved->compress_ctx; + saved->compress_ctx = server_extn.compress_ctx; + server_extn.compress_ctx = old_saved_ctx; + + if (server_extn.compress_ctx.algorithm == enum_compression_algorithm::MYSQL_ZSTD) { + server_extn.compress_ctx.u.zstd_ctx.cctx = nullptr; + server_extn.compress_ctx.u.zstd_ctx.dctx = nullptr; + } + } + net->extension = saved_extn; server_extn.compress_ctx.algorithm = MYSQL_UNCOMPRESSED;