From 99cf9b49f24eb57f5359312af033b344ab470541 Mon Sep 17 00:00:00 2001 From: Weng Haixing Date: Mon, 12 Jun 2017 10:29:17 +0800 Subject: [PATCH] MYSQL-OPT FEATURE-REQUEST: OPTIMIZE MASTER-TO-MASTER REPLICATION, STOP SERVER TO SEND BACK DUPLICATE BINLOG DESCRIPTION: ============ This patch introduced the ability to identify duplicate log-event which was generated by master , and stop send it back. Analysis: ========= Set up S1, S2 in mysql master-to-master replication mode(As shown in the figure), S2 would apply relay-log-events fetched from S1, in commit phase of replay, S2 log those events in its own binlog-file, and send them back to S1. In terms of the final results, S1 will never handle duplicate log-events sent by S2 (may not even write them into relay-log-file). Sending back duplicate log-events cause waste of bandwidth, this is particularly apparent for replication mode on cross city data centers. This ablility save network bandwidth in both master-to-master semisynchronous and asynchronous mode, furthermore it will improve performance of log replay in 'semisynchronous' mode, because S2 can instant commit after write binlog-file without wating response from S1. Before optimization: app | (write) ----------- ----------- | S1 | ------> | S2 | ----------- <------ ----------- duplicate log events With optimization: app | (write) ----------- ----------- | S1 | ------> | S2 | ----------- <--X--- ---------- --- mysql-test/r/mysqld--help-notwin.result | 3 ++ mysql-test/suite/perfschema/r/show_sanity.result | 2 ++ mysql-test/suite/sys_vars/r/all_vars.result | 2 ++ sql/mysqld.cc | 1 + sql/mysqld.h | 1 + sql/rpl_binlog_sender.cc | 42 ++++++++++++++++++++++-- sql/rpl_binlog_sender.h | 4 +++ sql/rpl_slave.cc | 3 +- sql/sql_class.h | 1 + sql/sys_vars.cc | 10 ++++++ 10 files changed, 66 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index e818dd4..364e64b 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -62,6 +62,8 @@ The following options may be given as the first argument: --binlog-do-db=name Tells the master it should log updates for the specified database, and exclude all others not explicitly mentioned. + --binlog-dump-control=name + While dump binlog, it decide to if dump duplicate binlog --binlog-error-action=name When statements cannot be written to the binary log due to a fatal error, the server can either ignore the error @@ -1288,6 +1290,7 @@ bind-address * binlog-cache-size 32768 binlog-checksum CRC32 binlog-direct-non-transactional-updates FALSE +binlog-dump-control ALL_DUMP binlog-error-action ABORT_SERVER binlog-format ROW binlog-group-commit-sync-delay 0 diff --git a/mysql-test/suite/perfschema/r/show_sanity.result b/mysql-test/suite/perfschema/r/show_sanity.result index 71bc92a..bac1e8f 100644 --- a/mysql-test/suite/perfschema/r/show_sanity.result +++ b/mysql-test/suite/perfschema/r/show_sanity.result @@ -407,6 +407,7 @@ where show_mode = "JUNK: GLOBAL-ONLY" and source = "I_S.SESSION_VARIABLES") order by show_mode, source, variable_name; SHOW_MODE SOURCE VARIABLE_NAME +5.6 I_S.SESSION_VARIABLES BINLOG_DUMP_CONTROL 5.6 I_S.SESSION_VARIABLES BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE 5.6 I_S.SESSION_VARIABLES BINLOG_TRANSACTION_DEPENDENCY_TRACKING 5.6 I_S.SESSION_VARIABLES GTID_EXECUTED @@ -435,6 +436,7 @@ where show_mode = "JUNK: GLOBAL-ONLY" and source = "I_S.SESSION_VARIABLES") order by show_mode, source, variable_name; SHOW_MODE SOURCE VARIABLE_NAME +5.6 I_S.SESSION_VARIABLES BINLOG_DUMP_CONTROL 5.6 I_S.SESSION_VARIABLES BINLOG_TRANSACTION_DEPENDENCY_HISTORY_SIZE 5.6 I_S.SESSION_VARIABLES BINLOG_TRANSACTION_DEPENDENCY_TRACKING 5.6 I_S.SESSION_VARIABLES GTID_EXECUTED diff --git a/mysql-test/suite/sys_vars/r/all_vars.result b/mysql-test/suite/sys_vars/r/all_vars.result index 5ddc39a..c2d9e4a 100644 --- a/mysql-test/suite/sys_vars/r/all_vars.result +++ b/mysql-test/suite/sys_vars/r/all_vars.result @@ -13,6 +13,8 @@ There should be *no* long test name listed below: select variable_name as `There should be *no* variables listed below:` from t2 left join t1 on variable_name=test_name where test_name is null ORDER BY variable_name; There should be *no* variables listed below: +BINLOG_DUMP_CONTROL +BINLOG_DUMP_CONTROL DISABLED_STORAGE_ENGINES DISABLED_STORAGE_ENGINES KEYRING_OPERATIONS diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 7bfe384..f0d9d2b 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -459,6 +459,7 @@ ulonglong slave_type_conversions_options; ulong opt_mts_slave_parallel_workers; ulonglong opt_mts_pending_jobs_size_max; ulonglong slave_rows_search_algorithms_options; +ulong binlog_dump_control_options; #ifdef HAVE_REPLICATION my_bool opt_slave_preserve_commit_order; diff --git a/sql/mysqld.h b/sql/mysqld.h index 360f61e..8056c4c 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -146,6 +146,7 @@ extern my_bool read_only, opt_readonly; extern my_bool super_read_only, opt_super_readonly; extern my_bool lower_case_file_system; extern ulonglong slave_rows_search_algorithms_options; +extern ulong binlog_dump_control_options; extern my_bool opt_require_secure_transport; #ifdef HAVE_REPLICATION diff --git a/sql/rpl_binlog_sender.cc b/sql/rpl_binlog_sender.cc index 3b469a1..5fb40a9 100644 --- a/sql/rpl_binlog_sender.cc +++ b/sql/rpl_binlog_sender.cc @@ -60,6 +60,7 @@ void Binlog_sender::init() DBUG_ENTER("Binlog_sender::init"); THD *thd= m_thd; + last_ignore_tv = 0; thd->push_diagnostics_area(&m_diag_area); init_heartbeat_period(); m_last_event_sent_ts= time(0); @@ -171,6 +172,7 @@ void Binlog_sender::cleanup() THD *thd= m_thd; + last_ignore_tv = 0; if (m_transmit_started) (void) RUN_HOOK(binlog_transmit, transmit_stop, (thd, m_flag)); @@ -409,6 +411,8 @@ int Binlog_sender::send_events(IO_CACHE *log_cache, my_off_t end_pos) my_off_t log_pos= my_b_tell(log_cache); my_off_t exclude_group_end_pos= 0; bool in_exclude_group= false; + bool ignore_binlog= false; +// time_t last_ignore_tv= 0; while (likely(log_pos < end_pos)) { @@ -499,12 +503,46 @@ int Binlog_sender::send_events(IO_CACHE *log_cache, my_off_t end_pos) m_packet.length(tmp.length()); } - if (unlikely(send_packet())) + if (!in_exclude_group && binlog_dump_control_options == IGNORE_MASTER_BINLOG) + { + ulong s_id= uint4korr(event_ptr + SERVER_ID_OFFSET); + s_id&= opt_server_id_mask; + ignore_binlog= s_id == thd->server_id; + + if (ignore_binlog) + DBUG_PRINT("info", ("ignore_binlog::ignore_binlog %d, packet sid %lu, server sid %d, unmask servr sid %d", + ignore_binlog, s_id, thd->server_id, thd->unmasked_server_id + )); + } + + if (ignore_binlog == false && unlikely(send_packet())) DBUG_RETURN(1); } - if (unlikely(after_send_hook(log_file, in_exclude_group ? log_pos : 0))) + if (unlikely(after_send_hook(log_file, in_exclude_group || ignore_binlog ? log_pos : 0))) DBUG_RETURN(1); + + if (ignore_binlog) + { + if (!last_ignore_tv) + last_ignore_tv= time(0); + else + { + + time_t cur= time(0); + bool hb_for_ignore_binlog= ((ulonglong)(cur - last_ignore_tv) + >= (ulonglong)(m_heartbeat_period/1000000000UL)); + if (hb_for_ignore_binlog) + { + if (send_heartbeat_event(log_pos)) + DBUG_RETURN(1); + last_ignore_tv= cur; + } + + } + } + + } /* diff --git a/sql/rpl_binlog_sender.h b/sql/rpl_binlog_sender.h index 806f6b9..075dfdc 100644 --- a/sql/rpl_binlog_sender.h +++ b/sql/rpl_binlog_sender.h @@ -150,6 +150,10 @@ private: */ bool m_observe_transmission; + + /*ingnore time event time in close binlog snd back*/ + time_t last_ignore_tv; + /* It is true if transmit_start hook is called. If the hook is not called * it will be false. */ diff --git a/sql/rpl_slave.cc b/sql/rpl_slave.cc index 42e42b0..6a85930 100644 --- a/sql/rpl_slave.cc +++ b/sql/rpl_slave.cc @@ -8384,7 +8384,8 @@ bool queue_event(Master_info* mi,const char* buf, ulong event_len) we update only the positions and not the file names, as a ROTATE EVENT from the master prior to this will update the file name. */ - if (mi->is_auto_position() && mi->get_master_log_pos() < + if ((mi->is_auto_position() || binlog_dump_control_options == IGNORE_MASTER_BINLOG) + && mi->get_master_log_pos() < hb.common_header->log_pos && mi->get_master_log_name() != NULL) { diff --git a/sql/sql_class.h b/sql/sql_class.h index 87ee8f3..ac80b67 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -110,6 +110,7 @@ enum enum_slave_type_conversions { SLAVE_TYPE_CONVERSIONS_ALL_LOSSY, enum enum_slave_rows_search_algorithms { SLAVE_ROWS_TABLE_SCAN = (1U << 0), SLAVE_ROWS_INDEX_SCAN = (1U << 1), SLAVE_ROWS_HASH_SCAN = (1U << 2)}; +enum enum_binlog_dump_control {ALL_DUMP= 0, IGNORE_MASTER_BINLOG= 1}; enum enum_binlog_row_image { /** PKE in the before image and changed columns in the after image */ BINLOG_ROW_IMAGE_MINIMAL= 0, diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 0452ae5..c8dc977 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3548,6 +3548,16 @@ static Sys_var_set Slave_rows_search_algorithms( DEFAULT(SLAVE_ROWS_INDEX_SCAN | SLAVE_ROWS_TABLE_SCAN), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_not_null_not_empty), ON_UPDATE(NULL)); +static const char *binlog_dump_control_names[]= {"ALL_DUMP", "IGNORE_DUPLICATE_BINLOG", 0}; +static Sys_var_enum Binlog_dump_control( + "binlog_dump_control", + "While dump binlog, it decide to if dump duplicate binlog", + GLOBAL_VAR(binlog_dump_control_options), CMD_LINE(REQUIRED_ARG), + binlog_dump_control_names, + DEFAULT(ALL_DUMP), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_not_null_not_empty), ON_UPDATE(NULL) + ); + static const char *mts_parallel_type_names[]= {"DATABASE", "LOGICAL_CLOCK", 0}; static Sys_var_enum Mts_parallel_type( "slave_parallel_type", -- 1.8.3.1