From d0a3d7d19aa33bfa0ee187ae0459f89403f8002c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Fri, 10 Jan 2025 17:19:19 +0100 Subject: [PATCH] Docs: update replication relatated documentation --- sql/rpl_source.cc | 261 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 210 insertions(+), 51 deletions(-) diff --git a/sql/rpl_source.cc b/sql/rpl_source.cc index 4fd0c5b3de3..a33d9d8c1f9 100644 --- a/sql/rpl_source.cc +++ b/sql/rpl_source.cc @@ -283,11 +283,28 @@ bool show_replicas(THD *thd) { /** @page page_protocol_replication Replication Protocol - Replication uses binlogs to ship changes done on the master to the slave - and can be written to @ref sect_protocol_replication_binlog_file and sent - over the network as @ref sect_protocol_replication_binlog_stream. + Replication uses binary logs (binlogs) to ship changes done on the replication + source to the replica and can be written to @ref sect_protocol_replication_binlog_file + and sent over the network as @ref sect_protocol_replication_binlog_stream. - @section sect_protocol_replication_binlog_file Binlog File + Binlogs are also used for point-in-time recovery to fill in the changes between + the last backup and the recovery point. + + The replication stream can also be used for change data capture (CDC). + + @section sect_protocol_replication_binlog_stream Binlog Network Stream + + Networks streams are setup and authenticated in the same way regular connections are. + + The replica can optionally choose to register itself by sending a + @subpage page_protocol_com_register_replica command. + + Network streams are requested with @subpage page_protocol_com_binlog_dump when + using filename and offset based positioning and @subpage page_protocol_com_binlog_dump_gtid + when using GTID based positioning. Each @ref page_protocol_replication_binlog_event is + prepended with a `00` OK-byte. + + @section sect_protocol_replication_binlog_file Binlog Files Binlog files start with a @ref sect_protocol_replication_binlog_file_header followed by a series of @subpage page_protocol_replication_binlog_event @@ -303,10 +320,13 @@ bool show_replicas(THD *thd) { ... ~~~~~ - @section sect_protocol_replication_binlog_stream Binlog Network Stream +*/ - Network streams are requested with @subpage page_protocol_com_binlog_dump and - prepend each @ref page_protocol_replication_binlog_event with `00` OK-byte. +/** + @page page_protocol_replication_binlog_event Binlog Event + + The events contain the actual data that should be shipped from the master to + the slave. Depending on the use, different events are sent. @section sect_protocol_replication_binlog_version Binlog Version @@ -320,6 +340,9 @@ bool show_replicas(THD *thd) { 3 | MySQL 4.0.2 - < 5.0.0 4 | MySQL 5.0.0+ + @note In WL#9219: Remove cross compatibility code for binlog V1-V3 + compatibility for versions older than 4 were removed. + @subsection sect_protocol_replication_binlog_version_v1 Version 1 Supported @ref sect_protocol_replication_binlog_event_sbr @@ -338,15 +361,38 @@ bool show_replicas(THD *thd) { Added the @ref sect_protocol_replication_event_format_desc and made the protocol extensible. - In MySQL 5.1.x the @ref sect_protocol_replication_binlog_event_rbr were - added. -*/ + @section sect_protocol_replication_binlog_event_header Binlog Event Header -/** - @page page_protocol_replication_binlog_event Binlog Event + A binlog event starts with @ref sect_protocol_replication_binlog_event_header + and is followed by a event specific part. - The events contain the actual data that should be shipped from the master to - the slave. Depending on the use, different events are sent. + The binlog event header starts each event and is either 13 or 19 bytes long, depending + on the @ref sect_protocol_replication_binlog_version + + + + + + + + + + + + + + + + + + + + + + + +
Binlog::EventHeader:
TypeNameDescription
@ref a_protocol_type_int4 "int<4>"timestampseconds since unix epoch
@ref a_protocol_type_int1 "int<1>"event_typeSee mysql::binlog::event::Log_event_type
@ref a_protocol_type_int4 "int<4>"server-idserver-id of the originating mysql-server. Used to filter out events + in circular replication
@ref a_protocol_type_int4 "int<4>"event-sizesize of the event (header, post-header, body)
if binlog-version > 1 {
@ref a_protocol_type_int4 "int<4>"log-posposition of the next event
@ref a_protocol_type_int2 "int<2>"flagsSee @ref group_cs_binglog_event_header_flags
@section sect_protocol_replication_binlog_event_mgmt Binlog Management @@ -357,6 +403,8 @@ bool show_replicas(THD *thd) { @subsection sect_protocol_replication_event_start_v3 START_EVENT_V3 + @note @ref sect_protocol_replication_event_format_desc is used instead for binlog v4. + @@ -396,12 +444,10 @@ bool show_replicas(THD *thd) { -
Binlog::START_EVENT_V3:
TypeNameDescription
@ref a_protocol_type_int4 "int<4>" create-timestamp seconds since Unix epoch when the binlog was created
@ref a_protocol_type_int1 "int<1>" event-header-length Length of the @ref sect_protocol_replication_binlog_event_header of next events. Should always be 19. - @ref sect_protocol_basic_dt_string_eof "string<EOF>" event type header lengths a array indexed by `binlog-event-type - 1` to extract the length @@ -549,17 +595,43 @@ bool show_replicas(THD *thd) { Incident message with length `message_length` - @subsection sect_protocol_replication_event_heartbeat HEARTBEAT_EVENT + @subsection sect_protocol_replication_event_heartbeat HEARTBEAT_LOG_EVENT An artificial event generated by the master. It isn't written to the relay logs. It is added by the master after the replication connection was idle for `x` seconds to update the slave's Seconds_behind_source timestamp in the - SHOW REPLICA STATUS output. + `SHOW REPLICA STATUS` output. It has no payload nor post-header. + @subsection sect_protocol_replication_event_heartbeat_log_v2 HEARTBEAT_LOG_EVENT_V2 + + Similar to @ref sect_protocol_replication_event_heartbeat but able to handlde binlogs sizes + over 4 GiB. + + @subsection sect_protocol_replication_event_gtid_log GTID_LOG_EVENT + + This is to indicate the GTID assigned to a specific transaction. + + @subsection sect_protocol_replication_event_anonymous_gtid_log ANONYMOUS_GTID_LOG_EVENT + + This is used for transactions that have no GTID assigned. + + @subsection sect_protocol_replication_event_previous_gtids_log PREVIOUS_GTIDS_LOG_EVENT + + This is a list of GTID sets from previous binlog files. + + @subsection sect_protocol_replication_event_gtid_tagged_log GTID_TAGGED_LOG_EVENT + + This is similar to @ref sect_protocol_replication_event_gtid_log and adds suppport for tagged GTID. + + @subsection sect_protocol_replication_event_view_change VIEW_CHANGE_EVENT + + This event is not written to binlog files but send over the replication stream to indicate + group replication view changes. + @section sect_protocol_replication_binlog_event_sbr Statement Based Replication Events Statement Based Replication or SBR sends the SQL queries a client sent to @@ -774,16 +846,33 @@ bool show_replicas(THD *thd) { @subsection sect_protocol_replication_event_intvar INTVAR_EVENT + + Used to send the `LAST_INSERT_ID` if it is used in the statement. + @subsection sect_protocol_replication_event_rand RAND_EVENT + + Used to send information about random values if the `RAND()` function is used in the statement. + @subsection sect_protocol_replication_event_uservar USER_VAR_EVENT + + Used to send user variables if used in the statement. + @subsection sect_protocol_replication_event_xid XID_EVENT + This is to persist a `COMMIT` operation. + + @subsection sect_protocol_replication_event_xa_prepare_log XA_PREPARE_LOG_EVENT + + This is to persist a `XA PREPARE` operation. + @section sect_protocol_replication_binlog_event_rbr Row Based Replication Events In Row Based replication the changed rows are sent to the slave which removes side-effects and makes it more reliable. Now all statements can be sent with RBR though. Most of the time you will see RBR and SBR side by side. + Row Based events were added in MySQL 5.1.x. + @subsection sect_protocol_replication_event_table_map TABLE_MAP_EVENT @subsection sect_protocol_replication_event_delete_rows_v0 DELETE_ROWS_EVENTv0 @subsection sect_protocol_replication_event_update_rows_v0 UPDATE_ROWS_EVENTv0 @@ -791,6 +880,12 @@ bool show_replicas(THD *thd) { @subsection sect_protocol_replication_event_delete_rows_v2 DELETE_ROWS_EVENTv2 @subsection sect_protocol_replication_event_update_rows_v2 UPDATE_ROWS_EVENTv2 @subsection sect_protocol_replication_event_write_rows_v2 WRITE_ROWS_EVENTv2 + @subsection sect_protocol_replication_event_partial_update_rows PARTIAL_UPDATE_ROWS_EVENT + + @section sect_protocol_replication_binlog_compression Binlog Compression + + @subsection sect_protocol_replication_event_transaction_context TRANSACTION_CONTEXT_EVENT + @subsection sect_protocol_replication_event_transaction_payload TRANSACTION_PAYLOAD_EVENT @section sect_protocol_replication_binlog_event_load_file LOAD INFILE replication @@ -805,46 +900,47 @@ bool show_replicas(THD *thd) { @subsection sect_protocol_replication_event_new_load NEW_LOAD_EVENT @subsection sect_protocol_replication_event_load_query_begin BEGIN_LOAD_QUERY_EVENT @subsection sect_protocol_replication_event_load_query_execute EXECUTE_LOAD_QUERY_EVENT +*/ - A binlog event starts with @ref sect_protocol_replication_binlog_event_header - and is followed by a event specific part. - @section sect_protocol_replication_binlog_event_header Binlog Event Header +/** + @page page_protocol_com_binlog_dump COM_BINLOG_DUMP - The binlog event header starts each event and is either 13 or 19 bytes long, depending - on the @ref sect_protocol_replication_binlog_version + @brief Request a @ref sect_protocol_replication_binlog_stream from the server + based on a filename and offset based position. + + @return @ref sect_protocol_replication_binlog_stream on success or + @ref page_protocol_basic_err_packet on error - + - - - - - - - - - - - - + + - - + + - + + + + + + +
Binlog::EventHeader:Payload
TypeNameDescription
@ref a_protocol_type_int4 "int<4>"timestampseconds since unix epoch
@ref a_protocol_type_int1 "int<1>"event_typeSee mysql::binlog::event::Log_event_type
@ref a_protocol_type_int4 "int<4>"server-idserver-id of the originating mysql-server. Used to filter out events - in circular replication
@ref a_protocol_type_int4 "int<4>"event-sizesize of the event (header, post-header, body)
if binlog-version > 1 {
status[0x12] COM_BINLOG_DUMP
@ref a_protocol_type_int4 "int<4>"log-posposition of the next event
binlog-posposition in the binlog-file to start the stream with
@ref a_protocol_type_int2 "int<2>" flagsSee @ref group_cs_binglog_event_header_flags
can right now has one possible value: + ::BINLOG_DUMP_NON_BLOCK
@ref a_protocol_type_int4 "int<4>"server-idServer id of this replica
@ref sect_protocol_basic_dt_string_eof "string<EOF>"binlog-filenamefilename of the binlog on the replication source
-*/ + @sa @ref page_protocol_com_binlog_dump_gtid +*/ /** - @page page_protocol_com_binlog_dump COM_BINLOG_DUMP + @page page_protocol_com_binlog_dump_gtid COM_BINLOG_DUMP_GTID @brief Request a @ref sect_protocol_replication_binlog_stream from the server + based on a GTID position. @return @ref sect_protocol_replication_binlog_stream on success or @ref page_protocol_basic_err_packet on error @@ -854,23 +950,86 @@ bool show_replicas(THD *thd) { TypeNameDescription @ref a_protocol_type_int1 "int<1>" status - [0x12] COM_BINLOG_DUMP - @ref a_protocol_type_int4 "int<4>" - binlog-pos - position in the binlog-file to start the stream with + [0x1e] COM_BINLOG_DUMP_GTID @ref a_protocol_type_int2 "int<2>" flags can right now has one possible value: ::BINLOG_DUMP_NON_BLOCK @ref a_protocol_type_int4 "int<4>" server-id - Server id of this slave - @ref sect_protocol_basic_dt_string_eof "string<EOF>" - binlog-filename - filename of the binlog on the master + Server id of this replica + @ref a_protocol_type_int4 "int<4>" + filename_length + Filename length, always 0 + @ref a_protocol_type_int8 "int<8>" + position + position, always 4 + @ref a_protocol_type_int4 "int<4>" + gtid_length + Length of GTID data + @ref sect_protocol_basic_dt_string_var "binary<gtid_length>" + gtid_data + GTID data containing an encoded GTID Set. - @sa com_binlog_dump + @sa @ref page_protocol_com_binlog_dump +*/ + +/** + @page page_protocol_com_register_replica COM_REGISTER_REPLICA + + @brief Register a replica to the replication source. This is used + in the output of `SHOW REPLICAS`. This can be used for automatic + topology discovery and such. + + @note The username, password and other fields of this packet + are not used for authentication. + + @note This command was previously known as `COM_REGISTER_SLAVE`. + + The replica UUID is registered by setting the `replica_uuid` variable + before sending `COM_REGISTER_REPLICA`. + + @return @ref page_protocol_basic_ok_packet on success or + @ref page_protocol_basic_err_packet on error + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Payload
TypeNameDescription
@ref a_protocol_type_int1 "int<1>"status[0x15] COM_REGISTER_REPLICA
@ref a_protocol_type_int4 "int<4>"server_idReplica server id
@ref a_protocol_type_int1 "int<1>"hostname_lengthReplica hostname length
@ref sect_protocol_basic_dt_string_var "string<hostname_length>"hostnameReplica hostname
@ref a_protocol_type_int1 "int<1>"username_lengthReplica username length
@ref sect_protocol_basic_dt_string_var "string<username_length>"usernameReplica username
@ref a_protocol_type_int1 "int<1>"password_lengthReplica password
@ref sect_protocol_basic_dt_string_var "string<password_length>"passwordReplica password, max 32 characters
@ref a_protocol_type_int2 "int<2>"portReplica port
@ref a_protocol_type_int4 "int<4>"portReplication rank, always 0
@ref a_protocol_type_int4 "int<4>"portSource server id, always 0
*/ /* clang-format on */ -- 2.47.1