diff --git a/plugin/clone/include/clone_client.h b/plugin/clone/include/clone_client.h index 3727c572a10..da2b4d83719 100644 --- a/plugin/clone/include/clone_client.h +++ b/plugin/clone/include/clone_client.h @@ -478,7 +478,7 @@ class Client { when called for first time. @param[in] len length of allocated buffer @return allocated buffer pointer */ - uchar *get_aligned_buffer(uint32_t len); + uchar *get_aligned_buffer(uint32_t len, bool for_o_direct_uneven_file_size); /** Limit total memory used for clone transfer buffer. @param[in] buffer_size configured buffer size diff --git a/plugin/clone/include/clone_os.h b/plugin/clone/include/clone_os.h index b09cb3d4f36..5cec98f3b11 100644 --- a/plugin/clone/include/clone_os.h +++ b/plugin/clone/include/clone_os.h @@ -134,4 +134,15 @@ inline uchar *clone_os_align(uchar *pointer) { return (aligned_ptr); } +/** Make the buffer length suitable for I/O with O_DIRECT, if needed. +@param[in] buf_len buffer length +@param[in] o_direct whether this buffer will be used for O_DIRECT +@return If o_direct, bumped buf_len to a CLONE_OS_ALIGN multiple if it's not +already one, otherwise buf_len without any changes. */ +inline size_t clone_os_pad_for_o_direct(size_t buf_len, bool o_direct) { + static_assert((CLONE_OS_ALIGN & (CLONE_OS_ALIGN - 1)) == 0); + return o_direct ? ((buf_len + (CLONE_OS_ALIGN - 1)) & ~(CLONE_OS_ALIGN - 1)) + : buf_len; +} + #endif /* CLONE_OS_H */ diff --git a/plugin/clone/src/clone_client.cc b/plugin/clone/src/clone_client.cc index 5ca7394d7ce..7fc366c2e95 100644 --- a/plugin/clone/src/clone_client.cc +++ b/plugin/clone/src/clone_client.cc @@ -524,8 +524,10 @@ void Client::check_and_throttle() { info.throttle(data_speed, net_speed); } -uchar *Client::get_aligned_buffer(uint32_t len) { - auto err = m_copy_buff.allocate(len + CLONE_OS_ALIGN); +uchar *Client::get_aligned_buffer(uint32_t len, + bool for_o_direct_uneven_file_size) { + auto err = m_copy_buff.allocate(clone_os_pad_for_o_direct( + len + CLONE_OS_ALIGN, for_o_direct_uneven_file_size)); if (err != 0) { return (nullptr); @@ -1780,6 +1782,7 @@ int Client_Cbk::buffer_cbk(uchar *from_buffer [[maybe_unused]], uint buf_len) { int Client_Cbk::apply_buffer_cbk(uchar *&to_buffer, uint &len) { Ha_clone_file dummy_file; + dummy_file.o_direct_uneven_file_size = false; dummy_file.type = Ha_clone_file::FILE_HANDLE; dummy_file.file_handle = nullptr; return (apply_cbk(dummy_file, false, to_buffer, len)); @@ -1838,7 +1841,8 @@ int Client_Cbk::apply_cbk(Ha_clone_file to_file, bool apply_file, if (!is_os_buffer_cache()) { /* Allocate aligned buffer */ - buf_ptr = client->get_aligned_buffer(length); + buf_ptr = client->get_aligned_buffer( + length, apply_file && to_file.o_direct_uneven_file_size); if (buf_ptr == nullptr) { err = ER_OUTOFMEMORY; diff --git a/plugin/clone/src/clone_local.cc b/plugin/clone/src/clone_local.cc index 8bc4aa7bf40..cb1b18979a4 100644 --- a/plugin/clone/src/clone_local.cc +++ b/plugin/clone/src/clone_local.cc @@ -345,6 +345,8 @@ int Local_Callback::apply_cbk(Ha_clone_file to_file, bool apply_file, uchar *buf_ptr; uint buf_len; + auto from_file = ext_link->get_file(); + if (is_os_buffer_cache() && is_zero_copy() && clone_os_supports_zero_copy()) { buf_ptr = nullptr; @@ -352,22 +354,24 @@ int Local_Callback::apply_cbk(Ha_clone_file to_file, bool apply_file, } else { /* For direct IO use client buffer. */ buf_len = client->limit_buffer(clone_buffer_size); - buf_ptr = client->get_aligned_buffer(buf_len); + buf_ptr = client->get_aligned_buffer( + buf_len, + apply_file && (from_file->m_file_desc.o_direct_uneven_file_size || + to_file.o_direct_uneven_file_size)); if (buf_ptr == nullptr) { return (ER_OUTOFMEMORY); } } - auto from_file = ext_link->get_file(); - if (apply_file) { error = clone_os_copy_file_to_file(from_file->m_file_desc, to_file, from_file->m_length, buf_ptr, buf_len, get_source_name(), get_dest_name()); } else { to_len = from_file->m_length; - to_buffer = client->get_aligned_buffer(to_len); + to_buffer = + client->get_aligned_buffer(to_len, false /* not for O_DIRECT */); if (to_buffer == nullptr) { return (ER_OUTOFMEMORY); /* purecov: inspected */ } diff --git a/plugin/clone/src/clone_os.cc b/plugin/clone/src/clone_os.cc index 164db1bd25a..1008f757645 100644 --- a/plugin/clone/src/clone_os.cc +++ b/plugin/clone/src/clone_os.cc @@ -164,9 +164,11 @@ int clone_os_copy_file_to_buf(Ha_clone_file from_file, uchar *to_buffer, auto len_left = length; while (len_left > 0) { + const auto padded_length = clone_os_pad_for_o_direct( + len_left, from_file.o_direct_uneven_file_size); uint ret_length = 0; - auto error = - read_from_file(from_file, to_buffer, len_left, src_name, ret_length); + auto error = read_from_file(from_file, to_buffer, padded_length, src_name, + ret_length); if (error != 0) { DBUG_PRINT("debug", ("Error: clone read failed." @@ -176,6 +178,8 @@ int clone_os_copy_file_to_buf(Ha_clone_file from_file, uchar *to_buffer, return (error); } + assert(ret_length <= len_left); + len_left -= ret_length; to_buffer += ret_length; } @@ -234,6 +238,9 @@ int clone_os_copy_file_to_file(Ha_clone_file from_file, Ha_clone_file to_file, while (length > 0) { auto request_size = (length > buff_len) ? buff_len : length; + request_size = clone_os_pad_for_o_direct( + request_size, from_file.o_direct_uneven_file_size); + assert(request_size <= buff_len); uint actual_size = 0; error = @@ -267,8 +274,15 @@ int clone_os_copy_buf_to_file(uchar *from_buffer, Ha_clone_file to_file, CLONE_OS_CHECK_FILE(to_file); while (length > 0) { + const auto last_block_used = length % CLONE_OS_ALIGN; + const auto block_pad_len = + to_file.o_direct_uneven_file_size && (last_block_used != 0) + ? CLONE_OS_ALIGN - last_block_used + : 0; + if (block_pad_len > 0) memset(from_buffer + length, 0, block_pad_len); + errno = 0; - auto ret_size = os_write(to_file, from_buffer, length); + auto ret_size = os_write(to_file, from_buffer, length + block_pad_len); if (errno == EINTR) { DBUG_PRINT("debug", ("clone write() interrupted")); @@ -289,7 +303,21 @@ int clone_os_copy_buf_to_file(uchar *from_buffer, Ha_clone_file to_file, } auto actual_size = static_cast(ret_size); - + assert(length + block_pad_len == actual_size); + + if (actual_size > length) { + assert(block_pad_len > 0); + assert(to_file.o_direct_uneven_file_size); + const auto file_size = my_tell(to_file.file_desc, MYF(MY_WME)); + if (file_size < 0) { + return ER_ERROR_ON_WRITE; + } + if (my_chsize(to_file.file_desc, file_size - block_pad_len, 0, + MYF(MY_WME)) != 0) { + return ER_ERROR_ON_WRITE; + } + actual_size -= block_pad_len; + } assert(length >= actual_size); length -= actual_size; diff --git a/plugin/clone/src/clone_server.cc b/plugin/clone/src/clone_server.cc index fa0acd80e64..a4e80f8f149 100644 --- a/plugin/clone/src/clone_server.cc +++ b/plugin/clone/src/clone_server.cc @@ -650,7 +650,8 @@ int Server_Cbk::file_cbk(Ha_clone_file from_file, uint len) { /* Add one byte for descriptor type */ auto buf_len = len + 1; - auto buf_ptr = server->alloc_copy_buffer(buf_len + CLONE_OS_ALIGN); + auto buf_ptr = server->alloc_copy_buffer(clone_os_pad_for_o_direct( + buf_len + CLONE_OS_ALIGN, from_file.o_direct_uneven_file_size)); if (buf_ptr == nullptr) { return (ER_OUTOFMEMORY); diff --git a/sql/handler.h b/sql/handler.h index 5f3798cc451..b88a4888c34 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1016,6 +1016,11 @@ bool ha_is_externally_disabled(const handlerton &); /** File reference for clone */ struct Ha_clone_file { + // Whether this file is opened with O_DIRECT flag and the size of this file is + // allowed to be a non-multiple of 4K. MyRocks sets this as needed. InnoDB, + // while using O_DIRECT, has all file sizes as 4K multiples. + bool o_direct_uneven_file_size; + /** File reference type */ enum { /** File handle */ diff --git a/storage/innobase/clone/clone0clone.cc b/storage/innobase/clone/clone0clone.cc index 8a4f7a2dbcb..7db22d5dd0e 100644 --- a/storage/innobase/clone/clone0clone.cc +++ b/storage/innobase/clone/clone0clone.cc @@ -2350,9 +2350,11 @@ int Clone_Handle::file_callback(Ha_clone_cbk *cbk, Clone_Task *task, uint len, /* Platform specific code to set file handle */ #ifdef _WIN32 + file.o_direct_uneven_file_size = false; file.type = Ha_clone_file::FILE_HANDLE; file.file_handle = static_cast(task->m_current_file_des.m_file); #else + file.o_direct_uneven_file_size = false; file.type = Ha_clone_file::FILE_DESC; file.file_desc = task->m_current_file_des.m_file; #endif /* _WIN32 */