From 674f4da8f0ffceb416e462bcfc6b98889f59d8c4 Mon Sep 17 00:00:00 2001 From: Stan P Date: Fri, 22 Apr 2016 14:22:34 +0300 Subject: [PATCH 1/5] Add timeout for error connection counter. Add test socket before write --- src/routing/include/mysqlrouter/routing.h | 5 +++++ src/routing/src/mysql_routing.cc | 15 +++++++++++---- src/routing/src/mysql_routing.h | 12 +++++++++++- src/routing/src/plugin_config.cc | 1 + src/routing/src/plugin_config.h | 3 +++ src/routing/src/routing.cc | 1 + 6 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/routing/include/mysqlrouter/routing.h b/src/routing/include/mysqlrouter/routing.h index 5bfbff5..382ecba 100644 --- a/src/routing/include/mysqlrouter/routing.h +++ b/src/routing/include/mysqlrouter/routing.h @@ -52,6 +52,11 @@ extern const int kDefaultDestinationConnectionTimeout; */ extern const unsigned long long kDefaultMaxConnectErrors; +/** @brief Timeout then reset counter for connect or handshake errors per host + * + */ +extern const unsigned long long kDefaultMaxConnectErrorsTimeout; + /** @brief Default bind address * */ diff --git a/src/routing/src/mysql_routing.cc b/src/routing/src/mysql_routing.cc index 50d8d0d..27163c4 100644 --- a/src/routing/src/mysql_routing.cc +++ b/src/routing/src/mysql_routing.cc @@ -57,6 +57,7 @@ MySQLRouting::MySQLRouting(routing::AccessMode mode, int port, const string &bin int max_connections, int destination_connect_timeout, unsigned long long max_connect_errors, + unsigned long long max_connect_errors_timeout, unsigned int client_connect_timeout, unsigned int net_buffer_length) : name(route_name), @@ -64,6 +65,7 @@ MySQLRouting::MySQLRouting(routing::AccessMode mode, int port, const string &bin max_connections_(set_max_connections(max_connections)), destination_connect_timeout_(set_destination_connect_timeout(destination_connect_timeout)), max_connect_errors_(max_connect_errors), + max_connect_errors_timeout_(max_connect_errors_timeout), client_connect_timeout_(client_connect_timeout), net_buffer_length_(net_buffer_length), bind_address_(TCPAddress(bind_address, port)), @@ -181,17 +183,22 @@ bool MySQLRouting::block_client_host(const std::array &client_ip_ar bool blocked = false; std::lock_guard lock(mutex_auth_errors_); - if (++auth_error_counters_[client_ip_array] >= max_connect_errors_) { + if (++auth_error_counters_[client_ip_array].count >= max_connect_errors_) { log_warning("[%s] blocking client host %s", name.c_str(), client_ip_str.c_str()); blocked = true; } else { log_info("[%s] %d authentication errors for %s (max %d)", - name.c_str(), auth_error_counters_[client_ip_array], client_ip_str.c_str(), max_connect_errors_); + name.c_str(), auth_error_counters_[client_ip_array].count, client_ip_str.c_str(), max_connect_errors_); } if (server >= 0) { auto fake_response = mysql_protocol::HandshakeResponsePacket(1, {}, "ROUTER", "", "fake_router_login"); - write(server, fake_response.data(), fake_response.size()); + int error_code; + socklen_t error_code_size = sizeof(error_code); + int res = getsockopt(server, SOL_SOCKET, SO_ERROR, &error_code, &error_code_size); + if (res == 0 && error_code == 0) { + write(server, fake_response.data(), fake_response.size()); + } } return blocked; @@ -348,7 +355,7 @@ void MySQLRouting::start() { continue; } - if (auth_error_counters_[in6_addr_to_array(client_addr.sin6_addr)] >= max_connect_errors_) { + if (auth_error_counters_[in6_addr_to_array(client_addr.sin6_addr)].count >= max_connect_errors_) { std::stringstream os; os << "Too many connection errors from " << get_peer_name(sock_client).first; auto server_error = mysql_protocol::ErrorPacket(0, 1129, os.str(), "HY000"); diff --git a/src/routing/src/mysql_routing.h b/src/routing/src/mysql_routing.h index d0602c2..bc3ed1a 100644 --- a/src/routing/src/mysql_routing.h +++ b/src/routing/src/mysql_routing.h @@ -33,6 +33,7 @@ #include "plugin_config.h" #include +#include #include #include #include @@ -80,6 +81,7 @@ using mysqlrouter::URI; * use 10.0.11.6 to setup the connection routing. * */ + class MySQLRouting { public: /** @brief Default constructor @@ -97,9 +99,15 @@ class MySQLRouting { int max_connections = routing::kDefaultMaxConnections, int destination_connect_timeout = routing::kDefaultDestinationConnectionTimeout, unsigned long long max_connect_errors = routing::kDefaultMaxConnectErrors, + unsigned long long max_connect_errors_timeout = routing::kDefaultMaxConnectErrorsTimeout, unsigned int connect_timeout = routing::kDefaultClientConnectTimeout, unsigned int net_buffer_length = routing::kDefaultNetBufferLength); + struct AuthErrorCounter { + size_t count; + std::time_t last_attempt; + }; + /** @brief Starts the service and accept incoming connections * * Starts the connection routing service and start accepting incoming @@ -253,6 +261,8 @@ class MySQLRouting { int destination_connect_timeout_; /** @brief Max connect errors blocking hosts when handshake not completed */ unsigned long long max_connect_errors_; + /** @brief Timeout fot reset counter for connect errors blocking hosts when handshake not completed */ + unsigned long long max_connect_errors_timeout_; /** @brief Timeout waiting for handshake response from client */ unsigned int client_connect_timeout_; /** @brief Size of buffer to store receiving packets */ @@ -272,7 +282,7 @@ class MySQLRouting { /** @brief Authentication error counters for IPv4 or IPv6 hosts */ std::mutex mutex_auth_errors_; - std::map, size_t> auth_error_counters_; + std::map, AuthErrorCounter> auth_error_counters_; std::vector> blocked_client_hosts_; }; diff --git a/src/routing/src/plugin_config.cc b/src/routing/src/plugin_config.cc index 8c69177..8c47224 100644 --- a/src/routing/src/plugin_config.cc +++ b/src/routing/src/plugin_config.cc @@ -38,6 +38,7 @@ string RoutingPluginConfig::get_default(const string &option) { {"connect_timeout", to_string(routing::kDefaultDestinationConnectionTimeout)}, {"max_connections", to_string(routing::kDefaultMaxConnections)}, {"max_connect_errors", to_string(routing::kDefaultMaxConnectErrors)}, + {"max_connect_errors_timeout", to_string(routing::kDefaultMaxConnectErrorsTimeout)}, {"client_connect_timeout", to_string(routing::kDefaultClientConnectTimeout)}, {"net_buffer_length", to_string(routing::kDefaultNetBufferLength)}, }; diff --git a/src/routing/src/plugin_config.h b/src/routing/src/plugin_config.h index 77d481b..bd6a16c 100644 --- a/src/routing/src/plugin_config.h +++ b/src/routing/src/plugin_config.h @@ -53,6 +53,7 @@ class RoutingPluginConfig final : public mysqlrouter::BasePluginConfig { mode(get_option_mode(section, "mode")), max_connections(get_uint_option(section, "max_connections", 1)), max_connect_errors(get_uint_option(section, "max_connect_errors", 1, UINT32_MAX)), + max_connect_errors_timeout(get_uint_option(section, "max_connect_errors_timeout", 300, UINT32_MAX)), // 5 minutes for reset error attempts client_connect_timeout(get_uint_option(section, "client_connect_timeout", 2, 31536000)), net_buffer_length(get_uint_option(section, "net_buffer_length", 1024, 1048576)) { } @@ -74,6 +75,8 @@ class RoutingPluginConfig final : public mysqlrouter::BasePluginConfig { const int max_connections; /** @brief `max_connect_errors` option read from configuration section */ const unsigned long long max_connect_errors; + /** @brief `max_connect_errors_timeout` option read from configuration section */ + const unsigned long long max_connect_errors_timeout; /** @brief `client_connect_timeout` option read from configuration section */ const unsigned int client_connect_timeout; /** @brief Size of buffer to receive packets */ diff --git a/src/routing/src/routing.cc b/src/routing/src/routing.cc index 9639cf8..e629d91 100644 --- a/src/routing/src/routing.cc +++ b/src/routing/src/routing.cc @@ -48,6 +48,7 @@ const int kDefaultDestinationConnectionTimeout = 1; const string kDefaultBindAddress = "127.0.0.1"; const unsigned int kDefaultNetBufferLength = 16384; // Default defined in latest MySQL Server const unsigned long long kDefaultMaxConnectErrors = 100; // Similar to MySQL Server +const unsigned long long kDefaultMaxConnectErrorsTimeout = 300; // 5 minutes const unsigned int kDefaultClientConnectTimeout = 9; // Default connect_timeout MySQL Server minus 1 const std::map kAccessModeNames = { From debf36b6ec8e306d0a79a9ad92bf6c5e0c49f941 Mon Sep 17 00:00:00 2001 From: Stan P Date: Sat, 23 Apr 2016 07:38:10 +0300 Subject: [PATCH 2/5] Check for socket is alive for any write() call --- src/routing/src/mysql_routing.cc | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/routing/src/mysql_routing.cc b/src/routing/src/mysql_routing.cc index 27163c4..2ac264b 100644 --- a/src/routing/src/mysql_routing.cc +++ b/src/routing/src/mysql_routing.cc @@ -77,6 +77,16 @@ MySQLRouting::MySQLRouting(routing::AccessMode mode, int port, const string &bin } } +bool check_socket_alive(int fd) { + int error_code; + socklen_t error_code_size = sizeof(error_code); + int res = getsockopt(fd, SOL_SOCKET, SO_ERROR, &error_code, &error_code_size); + if (res == 0 && error_code == 0) { + return true; + }; + return false; +} + /** @brief Reads from sender and writes it back to receiver using select * * This function reads data from the sender socket and writes it back @@ -138,7 +148,9 @@ int copy_mysql_protocol_packets(int sender, int receiver, fd_set *readfds, // We got error from MySQL Server while handshaking // We do not consider this a failed handshake auto server_error = mysql_protocol::ErrorPacket(buffer); - write(receiver, server_error.data(), server_error.size()); + if (check_socket_alive) { + write(receiver, server_error.data(), server_error.size()); + } // receiver socket closed by caller *curr_pktnr = 2; // we assume handshaking is done though there was an error *report_bytes_read = bytes_read; @@ -164,6 +176,9 @@ int copy_mysql_protocol_packets(int sender, int receiver, fd_set *readfds, size_t bytes_to_write = bytes_read; ssize_t written = 0; while (bytes_to_write > 0) { + if (!check_socket_alive(receiver)) { + break; + } if ((written = write(receiver, buffer.data(), bytes_to_write)) < 0) { log_debug("Write error: %s", strerror(errno)); return -1; @@ -193,10 +208,7 @@ bool MySQLRouting::block_client_host(const std::array &client_ip_ar if (server >= 0) { auto fake_response = mysql_protocol::HandshakeResponsePacket(1, {}, "ROUTER", "", "fake_router_login"); - int error_code; - socklen_t error_code_size = sizeof(error_code); - int res = getsockopt(server, SOL_SOCKET, SO_ERROR, &error_code, &error_code_size); - if (res == 0 && error_code == 0) { + if (check_socket_alive) { write(server, fake_response.data(), fake_response.size()); } } @@ -223,7 +235,9 @@ void MySQLRouting::routing_select_thread(int client, const in6_addr client_addr) os << "Can't connect to MySQL server on "; os << "'" << bind_address_.addr << "'"; auto server_error = mysql_protocol::ErrorPacket(0, 2003, os.str(), "HY000"); - write(client, server_error.data(), server_error.size()); + if (check_socket_alive) { + write(client, server_error.data(), server_error.size()); + } shutdown(client, SHUT_RDWR); shutdown(server, SHUT_RDWR); @@ -359,14 +373,18 @@ void MySQLRouting::start() { std::stringstream os; os << "Too many connection errors from " << get_peer_name(sock_client).first; auto server_error = mysql_protocol::ErrorPacket(0, 1129, os.str(), "HY000"); - write(sock_client, server_error.data(), server_error.size()); + if (check_socket_alive) { + write(sock_client, server_error.data(), server_error.size()); + } close(sock_client); // no shutdown() before close() continue; } if (info_active_routes_.load(std::memory_order_relaxed) >= max_connections_) { auto server_error = mysql_protocol::ErrorPacket(0, 1040, "Too many connections", "HY000"); - write(sock_client, server_error.data(), server_error.size()); + if (check_socket_alive) { + write(sock_client, server_error.data(), server_error.size()); + } close(sock_client); // no shutdown() before close() log_warning("[%s] reached max active connections (%d)", name.c_str(), max_connections_); continue; From c0e5d035ea0de51e460a9117140674705d01e525 Mon Sep 17 00:00:00 2001 From: Stan P Date: Sat, 23 Apr 2016 20:08:24 +0300 Subject: [PATCH 3/5] Added some checks for max_connect_errors_timeout --- src/routing/src/mysql_routing.cc | 38 ++++++++++++++++++++++++++++---------- src/routing/src/mysql_routing.h | 2 ++ src/routing/src/plugin_config.h | 2 +- src/routing/src/routing.cc | 2 +- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/routing/src/mysql_routing.cc b/src/routing/src/mysql_routing.cc index 2ac264b..aac683b 100644 --- a/src/routing/src/mysql_routing.cc +++ b/src/routing/src/mysql_routing.cc @@ -22,6 +22,7 @@ #include "plugin_config.h" #include "mysqlrouter/routing.h" +#include #include #include #include @@ -193,17 +194,32 @@ int copy_mysql_protocol_packets(int sender, int receiver, fd_set *readfds, return 0; } +bool MySQLRouting::check_client_errors_time(const std::array &client_ip_array) { + size_t timediff; + if (max_connect_errors_timeout_ == 0) { + return false; + } + std::time_t curtime = std::time(nullptr); + timediff = curtime - auth_error_counters_[client_ip_array].last_attempt; + if (timediff > max_connect_errors_timeout_) { + auth_error_counters_[client_ip_array].count = 0; + return true; + }; + return false; +} + bool MySQLRouting::block_client_host(const std::array &client_ip_array, const string &client_ip_str, int server) { bool blocked = false; std::lock_guard lock(mutex_auth_errors_); - + struct tm *curtime = localtime(&auth_error_counters_[client_ip_array].last_attempt); + auth_error_counters_[client_ip_array].last_attempt = std::time(0); if (++auth_error_counters_[client_ip_array].count >= max_connect_errors_) { log_warning("[%s] blocking client host %s", name.c_str(), client_ip_str.c_str()); blocked = true; } else { - log_info("[%s] %d authentication errors for %s (max %d)", - name.c_str(), auth_error_counters_[client_ip_array].count, client_ip_str.c_str(), max_connect_errors_); + log_info("[%s] %d authentication errors for %s (max %d). last attempt: %s", + name.c_str(), auth_error_counters_[client_ip_array].count, client_ip_str.c_str(), max_connect_errors_, asctime(curtime)); } if (server >= 0) { @@ -370,14 +386,16 @@ void MySQLRouting::start() { } if (auth_error_counters_[in6_addr_to_array(client_addr.sin6_addr)].count >= max_connect_errors_) { - std::stringstream os; - os << "Too many connection errors from " << get_peer_name(sock_client).first; - auto server_error = mysql_protocol::ErrorPacket(0, 1129, os.str(), "HY000"); - if (check_socket_alive) { - write(sock_client, server_error.data(), server_error.size()); + if (!check_client_errors_time(in6_addr_to_array(client_addr.sin6_addr))) { + std::stringstream os; + os << "Too many connection errors from " << get_peer_name(sock_client).first; + auto server_error = mysql_protocol::ErrorPacket(0, 1129, os.str(), "HY000"); + if (check_socket_alive) { + write(sock_client, server_error.data(), server_error.size()); + } + close(sock_client); // no shutdown() before close() + continue; } - close(sock_client); // no shutdown() before close() - continue; } if (info_active_routes_.load(std::memory_order_relaxed) >= max_connections_) { diff --git a/src/routing/src/mysql_routing.h b/src/routing/src/mysql_routing.h index bc3ed1a..013d1dc 100644 --- a/src/routing/src/mysql_routing.h +++ b/src/routing/src/mysql_routing.h @@ -202,6 +202,8 @@ class MySQLRouting { bool block_client_host(const std::array &client_ip_array, const string &client_ip_str, int server = -1); + bool check_client_errors_time(const std::array &client_ip_array); + /** @brief Returns a copy of the list of blocked client hosts * * Returns a copy of the list of the blocked client hosts. diff --git a/src/routing/src/plugin_config.h b/src/routing/src/plugin_config.h index bd6a16c..4ba7944 100644 --- a/src/routing/src/plugin_config.h +++ b/src/routing/src/plugin_config.h @@ -53,7 +53,7 @@ class RoutingPluginConfig final : public mysqlrouter::BasePluginConfig { mode(get_option_mode(section, "mode")), max_connections(get_uint_option(section, "max_connections", 1)), max_connect_errors(get_uint_option(section, "max_connect_errors", 1, UINT32_MAX)), - max_connect_errors_timeout(get_uint_option(section, "max_connect_errors_timeout", 300, UINT32_MAX)), // 5 minutes for reset error attempts + max_connect_errors_timeout(get_uint_option(section, "max_connect_errors_timeout", 0, UINT32_MAX)), // 5 minutes for reset error attempts client_connect_timeout(get_uint_option(section, "client_connect_timeout", 2, 31536000)), net_buffer_length(get_uint_option(section, "net_buffer_length", 1024, 1048576)) { } diff --git a/src/routing/src/routing.cc b/src/routing/src/routing.cc index e629d91..0f61895 100644 --- a/src/routing/src/routing.cc +++ b/src/routing/src/routing.cc @@ -48,7 +48,7 @@ const int kDefaultDestinationConnectionTimeout = 1; const string kDefaultBindAddress = "127.0.0.1"; const unsigned int kDefaultNetBufferLength = 16384; // Default defined in latest MySQL Server const unsigned long long kDefaultMaxConnectErrors = 100; // Similar to MySQL Server -const unsigned long long kDefaultMaxConnectErrorsTimeout = 300; // 5 minutes +const unsigned long long kDefaultMaxConnectErrorsTimeout = 0; // 5 minutes const unsigned int kDefaultClientConnectTimeout = 9; // Default connect_timeout MySQL Server minus 1 const std::map kAccessModeNames = { From c0569ce492a27b0e97c41bba3b1c7a8ffd38e26b Mon Sep 17 00:00:00 2001 From: Stan P Date: Fri, 29 Apr 2016 20:48:08 +0300 Subject: [PATCH 4/5] Fix some errors with check socket --- src/routing/src/mysql_routing.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/routing/src/mysql_routing.cc b/src/routing/src/mysql_routing.cc index aac683b..6a6b09f 100644 --- a/src/routing/src/mysql_routing.cc +++ b/src/routing/src/mysql_routing.cc @@ -149,7 +149,7 @@ int copy_mysql_protocol_packets(int sender, int receiver, fd_set *readfds, // We got error from MySQL Server while handshaking // We do not consider this a failed handshake auto server_error = mysql_protocol::ErrorPacket(buffer); - if (check_socket_alive) { + if (check_socket_alive(receiver)) { write(receiver, server_error.data(), server_error.size()); } // receiver socket closed by caller @@ -224,7 +224,7 @@ bool MySQLRouting::block_client_host(const std::array &client_ip_ar if (server >= 0) { auto fake_response = mysql_protocol::HandshakeResponsePacket(1, {}, "ROUTER", "", "fake_router_login"); - if (check_socket_alive) { + if (check_socket_alive(server)) { write(server, fake_response.data(), fake_response.size()); } } @@ -251,7 +251,7 @@ void MySQLRouting::routing_select_thread(int client, const in6_addr client_addr) os << "Can't connect to MySQL server on "; os << "'" << bind_address_.addr << "'"; auto server_error = mysql_protocol::ErrorPacket(0, 2003, os.str(), "HY000"); - if (check_socket_alive) { + if (check_socket_alive(client)) { write(client, server_error.data(), server_error.size()); } @@ -390,7 +390,7 @@ void MySQLRouting::start() { std::stringstream os; os << "Too many connection errors from " << get_peer_name(sock_client).first; auto server_error = mysql_protocol::ErrorPacket(0, 1129, os.str(), "HY000"); - if (check_socket_alive) { + if (check_socket_alive(sock_client)) { write(sock_client, server_error.data(), server_error.size()); } close(sock_client); // no shutdown() before close() @@ -400,7 +400,7 @@ void MySQLRouting::start() { if (info_active_routes_.load(std::memory_order_relaxed) >= max_connections_) { auto server_error = mysql_protocol::ErrorPacket(0, 1040, "Too many connections", "HY000"); - if (check_socket_alive) { + if (check_socket_alive(sock_client)) { write(sock_client, server_error.data(), server_error.size()); } close(sock_client); // no shutdown() before close() From 66ade44c708bec031cc3c84648a75449c24d4d01 Mon Sep 17 00:00:00 2001 From: Stan P Date: Fri, 6 May 2016 00:37:39 +0300 Subject: [PATCH 5/5] Handle any sigpipe signals. Fix log string. Fix error counter for auth error --- src/routing/src/mysql_routing.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/routing/src/mysql_routing.cc b/src/routing/src/mysql_routing.cc index 6a6b09f..859ded1 100644 --- a/src/routing/src/mysql_routing.cc +++ b/src/routing/src/mysql_routing.cc @@ -22,6 +22,8 @@ #include "plugin_config.h" #include "mysqlrouter/routing.h" +#include "signal.h" + #include #include #include @@ -78,6 +80,11 @@ MySQLRouting::MySQLRouting(routing::AccessMode mode, int port, const string &bin } } +/* Catch Signal Handler functio */ +void signal_callback_handler(int signum){ + log_error("Unexpected error: caught signal SIGPIPE %d",signum); +} + bool check_socket_alive(int fd) { int error_code; socklen_t error_code_size = sizeof(error_code); @@ -211,6 +218,7 @@ bool MySQLRouting::check_client_errors_time(const std::array &clien bool MySQLRouting::block_client_host(const std::array &client_ip_array, const string &client_ip_str, int server) { bool blocked = false; + char *time_str; std::lock_guard lock(mutex_auth_errors_); struct tm *curtime = localtime(&auth_error_counters_[client_ip_array].last_attempt); auth_error_counters_[client_ip_array].last_attempt = std::time(0); @@ -218,8 +226,10 @@ bool MySQLRouting::block_client_host(const std::array &client_ip_ar log_warning("[%s] blocking client host %s", name.c_str(), client_ip_str.c_str()); blocked = true; } else { + time_str = asctime(curtime); + time_str[strlen(time_str)-1] = '\0'; log_info("[%s] %d authentication errors for %s (max %d). last attempt: %s", - name.c_str(), auth_error_counters_[client_ip_array].count, client_ip_str.c_str(), max_connect_errors_, asctime(curtime)); + name.c_str(), auth_error_counters_[client_ip_array].count, client_ip_str.c_str(), max_connect_errors_, time_str); } if (server >= 0) { @@ -340,6 +350,7 @@ void MySQLRouting::routing_select_thread(int client, const in6_addr client_addr) if (!handshake_done) { auto ip_array = in6_addr_to_array(client_addr); log_debug("[%s] Routing failed for %s: %s", name.c_str(), c_ip.first.c_str(), extra_msg.c_str()); + check_client_errors_time(ip_array); block_client_host(ip_array, c_ip.first.c_str(), server); } @@ -358,6 +369,7 @@ void MySQLRouting::start() { socklen_t sin_size = sizeof client_addr; char client_ip[INET6_ADDRSTRLEN]; int opt_nodelay = 1; + signal(SIGPIPE, signal_callback_handler); try { setup_service();