Backtrace from the second sys outage at around 1730 on 2019-12-17 [root@server-2019 bfd]# grep -e '0x' core-bt.out |sort | uniq -c | sort -n | tail 8902 #10 0x0000000000f7d64f in Channel_info_tcpip_socket::create_thd ( 8902 #3 0x0000000000e797a5 in native_mutex_lock (mutex=) 8902 src_file=0x2daef70 "/export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc", src_line=2987, 8903 #14 0x00007f976b72ae65 in start_thread () from /lib64/libpthread.so.0 8903 #15 0x00007f97698b188d in clone () from /lib64/libc.so.6 8903 #8 0x0000000000df2936 in THD::THD(bool) () 8903 #9 0x00000000012b12ba in Channel_info::create_thd() () 10002 #0 0x00007f976b73150d in __lll_lock_wait () from /lib64/libpthread.so.0 10002 #1 0x00007f976b72ce91 in _L_lock_1093 () from /lib64/libpthread.so.0 10002 #2 0x00007f976b72ce32 in pthread_mutex_lock () from /lib64/libpthread.so.0 8902 threads waiting on native mutex lock in sql/sql_plugin.cc:2987 which is for acquiring the LOCK_global_system_variables mutex Sample one of them with gdb: (gdb) thread 10068 [Switching to thread 10068 (Thread 0x7f7551b6f700 (LWP 29666))] #0 0x00007f976b73150d in __lll_lock_wait () from /lib64/libpthread.so.0 (gdb) where #0 0x00007f976b73150d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f976b72ce91 in _L_lock_1093 () from /lib64/libpthread.so.0 #2 0x00007f976b72ce32 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000000e797a5 in native_mutex_lock (mutex=) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/include/thr_mutex.h:189 #4 my_mutex_lock (mp=) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/include/thr_mutex.h:189 #5 inline_mysql_mutex_lock ( src_file=0x2daef70 "/export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc", src_line=2987, that=) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/include/mysql/psi/mysql_mutex.h:250 #6 plugin_thdvar_init (thd=thd@entry=0x7f795817e2b0, enable_plugins=) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc:2987 #7 0x0000000000deb3bd in THD::init (this=this@entry=0x7f795817e2b0) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_class.cc:754 #8 0x0000000000df2936 in THD::THD(bool) () at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_class.cc:515 #9 0x00000000012b12ba in Channel_info::create_thd() () at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/channel_info.cc:45 #10 0x0000000000f7d64f in Channel_info_tcpip_socket::create_thd (this=0x4e8bc80) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/socket_connection.cc:238 #11 0x0000000000f7cd9c in init_new_thd (channel_info=0x4e8bc80) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/connection_handler_per_thread.cc:263 #12 handle_connection (arg=arg@entry=0x4e8bc80) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/connection_handler_per_thread.cc:263 #13 0x000000000240901c in pfs_spawn_thread (arg=0x4e8a4b0) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/storage/perfschema/pfs.cc:2854 #14 0x00007f976b72ae65 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f97698b188d in clone () from /lib64/libc.so.6 (gdb) Frame #5 confirms the thread is stuck in sql/sql_plugin.cc:2987. Let's see info on the mutex (gdb) print LOCK_global_system_variables $10 = {m_mutex = {m_u = {m_native = {__data = {__lock = 2, __count = 0, __owner = 25282, __nusers = 1, __kind = 3, __spins = 76, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\302b\000\000\001\000\000\000\003\000\000\000L", '\000' , __align = 2}, m_safe_ptr = 0x2}}, m_psi = 0x7f97575117c0} (gdb) It says it is owned by thread 25282. Let's find the corresponding thread in gdb: (gdb) thread find 25282 Thread 250 has target id 'Thread 0x7f7bfdedf700 (LWP 25282)' Switch to that thread and see where it is: (gdb) thread 250 [Switching to thread 250 (Thread 0x7f7bfdedf700 (LWP 25282))] #0 0x00007f976b73150d in __lll_lock_wait () from /lib64/libpthread.so.0 (gdb) where #0 0x00007f976b73150d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f976b72ce91 in _L_lock_1093 () from /lib64/libpthread.so.0 #2 0x00007f976b72ce32 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000000e798a0 in native_mutex_lock (mutex=0x3ad54c0 ) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/include/thr_mutex.h:189 #4 my_mutex_lock (mp=0x3ad54c0 ) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/include/thr_mutex.h:189 #5 inline_mysql_mutex_lock ( src_file=0x2daef70 "/export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc", src_line=2997, that=0x3ad54c0 ) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/include/mysql/psi/mysql_mutex.h:250 #6 plugin_thdvar_init (thd=thd@entry=0x7f7778143720, enable_plugins=) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc:2997 #7 0x0000000000deb3bd in THD::init (this=this@entry=0x7f7778143720) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_class.cc:754 #8 0x0000000000df2936 in THD::THD(bool) () at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_class.cc:515 #9 0x00000000012b12ba in Channel_info::create_thd() () at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/channel_info.cc:45 #10 0x0000000000f7d64f in Channel_info_tcpip_socket::create_thd (this=0x44de0f0) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/socket_connection.cc:238 #11 0x0000000000f7cd9c in init_new_thd (channel_info=0x44de0f0) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/connection_handler_per_thread.cc:263 #12 handle_connection (arg=arg@entry=0x3f37930) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/connection_handler_per_thread.cc:263 #13 0x000000000240901c in pfs_spawn_thread (arg=0x43546f0) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/storage/perfschema/pfs.cc:2854 #14 0x00007f976b72ae65 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f97698b188d in clone () from /lib64/libc.so.6 It seems to be stuck waiting on the LOCK_plugin mutex at sql/sql_plugin.cc:2997. (gdb) print LOCK_plugin $11 = {m_mutex = {m_u = {m_native = {__data = {__lock = 2, __count = 0, __owner = 64095, __nusers = 1, __kind = 3, __spins = 44, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000_\372\000\000\001\000\000\000\003\000\000\000,", '\000' , __align = 2}, m_safe_ptr = 0x2}}, m_psi = 0x7f9757513fc0} (gdb) thread find 64095 Thread 810 has target id 'Thread 0x7f79a9c07700 (LWP 64095)' (gdb) thread 810 [Switching to thread 810 (Thread 0x7f79a9c07700 (LWP 64095))] #0 0x00007f976b72e35e in pthread_rwlock_wrlock () from /lib64/libpthread.so.0 (gdb) where #0 0x00007f976b72e35e in pthread_rwlock_wrlock () from /lib64/libpthread.so.0 #1 0x0000000000e81ff4 in native_rw_wrlock (rwp=) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/include/thr_rwlock.h:102 #2 inline_mysql_rwlock_wrlock (that=, src_file=0x2daef70 "/export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc", src_line=2234) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/include/mysql/psi/mysql_rwlock.h:381 #3 mysql_install_plugin(THD*, MYSQL_LEX_CSTRING, MYSQL_LEX_STRING const*) () at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc:2234 #4 0x0000000000e829ce in Sql_cmd_install_plugin::execute (this=, thd=0x7f795412c100) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc:3669 #5 0x0000000000e599ba in mysql_execute_command(THD*, bool) () at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_parse.cc:4453 #6 0x0000000000e5ddfb in mysql_parse (thd=thd@entry=0x7f795412c100, parser_state=parser_state@entry=0x7f79a9c05be0) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_parse.cc:5257 #7 0x0000000000e607a8 in dispatch_command(THD*, COM_DATA const*, enum_server_command) () at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_parse.cc:1765 #8 0x0000000000e6125c in do_command (thd=thd@entry=0x7f795412c100) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_parse.cc:1273 #9 0x0000000000f7cf20 in handle_connection (arg=arg@entry=0x3f02220) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/conn_handler/connection_handler_per_thread.cc:302 #10 0x000000000240901c in pfs_spawn_thread (arg=0x4474890) at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/storage/perfschema/pfs.cc:2854 #11 0x00007f976b72ae65 in start_thread () from /lib64/libpthread.so.0 #12 0x00007f97698b188d in clone () from /lib64/libc.so.6 Stuck in sql/sql_plugin.cc:2234 waiting for the LOCK_system_variables_hash mutex. Also, it seems to be trying to install a plugin. Chase the lock: (gdb) print LOCK_system_variables_hash $12 = {m_rwlock = {__data = {__lock = 0, __nr_readers = 199, __readers_wakeup = 84, __writer_wakeup = 0, __nr_readers_queued = 0, __nr_writers_queued = 1, __writer = 0, __shared = 0, __pad1 = 0, __pad2 = 0, __flags = 0}, __size = "\000\000\000\000\307\000\000\000T", '\000' , "\001", '\000' , __align = 854698491904}, m_psi = 0x7f97574f03c0} There seem to be 199 readers from the lock, currently, so the write lock cannot be acquired. Try to find what plugin was being installed: (gdb) up 3 #3 mysql_install_plugin(THD*, MYSQL_LEX_CSTRING, MYSQL_LEX_STRING const*) () at /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc:2234 2234 /export/home/pb2/build/sb_0-36015269-1568970779.69/rpm/BUILD/mysql-8.0.18/mysql-8.0.18/sql/sql_plugin.cc: No such file or directory. (gdb) print thd $25 = (THD *) 0x7f795412c100 (gdb) p thd->m_query_string $26 = {str = 0x7f79540e0408 "INSTALL PLUGIN mysql_clone SONAME 'mysql_clone.so'", length = 50} So, it is the clone plugin.