Bug #119567 signal 6 crashed when too many connections

• Submitted: 17 Dec 2:53
• Reporter: yuxiang jiang (OCA)
• Status: Open
• Category: MySQL Server: InnoDB storage engine
• Severity: S2 (Serious)
• Version: 8.0.30
• OS: Any
• CPU Architecture: Any

[17 Dec 2:53] yuxiang jiang

Description:
crash stack
#########
#0  0x00007fdd986aaaa1 in pthread_kill () from /lib64/libpthread.so.0
#1  0x000000000147909b in handle_fatal_signal (sig=6) at /data3/cynosdb-8.0.30/release/build_dir_145/sql/signal_handler.cc:236
#2  <signal handler called>
#3  0x00007fdd972d3387 in raise () from /lib64/libc.so.6
#4  0x00007fdd972d4a78 in abort () from /lib64/libc.so.6
#5  0x0000000000e419d2 in my_server_abort () at /data3/cynosdb-8.0.30/release/build_dir_145/sql/signal_handler.cc:292
#6  0x0000000002942c4a in my_abort () at /data3/cynosdb-8.0.30/release/build_dir_145/mysys/my_init.cc:258
#7  0x0000000002e67e0d in ut_dbg_assertion_failed(char const*, char const*, unsigned long) ()
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/ut/ut0dbg.cc:99
#8  0x0000000002df7144 in sync_array_get_nth_cell (arr=<optimized out>, n=<optimized out>)
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:147
#9  sync_array_get_nth_cell (arr=<optimized out>, n=<optimized out>)
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:146
#10 0x0000000002df9bb9 in sync_array_find_thread (thread=..., arr=<optimized out>)
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:490
#11 sync_array_deadlock_step (depth=<optimized out>, thread=..., arr=<optimized out>)
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:509
#12 sync_array_detect_rwlock_deadlock<sync_array_detect_deadlock_low(sync_array_t*, sync_cell_t*, size_t)::<lambda(auto:5, bool)> > (
    conflicts=<optimized out>, depth=0, arr=0x7fdd73a38260, cell=0x7fdd73b58028)
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:632
#13 sync_array_detect_deadlock_low (depth=0, cell=0x7fdd73b58028, arr=0x7fdd73a38260)
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:689
#14 sync_array_detect_deadlock(sync_array_t*, sync_cell_t*, unsigned long) ()
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:707
#15 0x0000000002dfa09a in sync_array_detect_deadlock () at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:296
#16 0x0000000002ddbe6a in srv_error_monitor_thread() () at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/srv/srv0srv.cc:2378
#17 0x0000000002ae30b4 in __invoke_impl<void, void (*&)()> (__f=<synthetic pointer>)
    at /opt/rh/devtoolset-10/root/usr/include/c++/10/bits/invoke.h:89
#18 __invoke<void (*&)()> (__fn=<synthetic pointer>) at /opt/rh/devtoolset-10/root/usr/include/c++/10/bits/invoke.h:95
#19 __call<void> (__args=<optimized out>, this=<synthetic pointer>) at /opt/rh/devtoolset-10/root/usr/include/c++/10/functional:416
#20 operator()<> (this=<synthetic pointer>) at /opt/rh/devtoolset-10/root/usr/include/c++/10/functional:499
#21 operator()<void (*)()> (
    f=<unknown type in /data1/mysql_root/base/mysql-cynos-3.1.16.003.1049-linux-x86_64_ts85/mysql/bin/mysqld, CU 0x1bf2c24d, DIE 0x1bfdb90b>, 
    this=0x7fa59dc46f90) at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/include/os0thread-create.h:194
#22 __invoke_impl<void, Detached_thread, void (*)()> (
    __f=<unknown type in /data1/mysql_root/base/mysql-cynos-3.1.16.003.1049-linux-x86_64_ts85/mysql/bin/mysqld, CU 0x1bf2c24d, DIE 0x1bfdb8e3>)
    at /opt/rh/devtoolset-10/root/usr/include/c++/10/bits/invoke.h:60
#23 __invoke<Detached_thread, void (*)()> (
---Type <return> to continue, or q <return> to quit---
    __fn=<unknown type in /data1/mysql_root/base/mysql-cynos-3.1.16.003.1049-linux-x86_64_ts85/mysql/bin/mysqld, CU 0x1bf2c24d, DIE 0x1bfdb8ae>)
    at /opt/rh/devtoolset-10/root/usr/include/c++/10/bits/invoke.h:95
#24 _M_invoke<0, 1> (this=0x7fa59dc46f88) at /opt/rh/devtoolset-10/root/usr/include/c++/10/thread:264
#25 operator() (this=0x7fa59dc46f88) at /opt/rh/devtoolset-10/root/usr/include/c++/10/thread:271
#26 std::thread::_State_impl<std::thread::_Invoker<std::tuple<Detached_thread, void (*)()> > >::_M_run (this=0x7fa59dc46f80)
    at /opt/rh/devtoolset-10/root/usr/include/c++/10/thread:215
#27 0x0000000003bac9c0 in execute_native_thread_routine ()
#28 0x00007fdd986a5ea5 in start_thread () from /lib64/libpthread.so.0
#29 0x00007fdd9739bb0d in clone () from /lib64/libc.so.6
#########

variables when crashing
#########
(gdb) fr 10
#10 0x0000000002df9bb9 in sync_array_find_thread (thread=..., arr=<optimized out>)
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:490
490     in /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc
(gdb) p i
$6 = 6400
#########
(gdb) p *arr
$5 = {n_reserved = 5409, n_cells = 6400, cells = 0x7fdd73b57fe0, mutex = {m_impl = {m_mutex = {m_mutex = {__data = {__lock = 2, __count = 0, 
            __owner = 46183, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, 
          __size = "\002\000\000\000\000\000\000\000g\264\000\000\001", '\000' <repeats 26 times>, __align = 2}}, m_policy = {m_count = {
          m_spins = 0, m_waits = 0, m_calls = 0, m_enabled = false}, m_id = LATCH_ID_SYNC_ARRAY_MUTEX}}, m_filename = 0x0, m_line = 0, 
    m_ptr = 0x0}, res_count = 378086, next_free_slot = 6400, first_free_slot = 358, last_scan = 5549}
(gdb) fr
#12 sync_array_detect_rwlock_deadlock<sync_array_detect_deadlock_low(sync_array_t*, sync_cell_t*, size_t)::<lambda(auto:5, bool)> > (
    conflicts=<optimized out>, depth=0, arr=0x7fdd73a38260, cell=0x7fdd73b58028)
    at /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc:632
632     in /data3/cynosdb-8.0.30/release/build_dir_145/storage/innobase/sync/sync0arr.cc
#########
(gdb) p sync_array_size
$8 = 16
#########

code analyze
We have observed that except for the function sync_array_find_thread, all other iterations over the array using next_free_slot employ the "<" operator for condition checking. Based on its usage patterns, next_free_slot is intended to indicate the next available slot, and that slot itself is in an unusable state. Therefore, it is suspected that the condition here should be modified to use "<" rather than "<=". 
#########
*** storage/innobase/sync/sync0arr.cc:
sync_array_validate[92]        for (ulint i = 0; i < arr->next_free_slot; i++) {
sync_array_t[118]              next_free_slot(),
sync_array_reserve_cell[187]   ut_ad(arr->first_free_slot < arr->next_free_slot);
sync_array_reserve_cell[190]   } else if (arr->next_free_slot < arr->n_cells) {
sync_array_reserve_cell[192]   cell = sync_array_get_nth_cell(arr, arr->next_free_slot);
sync_array_reserve_cell[193]   ++arr->next_free_slot;
sync_array_reserve_cell[205]   ut_ad(arr->next_free_slot <= arr->n_cells);
sync_array_free_cell[263]      if (arr->next_free_slot > arr->n_cells / 2 && arr->n_reserved == 0) {
sync_array_free_cell[265]      for (ulint i = 0; i < arr->next_free_slot; ++i) {
sync_array_free_cell[273]      arr->next_free_slot = 0;
sync_array_find_thread[487]    for (ulint i = 0; i <= arr->next_free_slot; i++) {
sync_array_wake_threads_if_sema_free_low[787] for (ulint i = 0; i < arr->next_free_slot; ++i) {
sync_array_print_long_waits_low[846] for (ulint i = 0; i < arr->next_free_slot; i++) {
#########

How to repeat:
Not found yet，randomly

Suggested fix:
static sync_cell_t *sync_array_find_thread(
    sync_array_t *arr,      /*!< in: wait array */
    std::thread::id thread) /*!< in: thread id */
{
  for (ulint i = 0; i < arr->next_free_slot; i++) {
    sync_cell_t *cell;

    cell = sync_array_get_nth_cell(arr, i);

    if (cell->latch.mutex != nullptr && cell->thread_id == thread) {
      return (cell); /* Found */
    }
  }

  return (nullptr); /* Not found */
}