----------------------------------------------------------------------------- 1. The MySQL thread 2590 waits for LOCK_status, which is held by the OS thread 20338 (__owner = 20338) (gdb) thr 2590 [Switching to thread 2590 (Thread 0x7f2eed33d700 (LWP 6527))] #0 0x00007f302d3d452d in __lll_lock_wait () from /lib64/libpthread.so.0 (gdb) bt #0 0x00007f302d3d452d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f302d3cfe41 in _L_lock_1012 () from /lib64/libpthread.so.0 #2 0x00007f302d3cfde2 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000000fe61b0 in native_mutex_lock (mutex=) at /data//include/thr_mutex.h:84 #4 my_mutex_lock (mp=) at /data//include/thr_mutex.h:182 #5 inline_mysql_mutex_lock (src_line=73, src_file=0x18df608 "/data//storage/perfschema/table_session_status.cc", that=) at /data//include/mysql/psi/mysql_thread.h:730 #6 table_session_status::get_row_count () at /data//storage/perfschema/table_session_status.cc:73 #7 0x0000000000f8e83f in ha_perfschema::info (this=0x7f2ed6e92638, flag=18) at /data//storage/perfschema/ha_perfschema.cc:394 #8 0x0000000000d0b19a in JOIN::init_planner_arrays (this=this@entry=0x7f2ee41ecc08) at /data//sql/sql_optimizer.cc:5198 #9 0x0000000000d128e4 in JOIN::make_join_plan (this=this@entry=0x7f2ee41ecc08) at /data//sql/sql_optimizer.cc:5020 #10 0x0000000000d1464c in JOIN::optimize (this=0x7f2ee41ecc08) at /data//sql/sql_optimizer.cc:368 #11 0x0000000000d5b73d in st_select_lex::optimize (this=this@entry=0x7f2ea0765100, thd=thd@entry=0x7f2ea06b7800) at /data//sql/sql_select.cc:1019 #12 0x0000000000da6722 in st_select_lex_unit::optimize (this=0x7f2ee5f85038, thd=thd@entry=0x7f2ea06b7800) at /data//sql/sql_union.cc:703 #13 0x0000000000ce248d in TABLE_LIST::optimize_derived (this=this@entry=0x7f2ee5f86288, thd=0x7f2ea06b7800) at /data//sql/sql_derived.cc:208 #14 0x0000000000d1431a in JOIN::optimize (this=0x7f2ee41ec838) at /data//sql/sql_optimizer.cc:202 #15 0x0000000000d5b73d in st_select_lex::optimize (this=this@entry=0x7f2ea0763858, thd=thd@entry=0x7f2ea06b7800) at /data//sql/sql_select.cc:1019 #16 0x0000000000d5b929 in handle_query (thd=thd@entry=0x7f2ea06b7800, lex=lex@entry=0x7f2ea06b9a70, result=result@entry=0x7f2ea07657a8, added_options=added_options@entry=0, removed_options=removed_options@entry=0) at /data//sql/sql_select.cc:166 #17 0x00000000007b35a9 in execute_sqlcom_select (thd=thd@entry=0x7f2ea06b7800, all_tables=) at /data//sql/sql_parse.cc:5681 #18 0x0000000000d1d259 in mysql_execute_command (thd=thd@entry=0x7f2ea06b7800, first_level=first_level@entry=true) at /data//sql/sql_parse.cc:3009 #19 0x0000000000d20e7d in mysql_parse (thd=thd@entry=0x7f2ea06b7800, parser_state=parser_state@entry=0x7f2eed33a520) at /data//sql/sql_parse.cc:6132 #20 0x0000000000d21ad6 in dispatch_command (thd=thd@entry=0x7f2ea06b7800, com_data=com_data@entry=0x7f2eed33b4a0, command=COM_QUERY) at /data//sql/sql_parse.cc:1542 #21 0x0000000000d235e7 in do_command (thd=0x7f2ea06b7800) at /data//sql/sql_parse.cc:1052 #22 0x00007f2ef0db6d63 in ?? () #23 0x00007f2ea06b7800 in ?? () #24 0xdbd96fad58f82500 in ?? () #25 0x00007f2ef0fc3e00 in ?? () #26 0x00007f2ef0fc3e00 in ?? () #27 0x0000000002179860 in on_off_default_names () #28 0x00007f2ea06dfdb8 in ?? () #29 0x00007f2eed33b5f0 in ?? () #30 0x00007f2eed33b5b0 in ?? () #31 0x00007f2eed33b690 in ?? () #32 0x00007f2ef0db7cce in ?? () #33 0x00007f2eed33b5f0 in ?? () #34 0x0000000000000000 in ?? () (gdb) f 6 #6 table_session_status::get_row_count () at /data//storage/perfschema/table_session_status.cc:73 73 /data//storage/perfschema/table_session_status.cc: No such file or directory. (gdb) p LOCK_status $1 = {m_mutex = {__data = {__lock = 2, __count = 0, __owner = 20338, __nusers = 1, __kind = 3, __spins = 65, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000rO\000\000\001\000\000\000\003\000\000\000A", '\000' , __align = 2}, m_psi = 0x0} ----------------------------------------------------------------------------------------------- 2. OS thread 20338 (mysql thread number 2568) is waiting for the srv_innodb_monitor_mutex lock Thread 2568 (Thread 0x7f2eecab9700 (LWP 20338)): #0 0x00007f302d3d1975 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000110ba11 in wait (this=) at /data//storage/innobase/os/os0event.cc:171 #2 os_event::wait_low (this=0x7f302b778478, reset_sig_count=) at /data//storage/innobase/os/os0event.cc:358 #3 0x000000000110c35a in os_event_wait_low (event=, reset_sig_count=) at /data//storage/innobase/os/os0event.cc:603 #4 0x00000000011ceb07 in sync_array_wait_event (arr=arr@entry=0x7f302b6a8c18, cell=@0x7f2eecab3da0: 0x7f30295ee758) at /data//storage/innobase/sync/sync0arr.cc:475 #5 0x00000000011b82dc in TTASEventMutex::wait (this=this@entry=0x22b8e80 , filename=filename@entry=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", line=line@entry=1418, spin=4) at /data//storage/innobase/include/ut0mutex.ic:89 #6 0x00000000011badae in spin_and_try_lock (line=1418, filename=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", max_delay=, max_spins=60, this=0x22b8e80 ) at /data//storage/innobase/include/ib0mutex.h:850 #7 enter (line=1418, filename=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", max_delay=, max_spins=, this=0x22b8e80 ) at /data//storage/innobase/include/ib0mutex.h:707 #8 enter (name=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", line=1418, n_delay=, n_spins=, this=0x22b8e80 ) at /data//storage/innobase/include/ib0mutex.h:987 #9 srv_export_innodb_status () at /data//storage/innobase/srv/srv0srv.cc:1418 #10 0x000000000105e81a in innodb_export_status () at /data//storage/innobase/handler/ha_innodb.cc:16689 #11 show_innodb_vars (thd=, var=0x7f2eecab3f70, buff=) at /data//storage/innobase/handler/ha_innodb.cc:19806 #12 0x0000000000fbd077 in PFS_status_variable_cache::manifest (this=this@entry=0x7f2ed26a6068, thd=0x7f2edf3ba800, show_var_array=, status_vars=status_vars@entry=0x7f2eecab4880, prefix=prefix@entry=0x1831230 "", nested_array=nested_array@entry=false, strict=strict@entry=true) at /data//storage/perfschema/pfs_variable.cc:1204 #13 0x0000000000fbdaa9 in PFS_status_variable_cache::do_materialize_global (this=0x7f2ed26a6068) at /data//storage/perfschema/pfs_variable.cc:986 #14 0x0000000000fe5843 in materialize_global (this=0x7f2ed26a6068) at /data//storage/perfschema/pfs_variable.h:518 #15 table_global_status::rnd_init (this=0x7f2ed26a6040, scan=) at /data//storage/perfschema/table_global_status.cc:105 #16 0x0000000000f8e66c in ha_perfschema::rnd_init (this=0x7f2ef4ac8038, scan=) at /data//storage/perfschema/ha_perfschema.cc:324 #17 0x000000000084c0b9 in handler::ha_rnd_init (this=0x7f2ef4ac8038, scan=scan@entry=true) at /data//sql/handler.cc:2961 #18 0x0000000000c7e40b in init_read_record (info=info@entry=0x7f2ee65d5c48, thd=0x7f2edf3ba800, table=0x7f2ef4abe028, table@entry=0x0, qep_tab=qep_tab@entry=0x7f2ee65d5bf8, use_record_cache=use_record_cache@entry=1, print_error=print_error@entry=true, disable_rr_cache=, disable_rr_cache@entry=false) at /data//sql/records.cc:314 #19 0x0000000000ce9fee in join_init_read_record (tab=0x7f2ee65d5bf8) at /data//sql/sql_executor.cc:2524 #20 0x0000000000cedc53 in sub_select (join=0x7f2ee65d5518, qep_tab=0x7f2ee65d5bf8, end_of_records=) at /data//sql/sql_executor.cc:1303 #21 0x0000000000ce67b6 in do_select (join=0x7f2ee65d5518) at /data//sql/sql_executor.cc:950 #22 JOIN::exec (this=this@entry=0x7f2ee65d5518) at /data//sql/sql_executor.cc:199 #23 0x0000000000ce2414 in TABLE_LIST::materialize_derived (this=this@entry=0x7f2ee65d45c0, thd=thd@entry=0x7f2edf3ba800) at /data//sql/sql_derived.cc:326 #24 0x0000000000ce71df in join_materialize_derived (tab=) at /data//sql/sql_executor.cc:2546 #25 0x0000000000ce6cd2 in QEP_TAB::prepare_scan (this=0x7f2f0e0f83a8) at /data//sql/sql_executor.cc:1362 #26 0x0000000000cedb7a in sub_select (join=0x7f2ee65d5138, qep_tab=0x7f2f0e0f83a8, end_of_records=) at /data//sql/sql_executor.cc:1254 #27 0x0000000000ce67b6 in do_select (join=0x7f2ee65d5138) at /data//sql/sql_executor.cc:950 #28 JOIN::exec (this=0x7f2ee65d5138) at /data//sql/sql_executor.cc:199 #29 0x0000000000d5b951 in handle_query (thd=thd@entry=0x7f2edf3ba800, lex=lex@entry=0x7f2edf3bca70, result=result@entry=0x7f2ee6597708, added_options=added_options@entry=0, removed_options=removed_options@entry=0) at /data//sql/sql_select.cc:186 #30 0x00000000007b35a9 in execute_sqlcom_select (thd=thd@entry=0x7f2edf3ba800, all_tables=) at /data//sql/sql_parse.cc:5681 #31 0x0000000000d1d259 in mysql_execute_command (thd=thd@entry=0x7f2edf3ba800, first_level=first_level@entry=true) at /data//sql/sql_parse.cc:3009 #32 0x0000000000d20e7d in mysql_parse (thd=thd@entry=0x7f2edf3ba800, parser_state=parser_state@entry=0x7f2eecab6520) at /data//sql/sql_parse.cc:6132 #33 0x0000000000d21ad6 in dispatch_command (thd=thd@entry=0x7f2edf3ba800, com_data=com_data@entry=0x7f2eecab74a0, command=COM_QUERY) at /data//sql/sql_parse.cc:1542 #34 0x0000000000d235e7 in do_command (thd=0x7f2edf3ba800) at /data//sql/sql_parse.cc:1052 #35 0x00007f2ef0db6d63 in ?? () #36 0x00007f2edf3ba800 in ?? () #37 0xdbd96fad58f82500 in ?? () #38 0x00007f2ee3cb2b48 in ?? () #39 0x00007f2ef0fc5800 in ?? () #40 0x0000000002179860 in on_off_default_names () #41 0x00007f2edde0aa58 in ?? () #42 0x00007f2eecab75f0 in ?? () #43 0x00007f2eecab75b0 in ?? () #44 0x00007f2eecab7690 in ?? () #45 0x00007f2ef0db7cce in ?? () #46 0x00007f2eecab75f0 in ?? () #47 0x0000000000000000 in ?? () ----------------------------------------------------------------------------------------------------- 3. The thread srv_error_monitor_thread is also waiting for the srv_innodb_monitor_mutex. No thread holds the lock.(__owner=0). (gdb) thr 2648 [Switching to thread 2648 (Thread 0x7f2ef7bff700 (LWP 20799))] #0 0x00007f302d3d1975 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 (gdb) bt #0 0x00007f302d3d1975 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000110ba11 in wait (this=) at /data//storage/innobase/os/os0event.cc:171 #2 os_event::wait_low (this=0x7f302b778478, reset_sig_count=) at /data//storage/innobase/os/os0event.cc:358 #3 0x000000000110c35a in os_event_wait_low (event=, reset_sig_count=) at /data//storage/innobase/os/os0event.cc:603 #4 0x00000000011ceb07 in sync_array_wait_event (arr=arr@entry=0x7f302b6a8c18, cell=@0x7f2ef7bfd3f0: 0x7f30295ee458) at /data//storage/innobase/sync/sync0arr.cc:475 #5 0x00000000011b82dc in TTASEventMutex::wait (this=this@entry=0x22b8e80 , filename=filename@entry=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", line=line@entry=1171, spin=4) at /data//storage/innobase/include/ut0mutex.ic:89 #6 0x00000000011bb94a in spin_and_try_lock (line=1171, filename=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", max_delay=, max_spins=60, this=0x22b8e80 ) at /data//storage/innobase/include/ib0mutex.h:850 #7 enter (line=1171, filename=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", max_delay=, max_spins=, this=0x22b8e80 ) at /data//storage/innobase/include/ib0mutex.h:707 #8 enter (name=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", line=1171, n_delay=, n_spins=, this=0x22b8e80 ) at /data//storage/innobase/include/ib0mutex.h:987 #9 srv_refresh_innodb_monitor_stats () at /data//storage/innobase/srv/srv0srv.cc:1171 #10 srv_error_monitor_thread (arg=) at /data//storage/innobase/srv/srv0srv.cc:1758 #11 0x00007f302d3cddf5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007f302be8648d in clone () from /lib64/libc.so.6 (gdb) f 6 #6 0x00000000011bb94a in spin_and_try_lock (line=1171, filename=0x1907908 "/data//storage/innobase/srv/srv0srv.cc", max_delay=, max_spins=60, this=0x22b8e80 ) at /data//storage/innobase/include/ib0mutex.h:850 850 /data//storage/innobase/include/ib0mutex.h: No such file or directory. (gdb) p *this $2 = {m_lock_word = 1, m_waiters = 1, m_event = 0x7f302b778478, m_policy = {m_count = {m_spins = 0, m_waits = 0, m_calls = 0, m_enabled = false}, m_id = LATCH_ID_SRV_INNODB_MONITOR}} (gdb) p *this->m_event $3 = {m_set = false, signal_count = 1, mutex = {m_mutex = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 3, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' , "\003", '\000' , __align = 0}}, cond_var = {__data = {__lock = 0, __futex = 3, __total_seq = 3, __wakeup_seq = 0, __woken_seq = 0, __mutex = 0x7f302b778488, __nwaiters = 7, __broadcast_seq = 0}, __size = "\000\000\000\000\003\000\000\000\003", '\000' , "\210\204w+0\177\000\000\a\000\000\000\000\000\000", __align = 12884901888}, static cond_attr = {__size = "\002\000\000", __align = 2}, static cond_attr_has_monotonic_clock = true, Python Exception Cannot find type event_iter_t::_Node: static global_initialized = true, event_iter = } ========================================================== Note: I think this problem is related to memoryBarrier