stack info: Thread 77 (Thread 0x7f489be89700 (LWP 177094)): #0 0x00007f48a5c32727 in sigwaitinfo () from /lib64/libc.so.6 #1 0x0000000000eced3b in ?? () #2 0x0000000000edeb84 in pfs_spawn_thread () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 76 (Thread 0x7f2d73162700 (LWP 177130)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 75 (Thread 0x7f2d72761700 (LWP 177131)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 74 (Thread 0x7f2d71d60700 (LWP 177132)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 73 (Thread 0x7f2d7135f700 (LWP 177133)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 72 (Thread 0x7f2d7095e700 (LWP 177134)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 71 (Thread 0x7f2d6ff5d700 (LWP 177135)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 70 (Thread 0x7f2d6f55c700 (LWP 177136)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 69 (Thread 0x7f2d6eb5b700 (LWP 177137)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 68 (Thread 0x7f2d6e15a700 (LWP 177138)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 67 (Thread 0x7f2d6d759700 (LWP 177139)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 66 (Thread 0x7f2d6cd58700 (LWP 177140)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 65 (Thread 0x7f2d6c357700 (LWP 177141)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 64 (Thread 0x7f2d6b956700 (LWP 177142)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 63 (Thread 0x7f2d6af55700 (LWP 177143)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 62 (Thread 0x7f2d6a554700 (LWP 177144)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 61 (Thread 0x7f2d69b53700 (LWP 177145)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 60 (Thread 0x7f2d69152700 (LWP 177146)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 59 (Thread 0x7f2d68751700 (LWP 177147)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 58 (Thread 0x7f2d67d50700 (LWP 177148)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 57 (Thread 0x7f2d6734f700 (LWP 177149)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 56 (Thread 0x7f2d6694e700 (LWP 177150)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 55 (Thread 0x7f2d65f4d700 (LWP 177151)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 54 (Thread 0x7f2d6554c700 (LWP 177152)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 53 (Thread 0x7f2d64b4b700 (LWP 177153)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 52 (Thread 0x7f2d6414a700 (LWP 177154)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 51 (Thread 0x7f2d63749700 (LWP 177155)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 50 (Thread 0x7f2d62d48700 (LWP 177156)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 49 (Thread 0x7f2d62347700 (LWP 177157)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 48 (Thread 0x7f2d61946700 (LWP 177158)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 47 (Thread 0x7f2d60f45700 (LWP 177159)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 46 (Thread 0x7f2d60544700 (LWP 177160)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 45 (Thread 0x7f2d5fb43700 (LWP 177161)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 44 (Thread 0x7f2d5f142700 (LWP 177162)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 43 (Thread 0x7f2d5e741700 (LWP 177163)): #0 0x00007f48a77e8614 in ?? () from /lib64/libaio.so.1 #1 0x0000000001004222 in LinuxAIOHandler::collect() () #2 0x0000000001004bdd in LinuxAIOHandler::poll(fil_node_t**, void**, IORequest*) () #3 0x0000000001007f40 in os_aio_handler(unsigned long, fil_node_t**, void**, IORequest*) () #4 0x00000000011d7150 in fil_aio_wait(unsigned long) () #5 0x00000000010bc190 in io_handler_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 42 (Thread 0x7f2d5dd40700 (LWP 177164)): #0 0x00007f48a79f4a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000010094e3 in os_event::timed_wait(timespec const*) () #2 0x0000000001009710 in os_event::wait_time_low(unsigned long, long) () #3 0x000000000117c8b8 in buf_flush_page_cleaner_coordinator () #4 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 41 (Thread 0x7f2d5d33f700 (LWP 177165)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x000000000117bf2a in buf_flush_page_cleaner_worker () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 40 (Thread 0x7f2d57fff700 (LWP 177166)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x000000000117bf2a in buf_flush_page_cleaner_worker () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 39 (Thread 0x7f2d575fe700 (LWP 177167)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x000000000117bf2a in buf_flush_page_cleaner_worker () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 38 (Thread 0x7f2d56bfd700 (LWP 177168)): #0 0x00007f48a79f800d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000001009db0 in os_thread_sleep(unsigned long) () #2 0x000000000117b549 in buf_lru_manager () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 37 (Thread 0x7f2d561fc700 (LWP 177169)): #0 0x00007f48a79f800d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000001009db0 in os_thread_sleep(unsigned long) () #2 0x000000000117b549 in buf_lru_manager () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 36 (Thread 0x7f2d557fb700 (LWP 177170)): #0 0x00007f48a79f800d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000001009db0 in os_thread_sleep(unsigned long) () #2 0x000000000117b549 in buf_lru_manager () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 35 (Thread 0x7f2d54dfa700 (LWP 177171)): #0 0x00007f48a79f800d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000001009db0 in os_thread_sleep(unsigned long) () #2 0x000000000117b549 in buf_lru_manager () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 34 (Thread 0x7f2d534f4700 (LWP 177172)): #0 0x00007f48a79f4a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000010094e3 in os_event::timed_wait(timespec const*) () #2 0x0000000001009710 in os_event::wait_time_low(unsigned long, long) () #3 0x00000000010b5601 in srv_monitor_thread () #4 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 33 (Thread 0x7f2e0dd52700 (LWP 177316)): #0 0x00007f48a79f4a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000010094e3 in os_event::timed_wait(timespec const*) () #2 0x0000000001009710 in os_event::wait_time_low(unsigned long, long) () #3 0x0000000000fcf027 in lock_wait_timeout_thread () #4 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 32 (Thread 0x7f2e0d351700 (LWP 177317)): #0 0x00007f48a79f4a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000010094e3 in os_event::timed_wait(timespec const*) () #2 0x0000000001009710 in os_event::wait_time_low(unsigned long, long) () #3 0x00000000010b5aa2 in srv_error_monitor_thread () #4 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 31 (Thread 0x7f2e07fff700 (LWP 177318)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x00000000010ca411 in sync_array_wait_event(sync_array_t*, sync_cell_t*&) () #3 0x0000000000f851ee in PolicyMutex >::enter(unsigned int, unsigned int, char const*, unsigned int) () #4 0x0000000000fbb9e8 in ibuf_merge_in_background(bool) () #5 0x00000000010b769c in srv_master_thread () #6 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 30 (Thread 0x7f2e075fe700 (LWP 177319)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x00000000010b971d in srv_purge_coordinator_thread () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 29 (Thread 0x7f2e06bfd700 (LWP 177320)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x00000000010b7ec8 in srv_worker_thread () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 28 (Thread 0x7f2e061fc700 (LWP 177321)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x00000000010b7ec8 in srv_worker_thread () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 27 (Thread 0x7f2e057fb700 (LWP 177322)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x00000000010b7ec8 in srv_worker_thread () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 26 (Thread 0x7f2e04dfa700 (LWP 177323)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x00000000010ca411 in sync_array_wait_event(sync_array_t*, sync_cell_t*&) () #3 0x00000000010cc01b in rw_lock_s_lock_spin(rw_lock_t*, unsigned long, char const*, unsigned long) () #4 0x0000000001155f8d in ?? () #5 0x000000000115d17b in buf_page_get_gen(page_id_t const&, page_size_t const&, unsigned long, buf_block_t*, unsigned long, char const*, unsigned long, mtr_t*, bool) () #6 0x000000000113bb4c in btr_cur_search_to_nth_level(dict_index_t*, unsigned long, dtuple_t const*, page_cur_mode_t, unsigned long, btr_cur_t*, unsigned long, char const*, unsigned long, mtr_t*) () #7 0x0000000001145096 in btr_pcur_open_on_user_rec_func(dict_index_t*, dtuple_t const*, page_cur_mode_t, unsigned long, btr_pcur_t*, char const*, unsigned long, mtr_t*) () #8 0x0000000000fbc6e4 in ibuf_merge_or_delete_for_page(buf_block_t*, page_id_t const&, page_size_t const*, unsigned long) () #9 0x000000000115ef1c in buf_page_io_complete(buf_page_t*, bool) () #10 0x000000000118b920 in ?? () #11 0x000000000118d74c in buf_read_page_background(page_id_t const&, page_size_t const&, bool) () #12 0x0000000001171a2c in ?? () #13 0x0000000001172005 in buf_dump_thread () #14 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 25 (Thread 0x7f2df7fff700 (LWP 177324)): #0 0x00007f48a79f4a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000010094e3 in os_event::timed_wait(timespec const*) () #2 0x0000000001009710 in os_event::wait_time_low(unsigned long, long) () #3 0x00000000011cfbb9 in dict_stats_thread () #4 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 24 (Thread 0x7f2def5fe700 (LWP 177325)): #0 0x00007f48a79f4a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000010094e3 in os_event::timed_wait(timespec const*) () #2 0x0000000001009710 in os_event::wait_time_low(unsigned long, long) () #3 0x000000000111dd7f in ib_wqueue_timedwait(ib_wqueue_t*, long) () #4 0x0000000001227513 in fts_optimize_thread(void*) () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 23 (Thread 0x7f2df75fe700 (LWP 177326)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x0000000001164418 in buf_resize_thread () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 22 (Thread 0x7f2e0c950700 (LWP 177327)): #0 0x00007f48a79f8585 in sigwait () from /lib64/libpthread.so.0 #1 0x000000000078d7cb in signal_hand () #2 0x0000000000edeb84 in pfs_spawn_thread () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 21 (Thread 0x7f2df6bfd700 (LWP 177328)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e2d073 in compress_gtid_table () #2 0x0000000000edeb84 in pfs_spawn_thread () #3 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 20 (Thread 0x7f2e0c8cf700 (LWP 177331)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000d6b63a in Per_thread_connection_handler::block_until_new_connection() () #2 0x0000000000d6b7cf in handle_connection () #3 0x0000000000edeb84 in pfs_spawn_thread () #4 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 19 (Thread 0x7f2e0c84e700 (LWP 177416)): #0 0x00007f48a79f77cd in write () from /lib64/libpthread.so.0 #1 0x0000000000ec58e8 in my_write () #2 0x0000000000eac66e in my_b_flush_io_cache () #3 0x0000000000e5b1ae in MYSQL_BIN_LOG::flush_and_sync(bool) () #4 0x0000000000e5b1f5 in MYSQL_BIN_LOG::after_append_to_relay_log(Master_info*) () #5 0x0000000000e838a9 in queue_event(Master_info*, char const*, unsigned long) () #6 0x0000000000e85b15 in handle_slave_io () #7 0x0000000000edeb84 in pfs_spawn_thread () #8 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #9 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 18 (Thread 0x7f2e0c7cd700 (LWP 177417)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9e55c in Mts_submode_logical_clock::wait_for_last_committed_trx(Relay_log_info*, long long, long long) () #2 0x0000000000e9e8d3 in Mts_submode_logical_clock::schedule_next_event(Relay_log_info*, Log_event*) () #3 0x0000000000e38711 in schedule_next_event(Log_event*, Relay_log_info*) () #4 0x0000000000e38b6e in Log_event::get_slave_worker(Relay_log_info*) () #5 0x0000000000e39590 in Log_event::apply_event(Relay_log_info*) () #6 0x0000000000e805d6 in apply_event_and_update_pos(Log_event**, THD*, Relay_log_info*) () #7 0x0000000000e8c98c in handle_slave_sql () #8 0x0000000000edeb84 in pfs_spawn_thread () #9 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 17 (Thread 0x7f2e0c74c700 (LWP 177418)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000000000100963b in os_event::wait_low(long) () #2 0x00000000010ca411 in sync_array_wait_event(sync_array_t*, sync_cell_t*&) () #3 0x00000000010cc01b in rw_lock_s_lock_spin(rw_lock_t*, unsigned long, char const*, unsigned long) () #4 0x000000000115d2d8 in buf_page_get_gen(page_id_t const&, page_size_t const&, unsigned long, buf_block_t*, unsigned long, char const*, unsigned long, mtr_t*, bool) () #5 0x0000000001206826 in flst_remove(unsigned char*, unsigned char*, mtr_t*) () #6 0x0000000001126be9 in btr_page_alloc(dict_index_t*, unsigned long, unsigned char, unsigned long, mtr_t*, mtr_t*) () #7 0x000000000112a454 in btr_page_split_and_insert(unsigned long, btr_cur_t*, unsigned long**, mem_block_info_t**, dtuple_t const*, unsigned long, mtr_t*) () #8 0x00000000011387ea in btr_cur_pessimistic_insert(unsigned long, btr_cur_t*, unsigned long**, mem_block_info_t**, dtuple_t*, unsigned char**, big_rec_t**, unsigned long, que_thr_t*, mtr_t*) () #9 0x0000000000fbac84 in ?? () #10 0x0000000000fbb854 in ibuf_insert(ibuf_op_t, dtuple_t const*, dict_index_t*, page_id_t const&, page_size_t const&, que_thr_t*) () #11 0x000000000113cc98 in btr_cur_search_to_nth_level(dict_index_t*, unsigned long, dtuple_t const*, page_cur_mode_t, unsigned long, btr_cur_t*, unsigned long, char const*, unsigned long, mtr_t*) () #12 0x000000000104cb01 in row_ins_sec_index_entry_low(unsigned long, unsigned long, dict_index_t*, mem_block_info_t*, mem_block_info_t*, dtuple_t*, unsigned long, que_thr_t*, bool) () #13 0x0000000001050fda in row_ins_sec_index_entry(dict_index_t*, dtuple_t*, que_thr_t*, bool) () #14 0x0000000001053490 in row_ins_step(que_thr_t*) () #15 0x000000000106698b in ?? () #16 0x0000000000f7c0a1 in ha_innobase::write_row(unsigned char*) () #17 0x0000000000803cb2 in handler::ha_write_row(unsigned char*) () #18 0x0000000000e42cf5 in Write_rows_log_event::write_row(Relay_log_info const*, bool) () #19 0x0000000000e43019 in Write_rows_log_event::do_exec_row(Relay_log_info const*) () #20 0x0000000000e2f506 in Rows_log_event::do_apply_row(Relay_log_info const*) () #21 0x0000000000e40ede in Rows_log_event::do_apply_event(Relay_log_info const*) () #22 0x0000000000e9cf0f in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #23 0x0000000000e801bb in handle_slave_worker () #24 0x0000000000edeb84 in pfs_spawn_thread () #25 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #26 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 16 (Thread 0x7f2e0c6cb700 (LWP 177419)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x7f2e0c64a700 (LWP 177420)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7f2e0c5c9700 (LWP 177421)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x7f2e0c548700 (LWP 177422)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 12 (Thread 0x7f2e0c4c7700 (LWP 177423)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7f2e0c446700 (LWP 177424)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7f2e0c3c5700 (LWP 177425)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7f2e0c344700 (LWP 177426)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x7f2e0c2c3700 (LWP 177427)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x7f2e0c242700 (LWP 177428)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7f2e0c1c1700 (LWP 177429)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7f2e0c140700 (LWP 177430)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7f2e0c0bf700 (LWP 177431)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7f2e043f9700 (LWP 177432)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7f2e04378700 (LWP 177433)): #0 0x00007f48a79f468c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000e9b244 in pop_jobs_item(Slave_worker*, slave_job_item*) () #2 0x0000000000e9cde3 in slave_worker_exec_job_group(Slave_worker*, Relay_log_info*) () #3 0x0000000000e801bb in handle_slave_worker () #4 0x0000000000edeb84 in pfs_spawn_thread () #5 0x00007f48a79f0aa1 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f48a5ce7c4d in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7f48a7e117e0 (LWP 177086)): #0 0x00007f48a5cde403 in poll () from /lib64/libc.so.6 #1 0x0000000000d6d73c in Mysqld_socket_listener::listen_for_connection_event() () #2 0x00000000007973e8 in mysqld_main(int, char**) () #3 0x00007f48a5c1dd20 in __libc_start_main () from /lib64/libc.so.6 #4 0x000000000078b195 in _start () mysql.err 2019-08-07T12:55:54.797249+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849490904832 has waited at ibuf0ibuf.cc line 4582 for 241.00 seconds the semaphore: S-lock on RW-latch at 0x7f4b9cf882b0 created in file buf0buf.cc line 1425 a writer (thread id 139849051821824) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: fffffffff0000000 Last time read locked in file ibuf0ibuf.cc line 3468 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/btr/btr0btr.cc line 179 2019-08-07T12:55:54.797320+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849543354112 has waited at ibuf0ibuf.cc line 2731 for 241.00 seconds the semaphore: Mutex at 0x1dfb880, Mutex IBUF created ibuf0ibuf.cc:527, lock var 1 2019-08-07T12:55:54.797346+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849051821824 has waited at buf0buf.cc line 4078 for 241.00 seconds the semaphore: S-lock on RW-latch at 0x7f382765e890 created in file buf0buf.cc line 1425 a writer (thread id 139963820316640) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: 0 Last time read locked in file not yet reserved line 0 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/buf/buf0buf.cc line 5334 InnoDB: ###### Starts InnoDB Monitor for 30 secs to print diagnostic info: InnoDB: Pending preads 0, pwrites 0 ===================================== 2019-08-07 12:56:04 0x7f30892f7700 INNODB MONITOR OUTPUT ===================================== Per second averages calculated from the last 33 seconds ----------------- BACKGROUND THREAD ----------------- srv_master_thread loops: 36 srv_active, 0 srv_shutdown, 26 srv_idle srv_master_thread log flush and writes: 61 ---------- SEMAPHORES ---------- OS WAIT ARRAY INFO: reservation count 77273 --Thread 139849490904832 has waited at ibuf0ibuf.cc line 4582 for 251.00 seconds the semaphore: S-lock on RW-latch at 0x7f4b9cf882b0 created in file buf0buf.cc line 1425 a writer (thread id 139849051821824) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: fffffffff0000000 Last time read locked in file ibuf0ibuf.cc line 3468 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/btr/btr0btr.cc line 179 --Thread 139846611470080 has waited at buf0flu.cc line 1229 for 74.00 seconds the semaphore: SX-lock on RW-latch at 0x7f3e721d2fb0 created in file buf0buf.cc line 1425 a writer (thread id 139849051821824) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: 0 Last time read locked in file btr0cur.cc line 378 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/btr/btr0cur.cc line 302 --Thread 139849543354112 has waited at ibuf0ibuf.cc line 2731 for 251.00 seconds the semaphore: Mutex at 0x1dfb880, Mutex IBUF created ibuf0ibuf.cc:527, lock var 1 --Thread 139849051821824 has waited at buf0buf.cc line 4078 for 251.00 seconds the semaphore: S-lock on RW-latch at 0x7f382765e890 created in file buf0buf.cc line 1425 a writer (thread id 139963820316640) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: 0 Last time read locked in file not yet reserved line 0 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/buf/buf0buf.cc line 5334 OS WAIT ARRAY INFO: signal count 45736 RW-shared spins 0, rounds 9564, OS waits 1629 RW-excl spins 0, rounds 2741, OS waits 44 RW-sx spins 176, rounds 753, OS waits 8 Spin rounds per wait: 9564.00 RW-shared, 2741.00 RW-excl, 4.28 RW-sx ------------ TRANSACTIONS ------------ Trx id counter 1272726148 Purge done for trx's n:o < 1272726148 undo n:o < 0 state: running but idle History list length 26 LIST OF TRANSACTIONS FOR EACH SESSION: ---TRANSACTION 421438269377720, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269376592, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269375464, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269374336, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269373208, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269372080, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269370952, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269369824, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269368696, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269367568, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269366440, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269365312, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269364184, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269363056, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269361928, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269360800, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269358544, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269357416, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 421438269355160, not started 0 lock struct(s), heap size 1136, 0 row lock(s) ---TRANSACTION 1272718759, ACTIVE 251 sec inserting mysql tables in use 1, locked 1 1 lock struct(s), heap size 1136, 0 row lock(s), undo log entries 1 MySQL thread id 6, OS thread handle 139849051821824, query id 266869 System lock INSERT INTO mbk_redpacket_user_freeride(user_id,order_id,is_valid, type, start_timestamp,end_timestamp,mt_userid,city_code ) VALUES('7450313500641429314560539522', 'ORD1564498013938588109522', 1, 200, 1564498022247, 1567094399247, '327238989', '0577') -------- FILE I/O -------- I/O thread 0 state: waiting for completed aio requests (insert buffer thread) I/O thread 1 state: waiting for completed aio requests (log thread) I/O thread 2 state: waiting for completed aio requests (read thread) I/O thread 3 state: waiting for completed aio requests (read thread) I/O thread 4 state: waiting for completed aio requests (read thread) I/O thread 5 state: waiting for completed aio requests (read thread) I/O thread 6 state: waiting for completed aio requests (read thread) I/O thread 7 state: waiting for completed aio requests (read thread) I/O thread 8 state: waiting for completed aio requests (read thread) I/O thread 9 state: waiting for completed aio requests (read thread) I/O thread 10 state: waiting for completed aio requests (read thread) I/O thread 11 state: waiting for completed aio requests (read thread) I/O thread 12 state: waiting for completed aio requests (read thread) I/O thread 13 state: waiting for completed aio requests (read thread) I/O thread 14 state: waiting for completed aio requests (read thread) I/O thread 15 state: waiting for completed aio requests (read thread) I/O thread 16 state: waiting for completed aio requests (read thread) I/O thread 17 state: waiting for completed aio requests (read thread) I/O thread 18 state: waiting for completed aio requests (write thread) I/O thread 19 state: waiting for completed aio requests (write thread) I/O thread 20 state: waiting for completed aio requests (write thread) I/O thread 21 state: waiting for completed aio requests (write thread) I/O thread 22 state: waiting for completed aio requests (write thread) I/O thread 23 state: waiting for completed aio requests (write thread) I/O thread 24 state: waiting for completed aio requests (write thread) I/O thread 25 state: waiting for completed aio requests (write thread) I/O thread 26 state: waiting for completed aio requests (write thread) I/O thread 27 state: waiting for completed aio requests (write thread) I/O thread 28 state: waiting for completed aio requests (write thread) I/O thread 29 state: waiting for completed aio requests (write thread) I/O thread 30 state: waiting for completed aio requests (write thread) I/O thread 31 state: waiting for completed aio requests (write thread) I/O thread 32 state: waiting for completed aio requests (write thread) I/O thread 33 state: waiting for completed aio requests (write thread) Pending normal aio reads: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] , aio writes: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] , ibuf aio reads:, log i/o's:, sync i/o's: Pending flushes (fsync) log: 0; buffer pool: 0 510525 OS file reads, 164422 OS file writes, 1285 OS fsyncs 0.00 reads/s, 0 avg bytes/read, 0.00 writes/s, 0.00 fsyncs/s ------------------------------------- INSERT BUFFER AND ADAPTIVE HASH INDEX ------------------------------------- InnoDB: ###### Diagnostic info printed to the standard error stream 2019-08-07T12:56:25.797957+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849490904832 has waited at ibuf0ibuf.cc line 4582 for 272.00 seconds the semaphore: S-lock on RW-latch at 0x7f4b9cf882b0 created in file buf0buf.cc line 1425 a writer (thread id 139849051821824) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: fffffffff0000000 Last time read locked in file ibuf0ibuf.cc line 3468 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/btr/btr0btr.cc line 179 2019-08-07T12:56:25.798022+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849543354112 has waited at ibuf0ibuf.cc line 2731 for 272.00 seconds the semaphore: Mutex at 0x1dfb880, Mutex IBUF created ibuf0ibuf.cc:527, lock var 1 2019-08-07T12:56:25.798049+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849051821824 has waited at buf0buf.cc line 4078 for 272.00 seconds the semaphore: S-lock on RW-latch at 0x7f382765e890 created in file buf0buf.cc line 1425 a writer (thread id 139963820316640) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: 0 Last time read locked in file not yet reserved line 0 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/buf/buf0buf.cc line 5334 InnoDB: ###### Starts InnoDB Monitor for 30 secs to print diagnostic info: InnoDB: Pending preads 0, pwrites 0 InnoDB: ###### Diagnostic info printed to the standard error stream 2019-08-07T12:56:56.799441+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849490904832 has waited at ibuf0ibuf.cc line 4582 for 303.00 seconds the semaphore: S-lock on RW-latch at 0x7f4b9cf882b0 created in file buf0buf.cc line 1425 a writer (thread id 139849051821824) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: fffffffff0000000 Last time read locked in file ibuf0ibuf.cc line 3468 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/btr/btr0btr.cc line 179 2019-08-07T12:56:56.799513+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849543354112 has waited at ibuf0ibuf.cc line 2731 for 303.00 seconds the semaphore: Mutex at 0x1dfb880, Mutex IBUF created ibuf0ibuf.cc:527, lock var 1 2019-08-07T12:56:56.799540+08:00 0 [Warning] InnoDB: A long semaphore wait: --Thread 139849051821824 has waited at buf0buf.cc line 4078 for 303.00 seconds the semaphore: S-lock on RW-latch at 0x7f382765e890 created in file buf0buf.cc line 1425 a writer (thread id 139963820316640) has reserved it in mode exclusive number of readers 0, waiters flag 1, lock_word: 0 Last time read locked in file not yet reserved line 0 Last time write locked in file /data/percona-server-5.7.26-29/storage/innobase/buf/buf0buf.cc line 5334 InnoDB: ###### Starts InnoDB Monitor for 30 secs to print diagnostic info: InnoDB: Pending preads 0, pwrites 0 InnoDB: ###### Diagnostic info printed to the standard error stream my.cnf [mysql] port=5002 prompt=\\u@\\d\\r:\\m:\\s> no-auto-rehash [client] port=5002 socket=/opt/tmp/mysql.sock [mysqld] basedir=/usr datadir=/opt/local/mysql/var/ tmpdir=/opt/tmp log-error=mysql.err log_timestamps=SYSTEM #slow_query_log_file=/opt/tmp/mysql.slow #general_log_file=/opt/tmp/mysql.genlog socket=/opt/tmp/mysql.sock pid-file=/opt/local/mysql/var/mysql.pid #innodb innodb_file_format=Barracuda innodb_data_home_dir=/opt/local/mysql/var/ innodb_log_group_home_dir=/opt/local/mysql/var/ innodb_data_file_path=ibdata1:100M:autoextend innodb_buffer_pool_size=100G innodb_buffer_pool_instances=4 innodb_log_files_in_group=3 innodb_log_file_size=1G innodb_log_buffer_size=16M innodb_checksum_algorithm=crc32 innodb_default_row_format=DYNAMIC internal_tmp_disk_storage_engine=innodb innodb_fast_shutdown=1 innodb_doublewrite=0 innodb_checksums=0 innodb_flush_log_at_trx_commit=2 innodb_flush_log_at_timeout=1 innodb_max_dirty_pages_pct=75innodb_io_capacity的2倍 innodb_io_capacity=1000 innodb_thread_concurrency=0 innodb_read_io_threads=16 innodb_write_io_threads=16 innodb_open_files=60000 innodb_file_per_table=1 innodb_flush_method=O_DIRECT innodb_flush_neighbors=1 innodb_change_buffering=all innodb_adaptive_flushing=0 innodb_old_blocks_time=1000 innodb_use_native_aio=1 lock_wait_timeout=60 innodb_print_all_deadlocks=ON innodb_rollback_on_timeout=0 innodb_lock_wait_timeout=50 innodb_buffer_pool_dump_at_shutdown=ON innodb_buffer_pool_load_at_startup=ON innodb_buffer_pool_dump_pct=100 innodb_strict_mode=OFF innodb_buffer_pool_chunk_size=128M max_heap_table_size=128M transaction-isolation=READ-COMMITTED #myisam key_buffer_size=32M myisam_sort_buffer_size=128M concurrent_insert=ALWAYS delayed_insert_timeout=300 read_rnd_buffer_size=128K myisam-recover-options=default #replication skip-slave-start relay-log=relay slave_load_tmpdir=/opt/tmp binlog_group_commit_sync_delay=0 binlog_group_commit_sync_no_delay_count=0 slave_parallel_type=logical_clock slave_parallel_workers=16 slave_pending_jobs_size_max=16M binlog_order_commits=1 slave_preserve_commit_order=1 binlog_gtid_simple_recovery=1 #binlog log-bin=mysql-bin server_id=7037027 report_host=10.21.105.29 report_port=5002 binlog_cache_size=1M max_binlog_cache_size=2G max_binlog_size=1024M binlog-format=ROW sync_binlog=0 log-slave-updates read_only=ON expire_logs_days=7 max_binlog_files=0 binlog_rows_query_log_events=ON binlog_error_action=ABORT_SERVER #server default-storage-engine=INNODB character-set-server=utf8mb4 collation_server=utf8mb4_unicode_ci loose-character-set-client=utf8mb4 open_files_limit=140000 log_slow_admin_statements=1 log_slow_verbosity=microtime,query_plan,innodb log_warnings=2 log_queries_not_using_indexes=0 long_query_time=0.1 slow_query_log=1 log_slow_slave_statements=1 query_cache_type=0 query_cache_size=256M query_cache_limit=1M query_cache_min_res_unit=1K table_definition_cache=65536 table_open_cache=65536 thread_stack=512K thread_cache_size=256 sort_buffer_size=8M join_buffer_size=8M read_buffer_size=128K port=5002 skip-name-resolve skip-ssl max_connections=10000 max_connect_errors=65536 max_allowed_packet=16M connect_timeout=8 net_read_timeout=30 net_write_timeout=60 slave_net_timeout=16 back_log=1024 innodb_stats_on_metadata=OFF log_statements_unsafe_for_binlog=OFF lower_case_table_names = 1 performance_schema=ON show_compatibility_56=ON gtid_mode=ON enforce-gtid-consistency=ON #replicate relay_log_recovery=1 relay_log_info_repository=TABLE master_info_repository=TABLE slave_compressed_protocol=ON #userstat userstat=ON default_password_lifetime=0 # max_execution_time=2000 # kill_idle_transaction=1000 # audit log plugin setting loose_audit_log_policy=NONE loose_audit_log_format=JSON sql_mode='NO_ENGINE_SUBSTITUTION' secure_file_priv='' # 增加extra_port extra_port=5003 extra_max_connections=10 report-host=xxx.xxx.xxx.xxx report-port=5002