Logging: ./mtr --force --mem --mysqld=--core-file --embedded gcc296 --repeat=1000 (...) main.gcc296 [ pass ] 11 main.gcc296 [ fail ] timeout after 900 seconds Test ended at 2009-02-03 15:33:47 - saving '/nfstmp1/johnemb/replay/mysql-6.0-bugteam/mysql-test/var/log/main.gcc296/' to '/nfstmp1/johnemb/replay/mysql-6.0-bugteam/mysql-test/var/log/main.gcc296/' - found 'core.26277', moving it to '/nfstmp1/johnemb/replay/mysql-6.0-bugteam/mysql-test/var/log/main.gcc296' - found 'core.26277' (1/5) Trying 'dbx' to get a backtrace Trying 'gdb' to get a backtrace Core generated by '/nfstmp1/johnemb/replay/mysql-6.0-bugteam/libmysqld/examples/mysqltest_embedded' Output from gdb follows. The first stack trace is from the failing thread. The following stack traces are from all threads (so the failing one is duplicated). -------------------------- Using host libthread_db library "/lib64/libthread_db.so.1". warning: core file may not match specified executable file. Core was generated by `/nfstmp1/johnemb/replay/mysql-6.0-bugteam/libmysqld/examples/mysqltest_embedded'. Program terminated with signal 6, Aborted. #0 0x00002b164167f4c5 in pthread_kill () from /lib64/libpthread.so.0 #0 0x00002b164167f4c5 in pthread_kill () from /lib64/libpthread.so.0 #1 0x00000000005aed13 in signal_handler (sig=6) at mysqltest.cc:7423 #2 #3 0x00002b164136aaa5 in raise () from /lib64/libc.so.6 #4 0x0000000000a21642 in Error::error (string=) at Error.cpp:94 #5 0x000000000094a6bc in SyncObject::unlock (this=) at SyncObject.cpp:976 #6 0x0000000000a013e2 in Cache::ioThread (this=0x2aaaaacffff0) at Cache.cpp:950 #7 0x0000000000959d34 in Thread::thread (this=0x2aaaaaed9330) at Thread.cpp:167 #8 0x0000000000959f19 in Thread::thread (parameter=0x66a5) at Thread.cpp:146 #9 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #10 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #11 0x0000000000000000 in ?? () Thread 12 (process 26277): #0 0x000000000070b000 in ma_control_file_open (create_if_missing=, print_error=1 '\001') at ma_control_file.c:372 #1 0x00000000006f6df5 in ha_maria_init (p=) at ha_maria.cc:3052 #2 0x0000000000845a1f in ha_initialize_handlerton (plugin=0x1bb07b0) at handler.cc:450 #3 0x00000000006676ba in plugin_initialize (plugin=0x1bb07b0) at sql_plugin.cc:1007 #4 0x000000000066ae0c in plugin_init (argc=0x1103034, argv=0x1b85af0, flags=) at sql_plugin.cc:1216 #5 0x00000000005cc672 in init_embedded_server (argc=9, argv=0x107f5e0, groups=) at ../sql/mysqld.cc:4128 #6 0x00000000005baf6e in main (argc=, argv=0x7fff69df10f8) at mysqltest.cc:7574 Current language: auto; currently c Thread 11 (process 26288): #0 0x00002b1641680c59 in __lll_mutex_unlock_wake () from /lib64/libpthread.so.0 #1 0x00002b1641683518 in __PRETTY_FUNCTION__.5857 () from /lib64/libpthread.so.0 #2 0x0000000000000001 in ?? () #3 0x00002aaaaad6cf20 in ?? () #4 0x0000000000000000 in ?? () Thread 10 (process 26289): #0 0x00002b164167e9e7 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a7ee in Synchronize::sleep (this=0x2aaaaaee2848, milliseconds=1000, callersMutex=0x0) at Synchronize.cpp:183 #2 0x0000000000a8a65c in Synchronize::sleep (this=0x2aaaaaee2864, milliseconds=1098916112) at Synchronize.cpp:136 #3 0x0000000000a1487c in Database::ticker (database=) at Database.cpp:1886 #4 0x0000000000959d34 in Thread::thread (this=0x2aaaaaee2848) at Thread.cpp:167 #5 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee2864) at Thread.cpp:146 #6 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #7 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 9 (process 26290): #0 0x00002b164167e7f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee2988) at Synchronize.cpp:123 #2 0x0000000000a47fb3 in PageWriter::writer (this=0x2aaaaad71e90) at PageWriter.cpp:171 #3 0x0000000000959d34 in Thread::thread (this=0x2aaaaaee2988) at Thread.cpp:167 #4 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee29a4) at Thread.cpp:146 #5 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #6 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 8 (process 26291): #0 SerialLogControl::getData (this=0x428037d0, length=35) at SerialLogControl.cpp:349 #1 0x0000000000a7ddb6 in SRLIndexPage::read (this=0x42803d20) at SRLIndexPage.cpp:85 #2 0x0000000000a638b8 in SerialLogControl::nextRecord (this=0x428037d0) at SerialLogControl.cpp:341 #3 0x0000000000a69656 in SerialLogTransaction::commit (this=0x2aaaaaee35c0) at SerialLogTransaction.cpp:88 #4 0x0000000000b13744 in Gopher::gopherThread (this=0x2aaaaaed9e08) at Gopher.cpp:71 #5 0x0000000000959d34 in Thread::thread (this=0x2aaaaaee2f30) at Thread.cpp:167 #6 0x0000000000959f19 in Thread::thread (parameter=0x428037d0) at Thread.cpp:146 #7 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #8 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #9 0x0000000000000000 in ?? () Current language: auto; currently c++ Thread 7 (process 26292): #0 SerialLogControl::getData (this=0x430047d0, length=35) at SerialLogControl.cpp:349 #1 0x0000000000a7ddb6 in SRLIndexPage::read (this=0x43004d20) at SRLIndexPage.cpp:85 #2 0x0000000000a638b8 in SerialLogControl::nextRecord (this=0x430047d0) at SerialLogControl.cpp:341 #3 0x0000000000a69656 in SerialLogTransaction::commit (this=0x2aaaaaef1870) at SerialLogTransaction.cpp:88 #4 0x0000000000b13744 in Gopher::gopherThread (this=0x2aaaaaed9dd0) at Gopher.cpp:71 #5 0x0000000000959d34 in Thread::thread (this=0x2aaaaaee3070) at Thread.cpp:167 #6 0x0000000000959f19 in Thread::thread (parameter=0x430047d0) at Thread.cpp:146 #7 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #8 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #9 0x0000000000000000 in ?? () Thread 6 (process 26293): #0 0x00002b1641680c59 in __lll_mutex_unlock_wake () from /lib64/libpthread.so.0 #1 0x00002b1641683518 in __PRETTY_FUNCTION__.5857 () from /lib64/libpthread.so.0 #2 0x0000000000000001 in ?? () #3 0x0000000000000000 in ?? () Thread 5 (process 26294): #0 0x00002b164167e7f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee32f0) at Synchronize.cpp:123 #2 0x0000000000b1369d in Gopher::gopherThread (this=0x2aaaaaed9d60) at Gopher.cpp:58 #3 0x0000000000959d34 in Thread::thread (this=0x2aaaaaee32f0) at Thread.cpp:167 #4 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee330c) at Thread.cpp:146 #5 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #6 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 4 (process 26295): #0 0x00002b164167e7f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee3430) at Synchronize.cpp:123 #2 0x0000000000b1369d in Gopher::gopherThread (this=0x2aaaaaed9d28) at Gopher.cpp:58 #3 0x0000000000959d34 in Thread::thread (this=0x2aaaaaee3430) at Thread.cpp:167 #4 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee344c) at Thread.cpp:146 #5 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #6 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 3 (process 26298): #0 0x00002b164167e7f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee4338) at Synchronize.cpp:123 #2 0x0000000000a588f6 in Scheduler::schedule (this=0x2aaaaaee1a38) at Scheduler.cpp:152 #3 0x0000000000959d34 in Thread::thread (this=0x2aaaaaee4338) at Thread.cpp:167 #4 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee4354) at Thread.cpp:146 #5 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #6 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 2 (process 26299): #0 0x00002b164167e9e7 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a7ee in Synchronize::sleep (this=0x2aaaaaeebbc0, milliseconds=14000, callersMutex=0x0) at Synchronize.cpp:183 #2 0x0000000000a8a65c in Synchronize::sleep (this=0x2aaaaaeebbdc, milliseconds=1166057680) at Synchronize.cpp:136 #3 0x0000000000a588b1 in Scheduler::schedule (this=0x2aaaaaee1b18) at Scheduler.cpp:150 #4 0x0000000000959d34 in Thread::thread (this=0x2aaaaaeebbc0) at Thread.cpp:167 #5 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaeebbdc) at Thread.cpp:146 #6 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #7 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 1 (process 26287): #0 0x00002b164167f4c5 in pthread_kill () from /lib64/libpthread.so.0 #1 0x00000000005aed13 in signal_handler (sig=6) at mysqltest.cc:7423 #2 #3 0x00002b164136aaa5 in raise () from /lib64/libc.so.6 #4 0x0000000000a21642 in Error::error (string=) at Error.cpp:94 #5 0x000000000094a6bc in SyncObject::unlock (this=) at SyncObject.cpp:976 #6 0x0000000000a013e2 in Cache::ioThread (this=0x2aaaaacffff0) at Cache.cpp:950 #7 0x0000000000959d34 in Thread::thread (this=0x2aaaaaed9330) at Thread.cpp:167 #8 0x0000000000959f19 in Thread::thread (parameter=0x66a5) at Thread.cpp:146 #9 0x00002b164167b193 in start_thread () from /lib64/libpthread.so.0 #10 0x00002b16413fa47d in clone () from /lib64/libc.so.6 #11 0x0000000000000000 in ?? () - found 'core.317' (2/5) Trying 'dbx' to get a backtrace Trying 'gdb' to get a backtrace Core generated by '/nfstmp1/johnemb/replay/mysql-6.0-bugteam/libmysqld/examples/mysqltest_embedded' Output from gdb follows. The first stack trace is from the failing thread. The following stack traces are from all threads (so the failing one is duplicated). -------------------------- Using host libthread_db library "/lib64/libthread_db.so.1". warning: core file may not match specified executable file. Core was generated by `/nfstmp1/johnemb/replay/mysql-6.0-bugteam/libmysqld/examples/mysqltest_embedded'. Program terminated with signal 6, Aborted. #0 0x00002b8ed13954c5 in pthread_kill () from /lib64/libpthread.so.0 #0 0x00002b8ed13954c5 in pthread_kill () from /lib64/libpthread.so.0 #1 0x00000000005aed13 in signal_handler (sig=6) at mysqltest.cc:7423 #2 #3 0x00002b8ed1396b48 in __lll_mutex_lock_wait () from /lib64/libpthread.so.0 #4 0x00002b8ed1399404 in default_attr () from /lib64/libpthread.so.0 #5 0x000000007fffffff in ?? () #6 0x00002b8ed1394c2e in pthread_cond_broadcast@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #7 0x0000000000a8a6c6 in Synchronize::wake (this=0x2aaaaaed9470) at Synchronize.cpp:234 #8 0x00000000009feb3e in Cache::shutdownThreads (this=0x2aaaaacffff0) at Cache.cpp:1025 #9 0x00000000009ff0be in Cache::shutdown (this=0x2aaaaaed9488) at Cache.cpp:1007 #10 0x0000000000a0f849 in Database::shutdown (this=0x2aaaaacd7bb8) at Database.cpp:1611 #11 0x0000000000a0a12e in Connection::shutdownDatabase (this=) at Connection.cpp:1804 #12 0x000000000093f54e in StorageDatabase::close (this=0x2aaaaaed81f0) at StorageDatabase.cpp:925 #13 0x00000000009424ba in StorageHandler::shutdownHandler (this=0x2aaaaacd7040) at StorageHandler.cpp:194 #14 0x000000000093b166 in StorageInterface::panic (hton=, flag=HA_PANIC_CLOSE) at ha_falcon.cpp:274 #15 0x000000000084499b in ha_finalize_handlerton (plugin=0x1ba7518) at handler.cc:393 #16 0x0000000000667626 in plugin_deinitialize (plugin=0x1ba7518, ref_check=true) at sql_plugin.cc:820 #17 0x000000000066888a in reap_plugins () at sql_plugin.cc:908 #18 0x000000000066921b in plugin_shutdown () at sql_plugin.cc:1512 #19 0x00000000005c79c7 in end_embedded_server () at ../sql/mysqld.cc:1366 #20 0x00000000005c3bc9 in mysql_server_end () at libmysql.c:202 #21 0x00000000005b0ce8 in cleanup_and_exit (exit_code=-1427270520) at mysqltest.cc:1148 #22 0x00000000005bb723 in main (argc=, argv=0x1) at mysqltest.cc:8040 Thread 7 (process 321): #0 0x00002b8ed13949e7 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a7ee in Synchronize::sleep (this=0x2aaaaaee2848, milliseconds=1000, callersMutex=0x0) at Synchronize.cpp:183 #2 0x0000000000a8a65c in Synchronize::sleep (this=0x2aaaaaee2864, milliseconds=1098916112) at Synchronize.cpp:136 #3 0x0000000000a1487c in Database::ticker (database=) at Database.cpp:1886 #4 0x0000000000959d34 in Thread::thread (this=0x2aaaaaee2848) at Thread.cpp:167 #5 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee2864) at Thread.cpp:146 #6 0x00002b8ed1391193 in start_thread () from /lib64/libpthread.so.0 #7 0x00002b8ed111047d in clone () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 6 (process 323): #0 0x00002b8ed13947f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee2f30) at Synchronize.cpp:123 #2 0x0000000000959e02 in Thread::thread (this=0x2aaaaaee2f30) at Thread.cpp:193 #3 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee2f4c) at Thread.cpp:146 #4 0x00002b8ed1391193 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b8ed111047d in clone () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 5 (process 324): #0 0x00002b8ed13947f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee3070) at Synchronize.cpp:123 #2 0x0000000000959e02 in Thread::thread (this=0x2aaaaaee3070) at Thread.cpp:193 #3 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee308c) at Thread.cpp:146 #4 0x00002b8ed1391193 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b8ed111047d in clone () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 4 (process 325): #0 0x00002b8ed13947f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee31b0) at Synchronize.cpp:123 #2 0x0000000000959e02 in Thread::thread (this=0x2aaaaaee31b0) at Thread.cpp:193 #3 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee31cc) at Thread.cpp:146 #4 0x00002b8ed1391193 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b8ed111047d in clone () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 3 (process 326): #0 0x00002b8ed13947f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee32f0) at Synchronize.cpp:123 #2 0x0000000000959e02 in Thread::thread (this=0x2aaaaaee32f0) at Thread.cpp:193 #3 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee330c) at Thread.cpp:146 #4 0x00002b8ed1391193 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b8ed111047d in clone () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 2 (process 327): #0 0x00002b8ed13947f6 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a8a912 in Synchronize::sleep (this=0x2aaaaaee3430) at Synchronize.cpp:123 #2 0x0000000000959e02 in Thread::thread (this=0x2aaaaaee3430) at Thread.cpp:193 #3 0x0000000000959f19 in Thread::thread (parameter=0x2aaaaaee344c) at Thread.cpp:146 #4 0x00002b8ed1391193 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b8ed111047d in clone () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 1 (process 317): #0 0x00002b8ed13954c5 in pthread_kill () from /lib64/libpthread.so.0 #1 0x00000000005aed13 in signal_handler (sig=6) at mysqltest.cc:7423 #2 #3 0x00002b8ed1396b48 in __lll_mutex_lock_wait () from /lib64/libpthread.so.0 #4 0x00002b8ed1399404 in default_attr () from /lib64/libpthread.so.0 #5 0x000000007fffffff in ?? () #6 0x00002b8ed1394c2e in pthread_cond_broadcast@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #7 0x0000000000a8a6c6 in Synchronize::wake (this=0x2aaaaaed9470) at Synchronize.cpp:234 #8 0x00000000009feb3e in Cache::shutdownThreads (this=0x2aaaaacffff0) at Cache.cpp:1025 #9 0x00000000009ff0be in Cache::shutdown (this=0x2aaaaaed9488) at Cache.cpp:1007 #10 0x0000000000a0f849 in Database::shutdown (this=0x2aaaaacd7bb8) at Database.cpp:1611 #11 0x0000000000a0a12e in Connection::shutdownDatabase (this=) at Connection.cpp:1804 #12 0x000000000093f54e in StorageDatabase::close (this=0x2aaaaaed81f0) at StorageDatabase.cpp:925 #13 0x00000000009424ba in StorageHandler::shutdownHandler (this=0x2aaaaacd7040) at StorageHandler.cpp:194 #14 0x000000000093b166 in StorageInterface::panic (hton=, flag=HA_PANIC_CLOSE) at ha_falcon.cpp:274 #15 0x000000000084499b in ha_finalize_handlerton (plugin=0x1ba7518) at handler.cc:393 #16 0x0000000000667626 in plugin_deinitialize (plugin=0x1ba7518, ref_check=true) at sql_plugin.cc:820 #17 0x000000000066888a in reap_plugins () at sql_plugin.cc:908 #18 0x000000000066921b in plugin_shutdown () at sql_plugin.cc:1512 #19 0x00000000005c79c7 in end_embedded_server () at ../sql/mysqld.cc:1366 #20 0x00000000005c3bc9 in mysql_server_end () at libmysql.c:202 #21 0x00000000005b0ce8 in cleanup_and_exit (exit_code=-1427270520) at mysqltest.cc:1148 #22 0x00000000005bb723 in main (argc=, argv=0x1) at mysqltest.cc:8040 Retrying test, attempt(2/3)... main.gcc296 [ pass ] 3 Retrying test, attempt(3/3)... main.gcc296 [ pass ] 2 main.gcc296 [ pass ] 6 main.gcc296 [ pass ] 10