rpl.rpl_innodb_bug30888 w6 [ fail ] timeout after 900 seconds
        Test ended at 2009-09-24 18:18:40

Test case timeout after 900 seconds

== /export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/mysql-test/var-ps_row/6/log/rpl_innodb_bug30888.log == 
BEGIN
DECLARE ins_count INT DEFAULT 1000; 
DECLARE del_count INT;
DECLARE cur_user VARCHAR(255);
DECLARE local_uuid VARCHAR(255);
DECLARE local_time TIMESTAMP;
SET local_time= NOW();
SET cur_user= CURRENT_USER();
SET local_uuid= UUID();
WHILE ins_count > 0 DO
INSERT INTO test.regular_tbl VALUES (NULL, NOW(), USER() , UUID(),
ins_count,'Going to test MBR for MySQL');
SET ins_count = ins_count - 1;
END WHILE;
SELECT MAX(id) FROM test.regular_tbl INTO del_count;
WHILE del_count > 0 DO
DELETE FROM test.regular_tbl WHERE id = del_count;
SET del_count = del_count - 2;
END WHILE;
END|

 == /export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/mysql-test/var-ps_row/6/tmp/analyze-timeout-mysqld.1.err ==
mysqltest: Could not open connection 'default' after 500 attempts: 2002 Can't connect to local MySQL server through socket '/export/home/pb2/test/sb_6-765532-1253799900.54/tmp/UuBo_EYMsw/6/mysqld.1.sock' (146)

 == /export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/mysql-test/var-ps_row/6/tmp/analyze-timeout-mysqld.2.err ==
mysqltest: Could not open connection 'default' after 500 attempts: 2002 Can't connect to local MySQL server through socket '/export/home/pb2/test/sb_6-765532-1253799900.54/tmp/UuBo_EYMsw/6/mysqld.2.sock' (146)


 - saving '/export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/mysql-test/var-ps_row/6/log/rpl.rpl_innodb_bug30888/' to '/export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/mysql-test/var-ps_row/log/rpl.rpl_innodb_bug30888/'
 - found 'core', moving it to '/export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/mysql-test/var-ps_row/log/rpl.rpl_innodb_bug30888'
 - found 'core' (14/5)

Trying 'dbx' to get a backtrace
Core generated by '/export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/bin/mysqltest'
Output from dbx follows. Stack trace is printed for all threads in order,
above this you should see info about which thread was the failing one.
----------------------------
For information about new features see `help changes'
To remove this message, put `dbxenv suppress_startup_message 7.6' in your .dbxrc
Reading mysqltest
core file header read successfully
Reading ld.so.1
Reading libpthread.so.1
Reading libthread.so.1
Reading librt.so.1
Reading libsocket.so.1
Reading libnsl.so.1
Reading libstdc++.so.6.0.3
Reading libm.so.2
Reading libgcc_s.so.1
Reading libc.so.1
Reading libaio.so.1
Reading libmd.so.1
Reading libm.so.1
t@1 (l@1) program terminated by signal ABRT (Abort)
0xfecf53a5: __lwp_kill+0x0015:	jae      __lwp_kill+0x23	[ 0xfecf53b3, .+0xe ]
Current function is my_write_core
dbx: warning: can't find file "/export/home/pb2/build/sb_1-765392-1253790812.3/mysql-5.4.5-beta/mysys/stacktrace.c" dbx: warning: see `help finding-files'
current thread: t@1
  [1] __lwp_kill(0x1, 0x6), at 0xfecf53a5 
  [2] _thr_kill(0x1, 0x6, 0x82e5f20), at 0xfecf2260 
=>[3] my_write_core(sig = 0), line 310 in "stacktrace.c"
  [4] signal_handler(sig = 0), line 7462 in "mysqltest.cc"
  [5] __sighndlr(0x6, 0x0, 0x8044900, 0x8086514), at 0xfecf3e9f 
  [6] call_user_handler(0x6, 0x0, 0x8044900), at 0xfecea31e 
  [7] sigacthandler(0x6, 0x0, 0x8044900, 0xf, 0x0, 0x0), at 0xfecea49e 
  ---- called from signal handler with signal 6 (SIGABRT) ------
  [8] _read(0x8, 0x83104d0, 0x4000, 0xfed25b74), at 0xfecf4f95 
  [9] vio_read(vio = (nil), buf = 0x83104d0 "^G", size = 16384U), line 44 in "viosocket.c"
  [10] vio_read_buff(vio = (nil), buf = 0x83144d8 "^V", size = 4U), line 83 in "viosocket.c"
  [11] my_real_read(net = (nil), complen = 0x8044bd8), line 839 in "net.c"
  [12] my_net_read(net = 0x8245934), line 1029 in "net.c"
  [13] cli_safe_read(mysql = 0x8044c18), line 691 in "client.c"
  [14] cli_read_query_result(mysql = (nil)), line 2803 in "client.c"
  [15] mysql_read_query_result(mysql = (nil)), line 5233 in "libmysql.c"
  [16] run_query_normal(cn = (nil), command = 0x8352d40, flags = 0, query = (nil), query_len = 0, ds = 0x8245540, ds_warnings = 0x8044d10), line 6479 in "mysqltest.cc"
  [17] run_query(cn = 0x8245934, command = 0x8352d40, flags = 3), line 7182 in "mysqltest.cc"
  [18] main(argc = 13, argv = (nil)), line 7840 in "mysqltest.cc"
current thread: t@1
  [1] __lwp_kill(0x1, 0x6), at 0xfecf53a5 
  [2] _thr_kill(0x1, 0x6, 0x82e5f20), at 0xfecf2260 
=>[3] my_write_core(sig = 0), line 310 in "stacktrace.c"
  [4] signal_handler(sig = 0), line 7462 in "mysqltest.cc"
  [5] __sighndlr(0x6, 0x0, 0x8044900, 0x8086514), at 0xfecf3e9f 
  [6] call_user_handler(0x6, 0x0, 0x8044900), at 0xfecea31e 
  [7] sigacthandler(0x6, 0x0, 0x8044900, 0xf, 0x0, 0x0), at 0xfecea49e 
  ---- called from signal handler with signal 6 (SIGABRT) ------
  [8] _read(0x8, 0x83104d0, 0x4000, 0xfed25b74), at 0xfecf4f95 
  [9] vio_read(vio = (nil), buf = 0x83104d0 "^G", size = 16384U), line 44 in "viosocket.c"
  [10] vio_read_buff(vio = (nil), buf = 0x83144d8 "^V", size = 4U), line 83 in "viosocket.c"
  [11] my_real_read(net = (nil), complen = 0x8044bd8), line 839 in "net.c"
  [12] my_net_read(net = 0x8245934), line 1029 in "net.c"
  [13] cli_safe_read(mysql = 0x8044c18), line 691 in "client.c"
  [14] cli_read_query_result(mysql = (nil)), line 2803 in "client.c"
  [15] mysql_read_query_result(mysql = (nil)), line 5233 in "libmysql.c"
  [16] run_query_normal(cn = (nil), command = 0x8352d40, flags = 0, query = (nil), query_len = 0, ds = 0x8245540, ds_warnings = 0x8044d10), line 6479 in "mysqltest.cc"
  [17] run_query(cn = 0x8245934, command = 0x8352d40, flags = 3), line 7182 in "mysqltest.cc"
  [18] main(argc = 13, argv = (nil)), line 7840 in "mysqltest.cc"

 - deleting it, already saved 5
 - found 'core' (15/5)

Trying 'dbx' to get a backtrace
Core generated by '/export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/libexec/mysqld'
Output from dbx follows. Stack trace is printed for all threads in order,
above this you should see info about which thread was the failing one.
----------------------------
For information about new features see `help changes'
To remove this message, put `dbxenv suppress_startup_message 7.6' in your .dbxrc
Reading mysqld
core file header read successfully
Reading ld.so.1
Reading libpthread.so.1
Reading libthread.so.1
Reading libdl.so.1
Reading librt.so.1
Reading libsocket.so.1
Reading libnsl.so.1
Reading libstdc++.so.6.0.3
Reading libm.so.2
Reading libgcc_s.so.1
Reading libc.so.1
Reading libaio.so.1
Reading libmd.so.1
Reading libm.so.1
t@1 (l@1) terminated by signal ABRT (Abort)
0xfecd53a5: __lwp_kill+0x0015:	jae      __lwp_kill+0x23	[ 0xfecd53b3, .+0xe ]
Current function is my_write_core
dbx: warning: can't find file "/export/home/pb2/build/sb_1-765392-1253790812.3/mysql-5.4.5-beta/mysys/stacktrace.c" dbx: warning: see `help finding-files'
current thread: t@1
  [1] __lwp_kill(0x1, 0x6), at 0xfecd53a5 
  [2] _thr_kill(0x1, 0x6, 0xfed00c90), at 0xfecd2260 
=>[3] my_write_core(sig = 0), line 310 in "stacktrace.c"
  [4] handle_segfault(sig = 0), line 2570 in "mysqld.cc"
  [5] __sighndlr(0x6, 0x0, 0x8044b38, 0x81f0fa4), at 0xfecd3e9f 
  [6] call_user_handler(0x6, 0x0, 0x8044b38), at 0xfecca31e 
  [7] sigacthandler(0x6, 0x0, 0x8044b38, 0xf, 0x0, 0x807007), at 0xfecca49e 
  ---- called from signal handler with signal 6 (SIGABRT) ------
  [8] __pollsys(0x8044d70, 0x2, 0x0, 0x0), at 0xfecd4e35 
  [9] _pselect(0x11, 0x8044f50, 0xfed00310, 0xfed00310, 0x0, 0x0), at 0xfec82fae 
  [10] _select(0x11, 0x8044f50, 0x0, 0x0, 0x0, 0x8044ed0, 0x80, 0x9035c88), at 0xfec832a4 
  [11] handle_connections_sockets(arg = (nil)), line 5035 in "mysqld.cc"
  [12] main(argc = 6, argv = (nil)), line 4519 in "mysqld.cc"
current thread: t@1
  [1] __lwp_kill(0x1, 0x6), at 0xfecd53a5 
  [2] _thr_kill(0x1, 0x6, 0xfed00c90), at 0xfecd2260 
=>[3] my_write_core(sig = 0), line 310 in "stacktrace.c"
  [4] handle_segfault(sig = 0), line 2570 in "mysqld.cc"
  [5] __sighndlr(0x6, 0x0, 0x8044b38, 0x81f0fa4), at 0xfecd3e9f 
  [6] call_user_handler(0x6, 0x0, 0x8044b38), at 0xfecca31e 
  [7] sigacthandler(0x6, 0x0, 0x8044b38, 0xf, 0x0, 0x807007), at 0xfecca49e 
  ---- called from signal handler with signal 6 (SIGABRT) ------
  [8] __pollsys(0x8044d70, 0x2, 0x0, 0x0), at 0xfecd4e35 
  [9] _pselect(0x11, 0x8044f50, 0xfed00310, 0xfed00310, 0x0, 0x0), at 0xfec82fae 
  [10] _select(0x11, 0x8044f50, 0x0, 0x0, 0x0, 0x8044ed0, 0x80, 0x9035c88), at 0xfec832a4 
  [11] handle_connections_sockets(arg = (nil)), line 5035 in "mysqld.cc"
  [12] main(argc = 6, argv = (nil)), line 4519 in "mysqld.cc"
current thread: t@2
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c597fc, 0x8c597d8, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c597fc, 0x8c597d8, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c597fc, 0x8c597d8), at 0xfecceb17 
  [5] cond_wait(0x8c597fc, 0x8c597d8), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c597fc, 0x8c597d8, 0xfeb8ee18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 0, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4273532856U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea80200), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c597d8, 0xfea80200, 0xfecfe000), at 0xfecd3d80 
current thread: t@3
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c59844, 0x8c59820, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c59844, 0x8c59820, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c59844, 0x8c59820), at 0xfecceb17 
  [5] cond_wait(0x8c59844, 0x8c59820), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c59844, 0x8c59820, 0xfe1fee18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 1U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4263505848U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea80a00), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59820, 0xfea80a00, 0xfecfe000), at 0xfecd3d80 
current thread: t@4
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c5988c, 0x8c59868, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c5988c, 0x8c59868, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c5988c, 0x8c59868), at 0xfecceb17 
  [5] cond_wait(0x8c5988c, 0x8c59868), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c5988c, 0x8c59868, 0xfe0ffe18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 2U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4262461368U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea81200), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59868, 0xfea81200, 0xfecfe000), at 0xfecd3d80 
current thread: t@5
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c598d4, 0x8c598b0, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c598d4, 0x8c598b0, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c598d4, 0x8c598b0), at 0xfecceb17 
  [5] cond_wait(0x8c598d4, 0x8c598b0), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c598d4, 0x8c598b0, 0xfe000e18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 3U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4261416888U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea81a00), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c598b0, 0xfea81a00, 0xfecfe000), at 0xfecd3d80 
current thread: t@6
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c5991c, 0x8c598f8, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c5991c, 0x8c598f8, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c5991c, 0x8c598f8), at 0xfecceb17 
  [5] cond_wait(0x8c5991c, 0x8c598f8), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c5991c, 0x8c598f8, 0xfdf01e18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 4U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4260372408U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea82200), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c598f8, 0xfea82200, 0xfecfe000), at 0xfecd3d80 
current thread: t@7
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c59964, 0x8c59940, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c59964, 0x8c59940, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c59964, 0x8c59940), at 0xfecceb17 
  [5] cond_wait(0x8c59964, 0x8c59940), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c59964, 0x8c59940, 0xfde02e18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 5U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4259327928U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea82a00), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59940, 0xfea82a00, 0xfecfe000), at 0xfecd3d80 
current thread: t@8
=>[1] __pollsys(0xfdd03f10, 0x0, 0xfdd03f60, 0x0), at 0xfecd4e35 
  [2] _pollsys(0xfdd03f10, 0x0, 0xfdd03f60, 0x0), at 0xfecc9464 
  [3] _pselect(0x0, 0xfed00310, 0xfed00310, 0xfed00310, 0xfdd03f60, 0x0), at 0xfec82fae 
  [4] _select(0x0, 0x0, 0x0, 0x0, 0xfdd03f90, 0x1), at 0xfec832a4 
  [5] os_thread_sleep(tm = 0), line 294 in "os0thread.c"
  [6] srv_lock_timeout_and_monitor_thread(arg = (nil)), line 1978 in "srv0srv.c"
  [7] _thr_setup(0xfea83200), at 0xfecd3a91 
  [8] _lwp_start(0xfdd03f10, 0x0, 0xfdd03f60, 0x0, 0x8b54360, 0x0), at 0xfecd3d80 
current thread: t@9
=>[1] __pollsys(0xfdc04f10, 0x0, 0xfdc04f60, 0x0), at 0xfecd4e35 
  [2] _pollsys(0xfdc04f10, 0x0, 0xfdc04f60, 0x0), at 0xfecc9464 
  [3] _pselect(0x0, 0xfed00310, 0xfed00310, 0xfed00310, 0xfdc04f60, 0x0), at 0xfec82fae 
  [4] _select(0x0, 0x0, 0x0, 0x0, 0xfdc04f90, 0xfecbaa91), at 0xfec832a4 
  [5] os_thread_sleep(tm = 0), line 294 in "os0thread.c"
  [6] srv_error_monitor_thread(arg = (nil)), line 2214 in "srv0srv.c"
  [7] _thr_setup(0xfea83a00), at 0xfecd3a91 
  [8] _lwp_start(0xfdc04f10, 0x0, 0xfdc04f60, 0x0, 0x0, 0x0), at 0xfecd3d80 
current thread: t@10
=>[1] __pollsys(0xfdb05f00, 0x0, 0xfdb05f50, 0x0), at 0xfecd4e35 
  [2] _pollsys(0xfdb05f00, 0x0, 0xfdb05f50, 0x0), at 0xfecc9464 
  [3] _pselect(0x0, 0xfed00310, 0xfed00310, 0xfed00310, 0xfdb05f50, 0x0), at 0xfec82fae 
  [4] _select(0x0, 0x0, 0x0, 0x0, 0xfdb05f80, 0x4abb9b9d), at 0xfec832a4 
  [5] os_thread_sleep(tm = 0), line 294 in "os0thread.c"
  [6] srv_master_thread(arg = (nil)), line 2371 in "srv0srv.c"
  [7] _thr_setup(0xfea84200), at 0xfecd3a91 
  [8] _lwp_start(0xfdb05f00, 0x0, 0xfdb05f50, 0x0, 0x7, 0x0), at 0xfecd3d80 
current thread: t@11
=>[1] ___sigtimedwait(0xfea7eea0, 0xfea7eeb0, 0x0), at 0xfecd4205 
  [2] __sigtimedwait(0xfea7efb0, 0x0, 0x0), at 0xfecc9561 
  [3] _sigwait(0xfea7efb0), at 0xfecc95df 
  [4] __posix_sigwait(0xfea7efb0, 0xfea7ef8c, 0x0, 0x14), at 0xfecc1de6 
  [5] signal_hand(arg = (nil)), line 2772 in "mysqld.cc"
  [6] _thr_setup(0xfea84a00), at 0xfecd3a91 
  [7] _lwp_start(0xfea7eea0, 0xfea7eeb0, 0x0, 0x0, 0x0, 0xfecfe000), at 0xfecd3d80 
current thread: t@35
=>[1] __fdsync(0x9, 0x10), at 0xfecd4885 
  [2] _fsync(0x9), at 0xfecc104e 
  [3] fsync(0x9, 0xcb3c4, 0xfda04fd8, 0x837a0cc), at 0xfecc8a1d 
  [4] os_file_flush(file = 0), line 1881 in "os0file.c"
  [5] fil_flush(space_id = 0), line 4523 in "fil0fil.c"
  [6] log_write_up_to(lsn = 0, wait = 92U, flush_to_disk = 1U), line 1499 in "log0log.c"
  [7] trx_prepare_off_kernel(trx = 0x908a900), line 1904 in "trx0trx.c"
  [8] trx_prepare_for_mysql(trx = 0xfda054e8), line 1934 in "trx0trx.c"
  [9] innobase_xa_prepare(hton = 0x8909ff8, thd = 0xfda05518, all = false), line 9025 in "ha_innodb.cc"
  [10] ha_commit_trans(thd = 0x9012bb0, all = false), line 1172 in "handler.cc"
  [11] ha_autocommit_or_rollback(thd = (nil), error = -39823976), line 1363 in "handler.cc"
  [12] close_thread_tables(thd = 0x9012bb0), line 2003 in "sql_class.h"
  [13] sp_lex_keeper::reset_lex_and_exec_core(this = (nil), thd = 0xfda055f8, nextp = 0xfda0596c, open_tables = false, instr = 0x9093d48), line 2758 in "sp_head.cc"
  [14] sp_instr_stmt::execute(this = 0x9093d48, thd = (nil), nextp = 0xfda0596c), line 2864 in "sp_head.cc"
  [15] sp_head::execute(this = 0x908b0a0, thd = 0x9012bb0), line 1259 in "sp_head.cc"
  [16] sp_head::execute_procedure(this = 0x908b0a0, thd = 0x9012bb0, args = 0x9013ef8), line 1997 in "sp_head.cc"
  [17] mysql_execute_command(thd = 0x9012bb0), line 4432 in "sql_parse.cc"
  [18] mysql_parse(thd = (nil), inBuf = (nil), length = 4255147480U, found_semicolon = 0xfda06644), line 6018 in "sql_parse.cc"
  [19] dispatch_command(command = COM_QUERY, thd = 0xfda06d88, packet = 0x90144d9 "CALL test.proc_norm()", packet_length = 0), line 1233 in "sql_parse.cc"
  [20] do_command(thd = 0x9012bb0), line 866 in "sql_parse.cc"
  [21] handle_one_connection(arg = (nil)), line 1131 in "sql_connect.cc"
  [22] _thr_setup(0xfea85a00), at 0xfecd3a91 
  [23] _lwp_start(0x9, 0x10, 0xfecfe000, 0xfda04f78, 0xfecc8a1d, 0x9), at 0xfecd3d80 
current thread: t@37
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x87cd4c0, 0x87cd0e4, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x87cd4c0, 0x87cd0e4, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x87cd4c0, 0x87cd0e4), at 0xfecceb17 
  [5] cond_wait(0x87cd4c0, 0x87cd0e4), at 0xfecceb59 
  [6] _pthread_cond_wait(0x87cd4c0, 0x87cd0e4, 0x0, 0x90584b8, 0xfecfe000, 0xfd9a4098), at 0xfecceb92 
  [7] MYSQL_BIN_LOG::wait_for_update(this = (nil), thd = 0xfd9a4098, is_slave = false), line 4633 in "log.cc"
  [8] mysql_binlog_send(thd = 0x9058168, log_ident = (nil), pos = 4ULL, flags = 0), line 665 in "sql_repl.cc"
  [9] dispatch_command(command = COM_BINLOG_DUMP, thd = 0xfd9a4d88, packet = 0x906f839 "", packet_length = 0), line 599 in "sql_class.h"
  [10] do_command(thd = 0x9058168), line 866 in "sql_parse.cc"
  [11] handle_one_connection(arg = (nil)), line 1131 in "sql_connect.cc"
  [12] _thr_setup(0xfea86a00), at 0xfecd3a91 
  [13] _lwp_start(0x0, 0x0, 0x0, 0x87cd0e4, 0xfea86a00, 0xfecfe000), at 0xfecd3d80 

 - deleting it, already saved 5
 - found 'core' (16/5)

Trying 'dbx' to get a backtrace
Core generated by '/export/home/pb2/test/sb_6-765532-1253799900.54/mysql-5.4.5-beta-solaris10-i386-test/libexec/mysqld'
Output from dbx follows. Stack trace is printed for all threads in order,
above this you should see info about which thread was the failing one.
----------------------------
For information about new features see `help changes'
To remove this message, put `dbxenv suppress_startup_message 7.6' in your .dbxrc
Reading mysqld
core file header read successfully
Reading ld.so.1
Reading libpthread.so.1
Reading libthread.so.1
Reading libdl.so.1
Reading librt.so.1
Reading libsocket.so.1
Reading libnsl.so.1
Reading libstdc++.so.6.0.3
Reading libm.so.2
Reading libgcc_s.so.1
Reading libc.so.1
Reading libaio.so.1
Reading libmd.so.1
Reading libm.so.1
t@1 (l@1) terminated by signal ABRT (Abort)
0xfecd53a5: __lwp_kill+0x0015:	jae      __lwp_kill+0x23	[ 0xfecd53b3, .+0xe ]
Current function is my_write_core
dbx: warning: can't find file "/export/home/pb2/build/sb_1-765392-1253790812.3/mysql-5.4.5-beta/mysys/stacktrace.c" dbx: warning: see `help finding-files'
current thread: t@1
  [1] __lwp_kill(0x1, 0x6), at 0xfecd53a5 
  [2] _thr_kill(0x1, 0x6, 0xfed00c90), at 0xfecd2260 
=>[3] my_write_core(sig = 0), line 310 in "stacktrace.c"
  [4] handle_segfault(sig = 0), line 2570 in "mysqld.cc"
  [5] __sighndlr(0x6, 0x0, 0x8044b38, 0x81f0fa4), at 0xfecd3e9f 
  [6] call_user_handler(0x6, 0x0, 0x8044b38), at 0xfecca31e 
  [7] sigacthandler(0x6, 0x0, 0x8044b38, 0xf, 0x0, 0x807007), at 0xfecca49e 
  ---- called from signal handler with signal 6 (SIGABRT) ------
  [8] __pollsys(0x8044d70, 0x2, 0x0, 0x0), at 0xfecd4e35 
  [9] _pselect(0x11, 0x8044f50, 0xfed00310, 0xfed00310, 0x0, 0x0), at 0xfec82fae 
  [10] _select(0x11, 0x8044f50, 0x0, 0x0, 0x0, 0x8044ed0, 0x80, 0x901f6f0), at 0xfec832a4 
  [11] handle_connections_sockets(arg = (nil)), line 5035 in "mysqld.cc"
  [12] main(argc = 6, argv = (nil)), line 4519 in "mysqld.cc"
current thread: t@1
  [1] __lwp_kill(0x1, 0x6), at 0xfecd53a5 
  [2] _thr_kill(0x1, 0x6, 0xfed00c90), at 0xfecd2260 
=>[3] my_write_core(sig = 0), line 310 in "stacktrace.c"
  [4] handle_segfault(sig = 0), line 2570 in "mysqld.cc"
  [5] __sighndlr(0x6, 0x0, 0x8044b38, 0x81f0fa4), at 0xfecd3e9f 
  [6] call_user_handler(0x6, 0x0, 0x8044b38), at 0xfecca31e 
  [7] sigacthandler(0x6, 0x0, 0x8044b38, 0xf, 0x0, 0x807007), at 0xfecca49e 
  ---- called from signal handler with signal 6 (SIGABRT) ------
  [8] __pollsys(0x8044d70, 0x2, 0x0, 0x0), at 0xfecd4e35 
  [9] _pselect(0x11, 0x8044f50, 0xfed00310, 0xfed00310, 0x0, 0x0), at 0xfec82fae 
  [10] _select(0x11, 0x8044f50, 0x0, 0x0, 0x0, 0x8044ed0, 0x80, 0x901f6f0), at 0xfec832a4 
  [11] handle_connections_sockets(arg = (nil)), line 5035 in "mysqld.cc"
  [12] main(argc = 6, argv = (nil)), line 4519 in "mysqld.cc"
current thread: t@2
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c59b6c, 0x8c59b48, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c59b6c, 0x8c59b48, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c59b6c, 0x8c59b48), at 0xfecceb17 
  [5] cond_wait(0x8c59b6c, 0x8c59b48), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c59b6c, 0x8c59b48, 0xfeb8ee18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 0, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4273532856U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea80200), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59b48, 0xfea80200, 0xfecfe000), at 0xfecd3d80 
current thread: t@3
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c59bb4, 0x8c59b90, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c59bb4, 0x8c59b90, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c59bb4, 0x8c59b90), at 0xfecceb17 
  [5] cond_wait(0x8c59bb4, 0x8c59b90), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c59bb4, 0x8c59b90, 0xfe1fee18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 1U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4263505848U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea80a00), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59b90, 0xfea80a00, 0xfecfe000), at 0xfecd3d80 
current thread: t@4
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c59bfc, 0x8c59bd8, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c59bfc, 0x8c59bd8, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c59bfc, 0x8c59bd8), at 0xfecceb17 
  [5] cond_wait(0x8c59bfc, 0x8c59bd8), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c59bfc, 0x8c59bd8, 0xfe0ffe18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 2U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4262461368U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea81200), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59bd8, 0xfea81200, 0xfecfe000), at 0xfecd3d80 
current thread: t@5
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c59c44, 0x8c59c20, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c59c44, 0x8c59c20, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c59c44, 0x8c59c20), at 0xfecceb17 
  [5] cond_wait(0x8c59c44, 0x8c59c20), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c59c44, 0x8c59c20, 0xfe000e18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 3U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4261416888U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea81a00), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59c20, 0xfea81a00, 0xfecfe000), at 0xfecd3d80 
current thread: t@6
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c59c8c, 0x8c59c68, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c59c8c, 0x8c59c68, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c59c8c, 0x8c59c68), at 0xfecceb17 
  [5] cond_wait(0x8c59c8c, 0x8c59c68), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c59c8c, 0x8c59c68, 0xfdf01e18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 4U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4260372408U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea82200), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59c68, 0xfea82200, 0xfecfe000), at 0xfecd3d80 
current thread: t@7
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8c59cd4, 0x8c59cb0, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8c59cd4, 0x8c59cb0, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8c59cd4, 0x8c59cb0), at 0xfecceb17 
  [5] cond_wait(0x8c59cd4, 0x8c59cb0), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8c59cd4, 0x8c59cb0, 0xfde02e18, 0x8386eea), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] os_aio_simulated_handle(global_segment = 5U, message1 = (nil), message2 = (nil), type = (nil)), line 3997 in "os0file.c"
  [9] fil_aio_wait(segment = 4259327928U), line 4412 in "fil0fil.c"
  [10] io_handler_thread(arg = (nil)), line 490 in "srv0start.c"
  [11] _thr_setup(0xfea82a00), at 0xfecd3a91 
  [12] _lwp_start(0x0, 0x0, 0x0, 0x8c59cb0, 0xfea82a00, 0xfecfe000), at 0xfecd3d80 
current thread: t@8
=>[1] __pollsys(0xfdd03f10, 0x0, 0xfdd03f60, 0x0), at 0xfecd4e35 
  [2] _pollsys(0xfdd03f10, 0x0, 0xfdd03f60, 0x0), at 0xfecc9464 
  [3] _pselect(0x0, 0xfed00310, 0xfed00310, 0xfed00310, 0xfdd03f60, 0x0), at 0xfec82fae 
  [4] _select(0x0, 0x0, 0x0, 0x0, 0xfdd03f90, 0x1), at 0xfec832a4 
  [5] os_thread_sleep(tm = 0), line 294 in "os0thread.c"
  [6] srv_lock_timeout_and_monitor_thread(arg = (nil)), line 1978 in "srv0srv.c"
  [7] _thr_setup(0xfea83200), at 0xfecd3a91 
  [8] _lwp_start(0xfdd03f10, 0x0, 0xfdd03f60, 0x0, 0x8b546c0, 0x0), at 0xfecd3d80 
current thread: t@9
=>[1] __pollsys(0xfdc04f10, 0x0, 0xfdc04f60, 0x0), at 0xfecd4e35 
  [2] _pollsys(0xfdc04f10, 0x0, 0xfdc04f60, 0x0), at 0xfecc9464 
  [3] _pselect(0x0, 0xfed00310, 0xfed00310, 0xfed00310, 0xfdc04f60, 0x0), at 0xfec82fae 
  [4] _select(0x0, 0x0, 0x0, 0x0, 0xfdc04f90, 0xfecbaa91), at 0xfec832a4 
  [5] os_thread_sleep(tm = 0), line 294 in "os0thread.c"
  [6] srv_error_monitor_thread(arg = (nil)), line 2214 in "srv0srv.c"
  [7] _thr_setup(0xfea83a00), at 0xfecd3a91 
  [8] _lwp_start(0xfdc04f10, 0x0, 0xfdc04f60, 0x0, 0x0, 0x0), at 0xfecd3d80 
current thread: t@10
=>[1] __fdsync(0x9, 0x10), at 0xfecd4885 
  [2] _fsync(0x9), at 0xfecc104e 
  [3] fsync(0x9, 0xc7188, 0xfdb05ef8, 0x837a0cc), at 0xfecc8a1d 
  [4] os_file_flush(file = 0), line 1881 in "os0file.c"
  [5] fil_flush(space_id = 0), line 4523 in "fil0fil.c"
  [6] log_write_up_to(lsn = 0, wait = 91U, flush_to_disk = 1U), line 1499 in "log0log.c"
  [7] log_buffer_sync_in_background(flush = 1U), line 1578 in "log0log.c"
  [8] srv_sync_log_buffer_in_background(), line 2284 in "srv0srv.c"
  [9] srv_master_thread(arg = (nil)), line 2513 in "srv0srv.c"
  [10] _thr_setup(0xfea84200), at 0xfecd3a91 
  [11] _lwp_start(0x9, 0x10, 0xfecfe000, 0xfdb05e98, 0xfecc8a1d, 0x9), at 0xfecd3d80 
current thread: t@11
=>[1] ___sigtimedwait(0xfea7eea0, 0xfea7eeb0, 0x0), at 0xfecd4205 
  [2] __sigtimedwait(0xfea7efb0, 0x0, 0x0), at 0xfecc9561 
  [3] _sigwait(0xfea7efb0), at 0xfecc95df 
  [4] __posix_sigwait(0xfea7efb0, 0xfea7ef8c, 0x0, 0x14), at 0xfecc1de6 
  [5] signal_hand(arg = (nil)), line 2772 in "mysqld.cc"
  [6] _thr_setup(0xfea84a00), at 0xfecd3a91 
  [7] _lwp_start(0xfea7eea0, 0xfea7eeb0, 0x0, 0x0, 0x0, 0xfecfe000), at 0xfecd3d80 
current thread: t@15
=>[1] ___nanosleep(0xfd9d56e8, 0xfd9d56f0), at 0xfecd4125 
  [2] _sleep(0x1, 0x2, 0xfd9d5720, 0x4abb9ba2, 0x4abb9ba3, 0x82ed054), at 0xfecc8664 
  [3] safe_sleep(thd = 0x903e8d0, sec = 0, thread_killed = 0x82ed054 = &`mysqld`slave.cc`io_slave_killed(THD*, Master_info*), thread_killed_arg = 0x9007f40), line 1860 in "slave.cc"
  [4] connect_to_master(thd = 0x903e8d0, mysql = (nil), mi = 0xfd9d57c8, reconnect = true, suppress_warnings = false), line 3862 in "slave.cc"
  [5] try_to_reconnect(thd = (nil), mysql = 0x9038170, mi = 0xfd9d5928, retry_count = (nil), suppress_warnings = false, messages = 0x86a3eb0), line 3906 in "slave.cc"
  [6] handle_slave_io(arg = 0xfd9d5fd8), line 2693 in "slave.cc"
  [7] _thr_setup(0xfea86200), at 0xfecd3a91 
  [8] _lwp_start(0xfd9d56e8, 0xfd9d56f0, 0xfd9d571c, 0x1, 0x1, 0x0), at 0xfecd3d80 
current thread: t@16
=>[1] __lwp_park(0x0, 0x0), at 0xfecd3df9 
  [2] cond_sleep_queue(0x8da4df4, 0x8da4dd0, 0x0), at 0xfecce50c 
  [3] cond_wait_queue(0x8da4df4, 0x8da4dd0, 0x0, 0x0), at 0xfecce61e 
  [4] _cond_wait(0x8da4df4, 0x8da4dd0), at 0xfecceb17 
  [5] cond_wait(0x8da4df4, 0x8da4dd0), at 0xfecceb59 
  [6] _pthread_cond_wait(0x8da4df4, 0x8da4dd0, 0x80, 0x838194a), at 0xfecceb92 
  [7] os_event_wait_low(event = (nil), reset_sig_count = 0), line 396 in "os0sync.c"
  [8] log_write_up_to(lsn = 0, wait = 92U, flush_to_disk = 1U), line 1407 in "log0log.c"
  [9] trx_prepare_off_kernel(trx = 0x9032e80), line 1904 in "trx0trx.c"
  [10] trx_prepare_for_mysql(trx = 0xfd9a4bc8), line 1934 in "trx0trx.c"
  [11] innobase_xa_prepare(hton = 0x890a378, thd = 0xfd9a4bf8, all = true), line 9025 in "ha_innodb.cc"
  [12] ha_commit_trans(thd = 0x906bb30, all = true), line 1172 in "handler.cc"
  [13] end_trans(thd = (nil), completion = COMMIT), line 723 in "sql_parse.cc"
  [14] apply_event_and_update_pos(ev = 0xfd9a4ce8, thd = (nil), rli = (nil), skip = true), line 1058 in "log_event.h"
  [15] handle_slave_sql(arg = 0x9007f40), line 2282 in "slave.cc"
  [16] _thr_setup(0xfea86a00), at 0xfecd3a91 
  [17] _lwp_start(0x0, 0x0, 0x0, 0x8da4dd0, 0xfea86a00, 0xfecfe000), at 0xfecd3d80 

 - deleting it, already saved 5

Retrying test, attempt(2/3)...
ndb.ndb_restore_print w3 [ pass ] 123573 
ndb.ndb_restore_compat w1 [ pass ] 145714 
ndb.ndb_row_format w3 [ pass ] 68780 
ndb.ndb_single_user w1 [ pass ] 72393 
ndb.ndb_sp w3 [ pass ] 14762 
ndb.ndb_temporary w3 [ pass ] 13624 
ndb.ndb_restore_different_endian_data w2 [ pass ] 248874 
rpl.rpl_circular_for_4_hosts w5 [ pass ] 211478 ndb.ndb_subquery w1 [ pass ] 89115 ndb.ndb_truncate w1 [ pass ] 25693 ndb.ndb_transaction w3 [ pass ] 92994 ndb.ndb_types w1 [ pass ] 4740 ndb.ndb_update_no_read w1 [ pass ] 16097 ndb.ndb_view w1 [ pass ] 8907
ndb.ndbapi w1 [ skipped ] Test requires: 'have_ndbapi_examples'
ndb.ndb_trigger w2 [ pass ] 148655 ndb.strict_autoinc_5ndb w2 [ pass ] 12008 ndb.ndb_update w3 [ pass ] 74274 ndb.ps_7ndb w1 [ pass ] 86563 ndb.ndb_binlog_basic w2 [ pass ] 66551 ndb.ndb_restore w4 [ pass ] 598715 ndb.ndb_binlog_format w2 [ pass ] 11136 ndb.ndb_binlog_discover w1 [ pass ] 55697 ndb.ndb_binlog_multi w2 [ pass ] 49807 ndb.ndb_binlog_log_bin w4 [ pass ] 75488 ndb.ndb_multi_row w1 [ pass ] 66434
rpl.rpl_innodb_bug30888                  w6 [ retry-pass ]  499031

Retrying test, attempt(3/3)...

ndb.ndb_binlog_ddl_multi w3 [ pass ] 214738
binlog.binlog_multi_engine w4 [ pass ] 23328 
rpl_ndb.rpl_ndb_circular_2ch w1 [ pass ] 28694
rpl.rpl_innodb_bug30888                  w6 [ retry-pass ]  230909