Log has core dumps, stack traces and back traces


Local MySQLD:

From test:

***********************************************
Creating TABLE_ALTER.t1, please stand by....
Table TABLE_ALTER.t1 Created!!!!!
Call loader
Sleep 10
Text loader connected to TABLE_ALTER.t1
Alter Table Add Column TINYTEXT
DBD::mysql::st execute failed: Lost connection to MySQL server during query at ./ndb_alter_dd.pl line 539.
DBD::mysql::st execute failed: Lost connection to MySQL server during query at ./ndb_alter_dd.pl line 539.
[ndbdev@ndb09 cid]$ Load continues
Insert Data Error: MySQL server has gone away at ./LoadTableText.pl line 57.
*** glibc detected *** /home/ndbdev/jmiller/builds/libexec/mysqld: double free or corruption (fasttop): 0x0adbb7d8 ***
======= Backtrace: =========
/lib/libc.so.6[0x5311e0]
/lib/libc.so.6(__libc_free+0x77)[0x53172b]
/home/ndbdev/jmiller/builds/libexec/mysqld(_ZdlPv+0x17)[0x8602b63]
/home/ndbdev/jmiller/builds/libexec/mysqld(_Z27ndbcluster_create_event_opsP3THDP19st_ndbcluster_sharePKN13NdbDictionary5TableEPKc+0xbfd)[0x836a7af]
/home/ndbdev/jmiller/builds/libexec/mysqld(_Z30ndbcluster_create_binlog_setupP3NdbPKcjS2_S2_c+0x6fa)[0x836b0ae]
/home/ndbdev/jmiller/builds/libexec/mysqld(_ZN13ha_ndbcluster6createEPKcP8st_tableP24st_ha_create_information+0x406)[0x835900a]
/home/ndbdev/jmiller/builds/libexec/mysqld(_Z27ha_create_table_from_engineP3THDPKcS2_+0x328)[0x833a542]
/home/ndbdev/jmiller/builds/libexec/mysqld(_Z28ndb_create_table_from_engineP3THDPKcS2_+0x5a)[0x83487f0]
/home/ndbdev/jmiller/builds/libexec/mysqld(_Z36ndbcluster_setup_binlog_table_sharesP3THD+0x58)[0x8366fde]
/home/ndbdev/jmiller/builds/libexec/mysqld(ndb_util_thread_func+0x4d5)[0x835759b]
/lib/libpthread.so.0[0x640bd4]
/lib/libc.so.6(__clone+0x5e)[0x5984fe]
======= Memory map: ========
00111000-00116000 r-xp 00000000 03:02 2119572    /lib/libcrypt-2.3.6.so
00116000-00117000 r-xp 00004000 03:02 2119572    /lib/libcrypt-2.3.6.so
00117000-00118000 rwxp 00005000 03:02 2119572    /lib/libcrypt-2.3.6.so
00118000-0013f000 rwxp 00118000 00:00 0
004af000-004b0000 r-xp 004af000 00:00 0          [vdso]
004b0000-004ca000 r-xp 00000000 03:02 2119522    /lib/ld-2.3.6.so
004ca000-004cb000 r-xp 00019000 03:02 2119522    /lib/ld-2.3.6.so
004cb000-004cc000 rwxp 0001a000 03:02 2119522    /lib/ld-2.3.6.so
004ce000-005f1000 r-xp 00000000 03:02 2119529    /lib/libc-2.3.6.so
005f1000-005f3000 r-xp 00122000 03:02 2119529    /lib/libc-2.3.6.so
005f3000-005f5000 rwxp 00124000 03:02 2119529    /lib/libc-2.3.6.so
005f5000-005f7000 rwxp 005f5000 00:00 0
005f9000-005fb000 r-xp 00000000 03:02 2119537    /lib/libdl-2.3.6.so
005fb000-005fc000 r-xp 00001000 03:02 2119537    /lib/libdl-2.3.6.so
005fc000-005fd000 rwxp 00002000 03:02 2119537    /lib/libdl-2.3.6.so
005ff000-00622000 r-xp 00000000 03:02 2119541    /lib/libm-2.3.6.so
00622000-00623000 r-xp 00022000 03:02 2119541    /lib/libm-2.3.6.so
00623000-00624000 rwxp 00023000 03:02 2119541    /lib/libm-2.3.6.so
00626000-00638000 r-xp 00000000 03:02 66466      /usr/lib/libz.so.1.2.2.2
00638000-00639000 rwxp 00011000 03:02 66466      /usr/lib/libz.so.1.2.2.2
0063b000-00649000 r-xp 00000000 03:02 2119533    /lib/libpthread-2.3.6.so
00649000-0064a000 r-xp 0000d000 03:02 2119533    /lib/libpthread-2.3.6.so
0064a000-0064b000 rwxp 0000e000 03:02 2119533    /lib/libpthread-2.3.6.so
0064b000-0064d000 rwxp 0064b000 00:00 0
0064f000-0065e000 r-xp 00000000 03:02 2119548    /lib/libresolv-2.3.6.so
0065e000-0065f000 r-xp 0000e000 03:02 2119548    /lib/libresolv-2.3.6.so
0065f000-00660000 rwxp 0000f000 03:02 2119548    /lib/libresolv-2.3.6.so
00660000-00662000 rwxp 00660000 00:00 0
00664000-0066c000 r-xp 00000000 03:02 2119560    /lib/librt-2.3.6.so
0066c000-0066d000 r-xp 00007000 03:02 2119560    /lib/librt-2.3.6.so
0066d000-0066e000 rwxp 00008000 03:02 2119560    /lib/librt-2.3.6.so
0066e000-00678000 rwxp 0066e000 00:00 0
006b6000-006bf000 r-xp 00000000 03:02 2119544    /lib/libgcc_s-4.0.2-20051126.so.1
006bf000-006c0000 rwxp 00009000 03:02 2119544    /lib/libgcc_s-4.0.2-20051126.so.1
009a5000-009b6000 r-xp 00000000 03:02 2119569    /lib/libnsl-2.3.6.so
009b6000-009b7000 r-xp 00010000 03:02 2119569    /lib/libnsl-2.3.6.so
009b7000-009b8000 rwxp 00011000 03:02 2119569    /lib/libnsl-2.3.6.so
009b8000-009ba000 rwxp 009b8000 00:00 0
00d53000-00d57000 r-xp 00000000 03:02 2119568    /lib/libnss_dns-2.3.6.so
00d57000-00d58000 r-xp 00003000 03:02 2119568    /lib/libnss_dns-2.3.6.so
00d58000-00d59000 rwxp 00004000 03:02 2119568    /lib/libnss_dns-2.3.6.so
00fbc000-00fc5000 r-xp 00000000 03:02 2119571    /lib/libnss_files-2.3.6.so
00fc5000-00fc6000 r-xp 00008000 03:02 2119571    /lib/libnss_files-2.3.6.so
00fc6000-00fc7000 rwxp 00009000 03:02 2119571    /lib/libnss_files-2.3.6.so
08048000-08764000 r-xp 00000000 03:02 2414138    /home/ndbdev/jmiller/builds/libexec/mysqld
08764000-087b7000 rw-p 0071b000 03:02 2414138    /home/ndbdev/jmiller/builds/libexec/mysqld
087b7000-087ca000 rw-p 087b7000 00:00 0
0a6e3000-0adc8000 rw-p 0a6e3000 00:00 0          [heap]
b17eb000-b1f21000 rw-p b17eb000 00:00 0
b1f21000-b2000000 ---p b1f21000 00:00 0
b201e000-b201f000 ---p b201e000 00:00 0
b201f000-b2a1f000 rw-p b201f000 00:00 0
b2a1f000-b2a20000 ---p b2a1f000 00:00 0
b2a20000-b3420000 rw-p b2a20000 00:00 0
b3420000-b3421000 ---p b3420000 00:00 0
b3421000-b3e21000 rw-p b3421000 00:00 0
b3e21000-b3e22000 ---p b3e21000 00:00 0
b3e22000-b4a28000 rw-p b3e22000 00:00 0
b4a55000-b4a88000 rw-p b4a55000 00:00 0
b4a88000-b4a89000 ---p b4a88000 00:00 0
b4a89000-b4ba5000 rw-p b4a89000 00:00 0
b4ba5000-b4ba6000 ---p b4ba5000 00:00 0
b4ba6000-b4bd6000 rw-p b4ba6000 00:00 0
b4bd6000-b4bd7000 ---p b4bd6000 00:00 0
b4bd7000-b4c07000 rw-p b4bd7000 00:00 0
b4c07000-b4c08000 ---p b4c07000 00:00 0
b4c08000-b4c0f000 rw-p b4c08000 00:00 0
b4c0f000-b4c10000 ---p b4c0f000 00:00 0
b4c10000-b4c17000 rw-p b4c10000 00:00 0
b4c17000-b4c18000 ---p b4c17000 00:00 0
b4c18000-b4c1f000 rw-p b4c18000 00:00 0
b4c1f000-b4c20000 ---p b4c1f000 00:00 0
b4c20000-b4c27000 rw-p b4c20000 00:00 0
b4c27000-b4c28000 ---p b4c27000 00:00 0
b4c28000-b4cb1000 rw-p b4c28000 00:00 0
b4cb1000-b4cb2000 ---p b4cb1000 00:00 0
b4cb2000-b56b2000 rw-p b4cb2000 00:00 0
b56b2000-b56b3000 ---p b56b2000 00:00 0
b56b3000-b60b3000 rw-p b56b3000 00:00 0
b60b3000-b60b4000 ---p b60b3000 00:00 0
b60b4000-b75db000 rw-p b60b4000 00:00 0
b75db000-b75dc000 ---p b75db000 00:00 0
b75dc000-b7fdf000 rw-p b75dc000 00:00 0
b7fe7000-b7fe8000 rw-p b7fe7000 00:00 0
bf8d2000-bf8e7000 rw-p bf8d2000 00:00 0          [stack]


From Stack Trace 

ndbdev@ndb08:/space/var> resolve_stack_dump -s mysqld.sym -n mysql.stack
0x82382ed handle_segfault + 559
0x4af402 (?)
0x4f7678 (?)
0x4ef269 (?)
0x836a3ad _Z27ndbcluster_create_event_opsP3THDP19st_ndbcluster_sharePKN13NdbDictionary5TableEPKc + 2043
0x836e631 _Z24ndb_handle_schema_changeP3THDP3NdbP17NdbEventOperationP19st_ndbcluster_share + 1925
0x836fe07 _Z39ndb_binlog_thread_handle_non_data_eventP3THDP3NdbP17NdbEventOperationR20ndb_binlog_index_row + 827
0x8371935 ndb_binlog_thread_func + 6937
0x640bd4 (?)
0x5984fe (?)


From Remote MySQLD:

Stack trace #1 

0x82382ed handle_segfault + 559
0xb8d402 (?)
0xbd5678 (?)
0xbcd269 (?)
0x836a3ad _Z27ndbcluster_create_event_opsP3THDP19st_ndbcluster_sharePKN13NdbDictionary5TableEPKc + 2043
0x836e631 _Z24ndb_handle_schema_changeP3THDP3NdbP17NdbEventOperationP19st_ndbcluster_share + 1925
0x836fe07 _Z39ndb_binlog_thread_handle_non_data_eventP3THDP3NdbP17NdbEventOperationR20ndb_binlog_index_row + 827
0x8371935 ndb_binlog_thread_func + 6937
0xd1ebd4 (?)
0xc764fe (?)

From Stack Trace #2

0x82382ed handle_segfault + 559
0x4af402 (?)
0x4f7678 (?)
0x52b230 (?)
0x5311e0 (?)
0x53172b (?)
0x8602b63 _ZdlPv + 23
0x836a7af _Z27ndbcluster_create_event_opsP3THDP19st_ndbcluster_sharePKN13NdbDictionary5TableEPKc + 3069
0x836b0ae _Z30ndbcluster_create_binlog_setupP3NdbPKcjS2_S2_c + 1786
0x835900a _ZN13ha_ndbcluster6createEPKcP8st_tableP24st_ha_create_information + 1030
0x833a542 _Z27ha_create_table_from_engineP3THDPKcS2_ + 808
0x83487f0 _Z28ndb_create_table_from_engineP3THDPKcS2_ + 90
0x8366fde _Z36ndbcluster_setup_binlog_table_sharesP3THD + 88
0x835759b ndb_util_thread_func + 1237
0x640bd4 (?)
0x5984fe (?)


Core Dump:
(gdb) bt
#0  0x00b8d402 in __kernel_vsyscall ()
#1  0x00d2164f in pthread_kill () from /lib/libpthread.so.0
#2  0x083da2c9 in write_core (sig=6) at stacktrace.c:229
#3  0x082383fd in handle_segfault (sig=6) at mysqld.cc:2240
#4  <signal handler called>
#5  0x00b8d402 in __kernel_vsyscall ()
#6  0x00bd3f90 in raise () from /lib/libc.so.6
#7  0x00bd5678 in abort () from /lib/libc.so.6
#8  0x00bcd269 in __assert_fail () from /lib/libc.so.6
#9  0x0836a3ad in ndbcluster_create_event_ops (thd=0x9621770, share=0xb18a9ca8,
    ndbtab=0xb1859bc8, event_name=0xb185fc30 "REPL$TABLE_ALTER/t1")
    at ha_ndbcluster_binlog.cc:3140
#10 0x0836e631 in ndb_handle_schema_change (thd=0x9621770, ndb=0x96577d8, pOp=0xb187e650,
    share=0xb18a9ca8) at ha_ndbcluster_binlog.cc:1820
#11 0x0836fe07 in ndb_binlog_thread_handle_non_data_event (thd=0x9621770, ndb=0x96577d8,
    pOp=0xb187e650, row=@0xb4c052ac) at ha_ndbcluster_binlog.cc:3616
#12 0x08371935 in ndb_binlog_thread_func (arg=0x0) at ha_ndbcluster_binlog.cc:4547
#13 0x00d1ebd4 in start_thread () from /lib/libpthread.so.0
#14 0x00c764fe in clone () from /lib/libc.so.6
(gdb) f 9
#9  0x0836a3ad in ndbcluster_create_event_ops (thd=0x9621770, share=0xb18a9ca8,
    ndbtab=0xb1859bc8, event_name=0xb185fc30 "REPL$TABLE_ALTER/t1")
    at ha_ndbcluster_binlog.cc:3140
3140              DBUG_ASSERT(share->flags & NSF_BLOB_FLAG);
(gdb) p
The history is empty.
(gdb) l
3135              attr1.rec= op->getPreValue(col_name);
3136            }
3137            else
3138            {
3139              DBUG_PRINT("info", ("%s blob", col_name));
3140              DBUG_ASSERT(share->flags & NSF_BLOB_FLAG);
3141              attr0.blob= op->getBlobHandle(col_name);
3142              attr1.blob= op->getPreBlobHandle(col_name);
3143              if (attr0.blob == NULL || attr1.blob == NULL)
3144              {
(gdb) print share->flags
$1 = 0
(gdb) print
$2 = 0
(gdb) print NSF_BLOB_FLAG
No symbol "NSF_BLOB_FLAG" in current context.
(gdb) quit