Test Error *********Starting Alter Load Test *********** *********************************************** Creating TABLE_ALTER.t1, please stand by.... Table TABLE_ALTER.t1 Created!!!!! Call loader Sleep 10 Loader is Connected to TABLE_ALTER.t1 Alter Table Add Column varchar(255) DEFAULT 'hello' Load continues Insert Data Error: Got error 157 'Unknown error code' from NDBCLUSTER at ./LoadTable.pl line 60. Jul 30 17:42 MySQLD (Remote) Insert testing Core: Core was generated by `/home/ndbdev/jmiller/builds/libexec/mysqld --basedir=/home/ndbdev/jmiller/build'. Program terminated with signal 11, Segmentation fault. #0 0x00b8d402 in __kernel_vsyscall () (gdb) bt #0 0x00b8d402 in __kernel_vsyscall () #1 0x00d2164f in ?? () #2 0x00000028 in ?? () #3 0x0843bf06 in ?? () #4 0xb4be3bb0 in ?? () #5 0x0000000b in ?? () #6 0x082760a9 in Qmgr::execNDB_FAILCONF (this=0xb, signal=0x33) at qmgr/QmgrMain.cpp:2637 #7 #8 0x00c0eaf0 in ?? () #9 0x0ac11268 in ?? () #10 0xb4be1868 in ?? () #11 0x0ac11268 in ?? () #12 0x0ac111b8 in ?? () #13 0x00d24ea0 in ?? () #14 0x00cc45a9 in ?? () #15 0x0869568f in ?? () #16 0x00cc45a9 in ?? () #17 0x00cd2880 in ?? () #18 0x00cd2888 in ?? () #19 0x00cd28a4 in ?? () #20 0x00000000 in ?? () (gdb) f 6 #6 0x082760a9 in Qmgr::execNDB_FAILCONF (this=0xb, signal=0x33) at qmgr/QmgrMain.cpp:2637 2637 ptrCheckGuard(failedNodePtr, MAX_NDB_NODES, nodeRec); (gdb) l 2632 CLEAR_ERROR_INSERT_VALUE; 2633 infoEvent("Discarding NDB_FAILCONF for %u", failedNodePtr.i); 2634 return; 2635 } 2636 2637 ptrCheckGuard(failedNodePtr, MAX_NDB_NODES, nodeRec); 2638 if (failedNodePtr.p->failState == WAITING_FOR_NDB_FAILCONF){ 2639 failedNodePtr.p->failState = NORMAL; 2640 } else { 2641 jam(); Error Log 070730 17:42:16 [Note] NDB Binlog: Ndb tables initially read only. 070730 17:42:16 [Note] NDB Binlog: DISCOVER TABLE Event: REPL$mysql/ndb_schema Error: Freeing unallocated data at line 51, 'ha_ndbcluster_binlog.h' Error: Freeing unallocated data at line 52, 'ha_ndbcluster_binlog.h' 070730 17:42:16 - mysqld got signal 6; Stack range sanity check OK, backtrace follows: 0x8275f5f 0xb8d402 0xbd5678 0xc09230 0xc108a8 0xc11587 0x86754aa 0x8649e38 0x85ba4b6 0x864a530 0x85f7bc9 0x85fa323 0x85fa551 0x85d74f3 0x85d786c 0x85d10f2 0x83c1405 0x83c21a5 0x83aeec9 0x838d686 0x839ce35 0x83bdbf1 0x83ad14e 0xd1ebd4 0xc764fe Stack Trace 0x8275f5f _Z10MYSQLparsePv + 83801 0xb8d402 (?) 0xbd5678 (?) 0xc09230 (?) 0xc108a8 (?) 0xc11587 (?) 0x86754aa _ZZN13Field_newdate10store_timeEP13st_mysql_time25enum_mysql_timestamp_typeE19__PRETTY_FUNCTION__ + 74 0x8649e38 _ZTV13Item_func_div + 408 0x85ba4b6 _Z24printBACKUP_COMPLETE_REPP8_IO_FILEPKjjt + 164 0x864a530 _ZTV13Item_func_abs + 432 0x85f7bc9 find_key_block + 684 0x85fa323 key_cache_write + 2041 0x85fa551 free_block + 438 0x85d74f3 _ZN6VectorIPK9ParserRowIN10ParserImpl5DummyEEE4backEv + 1 0x85d786c _ZN17SocketInputStream4getsEPci + 174 0x85d10f2 _ZN14PropertiesImpl4growEi + 90 0x83c1405 _Z17init_slave_threadP3THD14SLAVE_THD_TYPE + 223 0x83c21a5 _Z18end_relay_log_infoP17st_relay_log_info + 301 0x83aeec9 _ZN10field_real12get_opt_typeEP6Stringm + 1257 0x838d686 _Z17mysql_alter_tableP3THDPcS1_P24st_ha_create_informationP13st_table_listR4ListI12create_fieldERS6_I3KeyEjP8st_orderbP13st_alt + 5414 0x839ce35 _Z15add_status_varsP17st_mysql_show_var + 165 0x83bdbf1 _Z22terminate_slave_threadP3THDP15st_safe_mutex_tS2_P14pthread_cond_tPVj + 483 0x83ad14e _ZN13field_decimal3addEv + 1142 0xd1ebd4 (?) 0xc764fe (?) ---------------------------------------------------- Jul 30 17:42 MySQLD (Local to test) (Update Testing) Core: Core was generated by `/home/ndbdev/jmiller/builds/libexec/mysqld --basedir=/home/ndbdev/jmiller/build'. Program terminated with signal 11, Segmentation fault. #0 0x085ba5ab in printABORT_BACKUP_ORD (out=0x2, data=0x244244ba, len=0, b=27252) at BackupSignalData.cpp:109 109 sig->backupPtr, sig->backupId); (gdb) bt #0 0x085ba5ab in printABORT_BACKUP_ORD (out=0x2, data=0x244244ba, len=0, b=27252) at BackupSignalData.cpp:109 #1 0x085bb150 in printLCP_COMPLETE_REP (output=0x244244ba, theData=0x0, len=142816736, receiverBlockNo=2) at LCP.cpp:81 #2 0x085dbc3e in BaseString::assign (this=0x244244ba, s=0x0) at BaseString.cpp:102 #3 0x085d9d8c in SocketClient::init (this=0x971c800) at SocketClient.cpp:54 #4 0x085da445 in SocketAuthSimple (this=0x971c800, username=0x2

, passwd=0xb4c2a1c8 "\001") at SocketAuthenticator.cpp:27 #5 0x085afa4d in SHM_Transporter::setupBuffers (this=0x96ea970) at SHM_Transporter.cpp:111 #6 0x085affaa in SHM_Transporter::connect_client_impl (this=0x96ea970, sockfd=2) at SHM_Transporter.cpp:304 #7 0x08651a8a in Item_func_min_max::val_int()::__PRETTY_FUNCTION__ () #8 0x096ea970 in ?? () #9 0x00000002 in ?? () #10 0x00000000 in ?? () #0 0x085ba5ab in printABORT_BACKUP_ORD (out=0x2, data=0x244244ba, len=0, b=27252) at BackupSignalData.cpp:109 109 sig->backupPtr, sig->backupId); (gdb) l 104 fprintf(out, " BackupFailure: backupPtr: %d backupId: %d\n", 105 sig->backupPtr, sig->backupId); 106 return true; 107 case AbortBackupOrd::LogBufferFull: 108 fprintf(out, " LogBufferFull: backupPtr: %d backupId: %d\n", 109 sig->backupPtr, sig->backupId); 110 return true; 111 break; 112 case AbortBackupOrd::FileOrScanError: 113 fprintf(out, " FileOrScanError: backupPtr: %d backupId: %d\n", Error Log: Error: Freeing unallocated data at line 51, 'ha_ndbcluster_binlog.h' Error: Freeing unallocated data at line 52, 'ha_ndbcluster_binlog.h' 070730 17:42:30 - mysqld got signal 11; Stack range sanity check OK, backtrace follows: 0x8275f5f 0x530af0 0x531e78 0x533587 0x86754aa 0x85dce49 0x85d17c3 0x860c80e 0x85d7664 0x85d786c 0x85d10f2 0x83c1405 0x83c21a5 0x83aeec9 0x838d686 0x839ce35 0x83bdbf1 0x83ad213 0x640bd4 0x5984fe Stack Trace: Stack range sanity check OK, backtrace follows: 0x8275f5f _Z10MYSQLparsePv + 83801 0x530af0 (?) 0x531e78 (?) 0x533587 (?) 0x86754aa _ZZN13Field_newdate10store_timeEP13st_mysql_time25enum_mysql_timestamp_typeE19__PRETTY_FUNCTION__ + 74 0x85dce49 _ZN6VectorI10BaseStringE9push_backERKS0_ + 319 0x85d17c3 _ZNK10Properties7getCopyEPKcPPc + 141 0x860c80e set_dynamic + 189 0x85d7664 _ZN15FileInputStreamC2EP8_IO_FILE + 2 0x85d786c _ZN17SocketInputStream4getsEPci + 174 0x85d10f2 _ZN14PropertiesImpl4growEi + 90 0x83c1405 _Z17init_slave_threadP3THD14SLAVE_THD_TYPE + 223 0x83c21a5 _Z18end_relay_log_infoP17st_relay_log_info + 301 0x83aeec9 _ZN10field_real12get_opt_typeEP6Stringm + 1257 0x838d686 _Z17mysql_alter_tableP3THDPcS1_P24st_ha_create_informationP13st_table_listR4ListI12create_fieldERS6_I3KeyEjP8st_orderbP13st_alt + 5414 0x839ce35 _Z15add_status_varsP17st_mysql_show_var + 165 0x83bdbf1 _Z22terminate_slave_threadP3THDP15st_safe_mutex_tS2_P14pthread_cond_tPVj + 483 0x83ad213 _ZN14field_longlong3addEv + 165 0x640bd4 (?) 0x5984fe (?) ----------------------------------- Cluster Log: 2007-07-30 23:33:09 [MgmSrvr] ALERT -- Node 2: Node 3 Disconnected 2007-07-30 23:33:09 [MgmSrvr] INFO -- Node 2: Communication to Node 3 closed 2007-07-30 23:33:09 [MgmSrvr] ALERT -- Node 2: Network partitioning - arbitration required 2007-07-30 23:33:09 [MgmSrvr] INFO -- Node 2: President restarts arbitration thread [state=7] 2007-07-30 23:33:09 [MgmSrvr] ALERT -- Node 1: Node 3 Disconnected 2007-07-30 23:33:09 [MgmSrvr] ALERT -- Node 2: Arbitration won - positive reply from node 1 2007-07-30 23:33:09 [MgmSrvr] INFO -- Node 2: DICT: lock bs: 0 ops: 0 poll: 0 cnt: 0 queue: 2007-07-30 23:33:10 [MgmSrvr] ALERT -- Node 3: Forced node shutdown completed. Initiated by signal 11. Caused by error 6000: 'Error OS signal received(Internal error, programming error or missing error message, please report a bug). Temporary error, restart node'. 2007-07-30 23:33:10 [MgmSrvr] INFO -- Node 2: Started arbitrator node 1 [ticket=3bad0002190bfd71] 2007-07-30 23:33:12 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:10 [MgmSrvr] INFO -- Node 2: Started arbitrator node 1 [ticket=3bad0002190bfd71] 2007-07-30 23:33:12 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:12 [MgmSrvr] INFO -- Node 6: ndb_desc 2007-07-30 23:33:13 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:33:13 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:33:13 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:33:13 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:33:13 [MgmSrvr] INFO -- Node 2: Communication to Node 3 opened 2007-07-30 23:33:14 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:33:16 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:16 [MgmSrvr] INFO -- Node 6: ndb_desc 2007-07-30 23:33:16 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:33:16 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:33:17 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:33:17 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:33:17 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:33:17 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:33:20 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:33:21 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:21 [MgmSrvr] INFO -- Node 6: ndb_desc 2007-07-30 23:33:21 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:33:22 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:33:22 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:33:22 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:33:22 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:33:16 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:16 [MgmSrvr] INFO -- Node 6: ndb_desc 2007-07-30 23:33:16 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:33:16 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:33:17 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:33:17 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:33:17 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:33:17 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:33:20 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:33:21 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:21 [MgmSrvr] INFO -- Node 6: ndb_desc 2007-07-30 23:33:21 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:33:22 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:33:22 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:33:22 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:33:22 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:33:24 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:24 [MgmSrvr] INFO -- Node 6: ndb_desc 2007-07-30 23:33:25 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:33:25 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:33:26 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:33:26 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:33:26 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:33:27 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:33:29 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:33:30 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:30 [MgmSrvr] INFO -- Node 6: ndb_desc 2007-07-30 23:33:30 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:33:30 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:33:30 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:33:30 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:33:31 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:33:33 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:33:59 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:33:59 [MgmSrvr] INFO -- Node 6: ndb_show_tables 2007-07-30 23:33:59 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:34:00 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:34:00 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:34:00 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:34:00 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:34:03 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:34:03 [MgmSrvr] INFO -- Node 6: ndb_show_tables 2007-07-30 23:34:03 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:34:04 [MgmSrvr] INFO -- Node 2: Node 6 Connected 2007-07-30 23:34:04 [MgmSrvr] INFO -- Node 2: Node 6: API mysql-5.1.20 ndb-6.2.5 2007-07-30 23:34:04 [MgmSrvr] ALERT -- Node 2: Node 6 Disconnected 2007-07-30 23:34:04 [MgmSrvr] INFO -- Node 2: Communication to Node 6 closed 2007-07-30 23:34:04 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:34:07 [MgmSrvr] INFO -- Node 2: Local checkpoint 3 started. Keep GCI = 84 oldest restorable GCI = 85 2007-07-30 23:34:08 [MgmSrvr] INFO -- Node 2: Communication to Node 6 opened 2007-07-30 23:34:10 [MgmSrvr] ALERT -- Node 1: Node 2 Disconnected 2007-07-30 23:34:10 [MgmSrvr] ALERT -- Node 2: Forced node shutdown completed. Initiated by signal 6. Caused by error 6000: 'Error OS signal received(Internal error, programming error or missing error message, please report a bug). Temporary error, restart node'. 2007-07-30 23:34:11 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 3, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000072. 2007-07-30 23:34:11 [MgmSrvr] INFO -- Node 6: ndb_show_tables 2007-07-30 23:34:33 [MgmSrvr] INFO -- Mgmt server state: nodeid 6 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:49:59 [MgmSrvr] INFO -- Mgmt server state: nodeid 4 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000022. 2007-07-30 23:50:29 [MgmSrvr] INFO -- Mgmt server state: nodeid 4 reserved for ip 10.100.1.93, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000032. 2007-07-30 23:50:29 [MgmSrvr] INFO -- Node 4: mysqld --server-id=1 2007-07-30 23:52:25 [MgmSrvr] INFO -- Mgmt server state: nodeid 5 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000012. 2007-07-30 23:59:18 [MgmSrvr] INFO -- Shutting down server... 2007-07-30 23:59:31 [MgmSrvr] INFO -- Mgmt server state: nodeid 4 freed, m_reserved_nodes 0000000000000000000000000000000000000000000000000000000000000002. 2007-07-30 23:59:31 [MgmSrvr] INFO -- Shutdown complete ------------------------------- NDBD (ndb08) ID=2, Insert Testing NOTE: I made the mistake of blowing away the first set of logs and core files. The ones below are recreations. Core: #0 0x00b8d402 in __kernel_vsyscall () #1 0x00bd3f90 in raise () from /lib/libc.so.6 #2 0x00bd5678 in abort () from /lib/libc.so.6 #3 0x080e6795 in childAbort (code=Could not find the frame base for "childAbort(int, unsigned int)". ) at main.cpp:104 #4 0x082ece2a in NdbShutdown (type=NST_ErrorHandlerSignal, restartType=NRT_Default) at Emulator.cpp:254 #5 0x082f83a3 in ErrorReporter::handleError (messageID=6000, problemData=0xbfed25b4 "Signal 6 received; Aborted", objRef=0x8359593 "main.cpp", nst=NST_ErrorHandlerSignal) at ErrorReporter.cpp:210 #6 0x080e7a27 in handler_error (signum=6) at main.cpp:639 #7 #8 0x00b8d402 in __kernel_vsyscall () #9 0x00bd3f90 in raise () from /lib/libc.so.6 #10 0x00bd5678 in abort () from /lib/libc.so.6 #11 0x00bcd269 in __assert_fail () from /lib/libc.so.6 #12 0x082319be in Tup_varsize_page::get_index_word (this=0x77da0000, page_idx=61937) at dbtup/tuppage.hpp:169 #13 0x082319f1 in Tup_varsize_page::get_ptr (this=0x77da0000, page_idx=61937) at dbtup/tuppage.hpp:240 #14 0x08231f23 in Dbtup::get_ptr (this=0xb6f85008, pagePtr=0xbfed2b20, ref= {m_page_no = 4059165169, m_page_idx = 61937, static SZ32 = }) at dbtup/Dbtup.hpp:3269 #15 0x082cd36c in Dbtup::commit_operation (this=0xb6f85008, signal=0x83f71dc, gci=112, tuple_ptr=0x7ef7c6f4, pagePtr={p = 0x7ef78000, i = 44}, regOperPtr=0x7ea7434c, regFragPtr=0x7eca2b48, regTabPtr=0x8f172c4) at dbtup/DbtupCommit.cpp:268 #16 0x082ce5e0 in Dbtup::execTUP_COMMITREQ (this=0xb6f85008, signal=0x83f71dc) at dbtup/DbtupCommit.cpp:677 #17 0x0810b417 in SimulatedBlock::executeFunction (this=0xb6f85008, gsn=419, signal=0x83f71dc) at ./SimulatedBlock.hpp:581 #18 0x0810b571 in SimulatedBlock::EXECUTE_DIRECT (this=0x8e9e5e0, block=249, gsn=419, signal=0x83f71dc, len=4) at ../../../../../storage/ndb/src/kernel/vm/SimulatedBlock.hpp:758 #19 0x081bf74e in Dblqh::commitContinueAfterBlockedLab (this=0x8e9e5e0, signal=0x83f71dc) at dblqh/DblqhMain.cpp:6426 #20 0x081bfbe1 in Dblqh::localCommitLab (this=0x8e9e5e0, signal=0x83f71dc) at dblqh/DblqhMain.cpp:6375 ---Type to continue, or q to quit--- #21 0x081c520f in Dblqh::commitReqLab (this=0x8e9e5e0, signal=0x83f71dc, gci=112) at dblqh/DblqhMain.cpp:6288 #22 0x081df42a in Dblqh::execCOMMIT (this=0x8e9e5e0, signal=0x83f71dc) at dblqh/DblqhMain.cpp:5956 #23 0x081e02e6 in Dblqh::execPACKED_SIGNAL (this=0x8e9e5e0, signal=0x83f71dc) at dblqh/DblqhMain.cpp:2481 #24 0x0810b417 in SimulatedBlock::executeFunction (this=0x8e9e5e0, gsn=342, signal=0x83f71dc) at ./SimulatedBlock.hpp:581 #25 0x082e9ca2 in FastScheduler::doJob (this=0x83f3140) at FastScheduler.cpp:136 #26 0x082eabde in ThreadConfig::ipControlLoop (this=0x8d32aa8) at ThreadConfig.cpp:153 #27 0x080e7899 in main (argc=3, argv=0xbfed3644) at main.cpp:473 (gdb) f 12 #12 0x082319be in Tup_varsize_page::get_index_word (this=0x77da0000, page_idx=61937) at dbtup/tuppage.hpp:169 169 assert(page_idx < high_index); (gdb) l 164 assert(page_idx < high_index); 165 return (m_data + (DATA_WORDS - page_idx)); 166 } 167 168 Uint32 get_index_word(Uint32 page_idx) const { 169 assert(page_idx < high_index); 170 return * (m_data + (DATA_WORDS - page_idx)); 171 } 172 173 /** (gdb) print page_idx $1 = 61937 (gdb) (gdb) print high_index Cannot access memory at address 0x77da005c (gdb) ndb_2_error.log: Time: Monday 30 July 2007 - 23:34:09 Status: Temporary error, restart node Message: Error OS signal received (Internal error, programming error or missing error message, please report a bug) Error: 6000 Error data: Signal 6 received; Aborted Error object: main.cpp Program: /home/ndbdev/jmiller/builds/libexec/ndbd Pid: 15277 Trace: /space/run/ndb_2_trace.log.1 Version: mysql-5.1.20 ndb-6.2.5-beta ndb_2_out.log: ndbd: ../../../../../storage/ndb/src/kernel/blocks/dblqh/../dbtup/tuppage.hpp:169: Uint32 Tup_varsize_page::get_index_word(Uint32) const: Assertion `page_idx < high_index' failed. 2007-07-30 23:34:09 [ndbd] INFO -- Received signal 6. Running error handler. 2007-07-30 23:34:09 [ndbd] INFO -- Signal 6 received; Aborted 2007-07-30 23:34:09 [ndbd] INFO -- main.cpp 2007-07-30 23:34:09 [ndbd] INFO -- Error handler signal shutting down system 2007-07-30 23:34:10 [ndbd] INFO -- Error handler shutdown completed - aborting 2007-07-30 23:34:10 [ndbd] ALERT -- Node 2: Forced node shutdown completed. Initiated by signal 6. Caused by error 6000: 'Error OS signal received(Internal error, programming error or missing error message, please report a bug). Temporary error, restart node'. ndb_2_trace.log.1 DBLQH 006297 006318 006321 006356 DBLQH 006245 006287 006374 006420 DBTUP 020482 020647 020664 020713 030499 030609 030614 015368 015425 030884 030796 030796 030798 030796 030798 030906 015368 015569 015506 015368 015809 015928 030734 030991 SUMA 003255 003274 DBTUP 030996 030755 030991 SUMA 003255 003274 DBTUP 030996 031026 031053 SUMA 003398 003372 003372 003372 020675 --------------- Signal ---------------- r.bn: 247 "DBLQH", r.proc: 2, r.sigId: 1410144 gsn: 342 "PACKED_SIGNAL" prio: 1 s.bn: 245 "DBTC", s.proc: 2, s.sigId: 1410143 length: 4 trace: 1 #sec: 0 fragInf: 0 Signal data: H'0000000b H'00000070 H'00012e71 H'00500400 --------- Begin Packed Signals -------- --------------- Signal ---------------- r.bn: 247 "DBLQH", length: 4 "COMMIT" Signal data: H'0000000b H'00000070 H'00012e71 H'00500400 --------- End Packed Signals ---------- --------------- Signal ---------------- r.bn: 245 "DBTC", r.proc: 2, r.sigId: 1410143 gsn: 19 "TC_COMMITREQ" prio: 1 s.bn: 32773 "API", s.proc: 4, s.sigId: 0 length: 3 trace: 1 #sec: 0 fragInf: 0 H'00000025 H'00012e71 H'00500400 --------------- Signal ---------------- r.bn: 253 "NDBFS", r.proc: 2, r.sigId: 1410142 gsn: 261 "FSOPENREQ" prio: 0 s.bn: 250 "DBDICT", s.proc: 2, s.sigId: 1410141 length: 11 trace: 0 #sec: 0 fragInf: 0 UserReference: H'00fa0002, userPointer: H'00000000 FileNumber[1-4]: H'ffffffff H'ffffffff H'ffffffff H'01050100 FileFlags: H'00000011 Open write only --------------- Signal ---------------- r.bn: 250 "DBDICT", r.proc: 2, r.sigId: 1410141 gsn: 602 "ALTER_TAB_CONF" prio: 1 s.bn: 249 "DBTUP", s.proc: 2, s.sigId: 1410140 length: 8 trace: 0 #sec: 0 fragInf: 0 H'00f90002 H'00000079 H'00000042 H'00000005 H'01000009 H'00000000 H'00000001 H'ffffff00 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 2, r.sigId: 1410140 gsn: 600 "ALTER_TAB_REQ" prio: 1 s.bn: 250 "DBDICT", s.proc: 2, s.sigId: 1410139 length: 12 trace: 0 #sec: 0 fragInf: 0 H'00fa0002 H'00000079 H'80050005 H'00000000 H'00000042 H'00000005 H'01000009 H'00000000 H'00000001 H'00000001 H'00000001 H'00000001 --------------- Signal ---------------- ------------------------------- NDBD (ndb07) ID=2, Insert Testing Core: (gdb) bt #0 0x00522402 in __kernel_vsyscall () #1 0x00568f90 in raise () from /lib/libc.so.6 #2 0x0056a678 in abort () from /lib/libc.so.6 #3 0x080e6795 in childAbort (code=Could not find the frame base for "childAbort(int, unsigned int)". ) at main.cpp:104 #4 0x082ece2a in NdbShutdown (type=NST_ErrorHandlerSignal, restartType=NRT_Default) at Emulator.cpp:254 #5 0x082f83a3 in ErrorReporter::handleError (messageID=6000, problemData=0xbfa0d744 "Signal 11 received; Segmentation fault", objRef=0x8359593 "main.cpp", nst=NST_ErrorHandlerSignal) at ErrorReporter.cpp:210 #6 0x080e7a27 in handler_error (signum=11) at main.cpp:639 #7 #8 0x0823518a in Dbtup::varsize_reader (this=0xb6ebe008, outBuffer=0x83f7228 "\004", req_struct=0xbfa0dd84, ah_out=0x83f7238, attr_des2=0, srcPtr=0x46af47ac, srcBytes=2) at dbtup/DbtupRoutines.cpp:641 #9 0x08235a17 in Dbtup::readDynFixedSizeShrunkenNotNULL (this=0xb6ebe008, outBuffer=0x83f7228 "\004", req_struct=0xbfa0dd84, ahOut=0x83f7238, attrDes2=0) at dbtup/DbtupRoutines.cpp:903 #10 0x08235c39 in Dbtup::readDynFixedSizeShrunkenNULLable (this=0xb6ebe008, outBuffer=0x83f7228 "\004", req_struct=0xbfa0dd84, ahOut=0x83f7238, attrDes2=0) at dbtup/DbtupRoutines.cpp:934 #11 0x08235cb7 in Dbtup::readDynFixedSizeNULLable (this=0xb6ebe008, outBuffer=0x83f7228 "\004", req_struct=0xbfa0dd84, ahOut=0x83f7238, attrDes2=0) at dbtup/DbtupRoutines.cpp:815 #12 0x0823ae2c in Dbtup::readAttributes (this=0xb6ebe008, req_struct=0xbfa0dd84, inBuffer=0xb6ee7f88, inBufLen=5, outBuf=0x83f7228, maxRead=4126, xfrm_flag=false) at dbtup/DbtupRoutines.cpp:380 #13 0x0822c677 in Dbtup::interpreterStartLab (this=0xb6ebe008, signal=0x83f71dc, req_struct=0xbfa0dd84) at dbtup/DbtupExecQuery.cpp:1772 #14 0x0822cc90 in Dbtup::handleReadReq (this=0xb6ebe008, signal=0x83f71dc, regOperPtr=0x7e9ad6dc, regTabPtr=0x9bdb31c, req_struct=0xbfa0dd84) at dbtup/DbtupExecQuery.cpp:976 #15 0x0823114b in Dbtup::execTUPKEYREQ (this=0xb6ebe008, signal=0x83f71dc) at dbtup/DbtupExecQuery.cpp:731 #16 0x0810b417 in SimulatedBlock::executeFunction (this=0xb6ebe008, gsn=436, signal=0x83f71dc) at ./SimulatedBlock.hpp:581 ---Type to continue, or q to quit---f 8 #17 0x0810b571 in SimulatedBlock::EXECUTE_DIRECT (this=0x9b62648, block=249, gsn=436, signal=0x83f71dc, len=18) at ../../../../../storage/ndb/src/kernel/vm/SimulatedBlock.hpp:758 #18 0x081c241b in Dblqh::next_scanconf_tupkeyreq (this=0x9b62648, signal=0x83f71dc, scanPtr= {p = 0x9c8bcd8, i = 0}, regTcPtr=0x6f2f7ddc, fragPtrP=0x70660ad0, disk_page=4294967040) at dblqh/DblqhMain.cpp:9241 #19 0x081e3b7c in Dblqh::nextScanConfLoopLab (this=0x9b62648, signal=0x83f71dc) at dblqh/DblqhMain.cpp:9140 #20 0x081e4033 in Dblqh::nextScanConfScanLab (this=0x9b62648, signal=0x83f71dc) at dblqh/DblqhMain.cpp:9105 #21 0x081e4349 in Dblqh::execNEXT_SCANCONF (this=0x9b62648, signal=0x83f71dc) at dblqh/DblqhMain.cpp:7814 #22 0x0810b417 in SimulatedBlock::executeFunction (this=0x9b62648, gsn=330, signal=0x83f71dc) at ./SimulatedBlock.hpp:581 #23 0x0810b571 in SimulatedBlock::EXECUTE_DIRECT (this=0xb6ebe008, block=247, gsn=330, signal=0x83f71dc, len=6) at ../../../../../storage/ndb/src/kernel/vm/SimulatedBlock.hpp:758 #24 0x08256a1f in Dbtup::scanReply (this=0xb6ebe008, signal=0x83f71dc, scanPtr= {p = 0x9c07900, i = 0}) at dbtup/DbtupScan.cpp:395 #25 0x08256f27 in Dbtup::execACC_CHECK_SCAN (this=0xb6ebe008, signal=0x83f71dc) at dbtup/DbtupScan.cpp:268 #26 0x0810b417 in SimulatedBlock::executeFunction (this=0xb6ebe008, gsn=72, signal=0x83f71dc) at ./SimulatedBlock.hpp:581 #27 0x0810b571 in SimulatedBlock::EXECUTE_DIRECT (this=0xb6ebe008, block=249, gsn=72, signal=0x83f71dc, len=2) at ../../../../../storage/ndb/src/kernel/vm/SimulatedBlock.hpp:758 #28 0x08257953 in Dbtup::execNEXT_SCANREQ (this=0xb6ebe008, signal=0x83f71dc) at dbtup/DbtupScan.cpp:216 #29 0x0810b417 in SimulatedBlock::executeFunction (this=0xb6ebe008, gsn=332, signal=0x83f71dc) at ./SimulatedBlock.hpp:581 #30 0x082e9ca2 in FastScheduler::doJob (this=0x83f3140) at FastScheduler.cpp:136 #31 0x082eabde in ThreadConfig::ipControlLoop (this=0x99f6aa8) at ThreadConfig.cpp:153 #32 0x080e7899 in main (argc=3, argv=0xbfa0eb14) at main.cpp:473 (gdb) f 8 #8 0x0823518a in Dbtup::varsize_reader (this=0xb6ebe008, outBuffer=0x83f7228 "\004", req_struct=0xbfa0dd84, ah_out=0x83f7238, attr_des2=0, srcPtr=0x46af47ac, srcBytes=2) at dbtup/DbtupRoutines.cpp:641 641 memcpy(dst, srcPtr, srcBytes); (gdb) l 636 if (! charsetFlag || ! req_struct->xfrm_flag) 637 { 638 if (newIndexBuf <= max_read) { 639 jam(); 640 ah_out->setByteSize(srcBytes); 641 memcpy(dst, srcPtr, srcBytes); 642 zero32(dst, srcBytes); 643 req_struct->out_buf_index= newIndexBuf; 644 return true; 645 } ndb_3_error.log Time: Monday 30 July 2007 - 23:33:09 Status: Temporary error, restart node Message: Error OS signal received (Internal error, programming error or missing error message, please report a bug) Error: 6000 Error data: Signal 11 received; Segmentation fault Error object: main.cpp Program: /home/ndbdev/jmiller/builds/libexec/ndbd Pid: 27444 Trace: /space/run/ndb_3_trace.log.2 Version: mysql-5.1.20 ndb-6.2.5-beta ***EOM*** ndb_3_out.log Dbdict: name=TABLE_ALTER/def/t1,id=5,obj_ptr_i=5 Dbdict: name=sys/def/5/PRIMARY,id=6,obj_ptr_i=6 table 5 options 0 Dbdict: name=TABLE_ALTER/def/t1,id=5,obj_ptr_i=5 Dbdict: name=sys/def/5/PRIMARY,id=6,obj_ptr_i=6 table 5 options 0 Dbdict: name=TABLE_ALTER/def/t1,id=5,obj_ptr_i=5 Dbdict: name=sys/def/5/PRIMARY,id=6,obj_ptr_i=6 table 5 options 0 Dbdict: name=TABLE_ALTER/def/t1,id=5,obj_ptr_i=5 Dbdict: name=sys/def/5/PRIMARY,id=6,obj_ptr_i=6 table 5 options 0 Dbdict: name=TABLE_ALTER/def/t1,id=5,obj_ptr_i=5 Dbdict: name=sys/def/5/PRIMARY,id=6,obj_ptr_i=6 table 5 options 0 2007-07-30 23:33:09 [ndbd] INFO -- Received signal 11. Running error handler. 2007-07-30 23:33:09 [ndbd] INFO -- Signal 11 received; Segmentation fault 2007-07-30 23:33:09 [ndbd] INFO -- main.cpp 2007-07-30 23:33:09 [ndbd] INFO -- Error handler signal shutting down system 2007-07-30 23:33:09 [ndbd] INFO -- Error handler shutdown completed - aborting 2007-07-30 23:33:10 [ndbd] ALERT -- Node 3: Forced node shutdown completed. Initiated by signal 11. Caused by error 6000: 'Error OS signal received(Internal error, programming error or missing error message, please report a bug). Temporary error, restart node'. DBLQH 002646 002669 002978 009358 009369 009406 002978 DBTUP 032396 DBTUP 032217 DBTUP 032137 032148 032150 DBTUP 032223 032260 032816 032825 032836 032873 032284 032368 DBLQH 007776 007781 007813 009099 002978 009104 009216 DBTUP 000609 000063 000100 000119 000726 000948 000975 001733 001762 015368 015425 015368 015569 015506 015368 015809 015887 015639 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654222 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654221 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654221 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654220 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654220 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654219 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654219 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654218 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654218 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654217 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654217 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654216 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654216 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654215 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654215 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654214 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654214 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654213 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffffff H'00000002 --------------- Signal ---------------- r.bn: 249 "DBTUP", r.proc: 3, r.sigId: 654213 gsn: 332 "NEXT_SCANREQ" prio: 1 s.bn: 247 "DBLQH", s.proc: 3, s.sigId: 654212 length: 3 trace: 4 #sec: 0 fragInf: 0 H'00000000 H'ffffff00 H'00000001 --------------- Signal ---------------- r.bn: 247 "DBLQH", r.proc: 3, r.sigId: 654212 gsn: 28 "SCAN_NEXTREQ" prio: 1 s.bn: 244 "BACKUP", s.proc: 3, s.sigId: 654211 length: 6 trace: 4 #sec: 0 fragInf: 0 senderData: H'00000000 transId(1, 2): (H'00000000, H'0f400300) Close scan: 0 H'00000000 H'00000000 H'00000000 H'0f400300 H'00000010 H'00000000