=== modified file 'storage/ndb/include/kernel/GlobalSignalNumbers.h' --- storage/ndb/include/kernel/GlobalSignalNumbers.h 2007-12-14 12:48:05 +0000 +++ storage/ndb/include/kernel/GlobalSignalNumbers.h 2008-06-26 11:57:56 +0000 @@ -181,7 +181,7 @@ extern const GlobalSignalNumber NO_OF_SI /* 120 not unused */ /* 121 unused */ #define GSN_NODE_VERSION_REP 122 -/* 123 unused */ +#define GSN_NODE_FAIL_PROGRESS_REP 123 /* 124 unused */ #define GSN_CHECK_LCP_STOP 125 #define GSN_CLOSE_COMCONF 126 /* local */ === modified file 'storage/ndb/include/kernel/signaldata/NodeFailRep.hpp' --- storage/ndb/include/kernel/signaldata/NodeFailRep.hpp 2006-10-30 12:17:01 +0000 +++ storage/ndb/include/kernel/signaldata/NodeFailRep.hpp 2008-06-26 11:56:43 +0000 @@ -39,4 +39,14 @@ struct NodeFailRep { Uint32 theNodes[NdbNodeBitmask::Size]; }; +struct NodeFailProgressRep +{ + STATIC_CONST( SignalLength = 3 ); + + Uint32 srcRef; + Uint32 dstRef; + Uint32 dstGsn; + Uint32 theData[1]; +}; + #endif === modified file 'storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp' --- storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp 2008-04-25 06:30:39 +0000 +++ storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp 2008-06-27 07:58:41 +0000 @@ -494,6 +494,7 @@ Dbtc::execWAIT_DROP_TAB_REF(Signal* sign void Dbtc::checkWaitDropTabFailedLqh(Signal* signal, Uint32 nodeId, Uint32 tableId) { + ndbrequire(nodeId >= 2 && nodeId <= 5); TableRecordPtr tabPtr; tabPtr.i = tableId; @@ -7022,6 +7023,9 @@ void Dbtc::execNODE_FAILREP(Signal* sign const Uint32 tnoOfNodes = nodeFail->noOfNodes; const Uint32 tnewMasterId = nodeFail->masterNodeId; + printf("execNODE_FAILREP newmaster: %u oldmaster: %u: ", + tnewMasterId, cmasterNodeId); + arrGuard(tnoOfNodes, MAX_NDB_NODES); Uint32 i; int index = 0; @@ -7029,10 +7033,12 @@ void Dbtc::execNODE_FAILREP(Signal* sign { if(NdbNodeBitmask::get(nodeFail->theNodes, i)) { + printf("%u ", i); cdata[index] = i; index++; }//if }//for + printf("\n"); cmasterNodeId = tnewMasterId; @@ -7041,7 +7047,7 @@ void Dbtc::execNODE_FAILREP(Signal* sign for (i = 0; i < tnoOfNodes; i++) { jam(); - hostptr.i = cdata[i]; + Uint32 nodeId = hostptr.i = cdata[i]; ptrCheckGuard(hostptr, chostFilesize, hostRecord); /*------------------------------------------------------------*/ @@ -7074,11 +7080,13 @@ void Dbtc::execNODE_FAILREP(Signal* sign jam(); signal->theData[0] = hostptr.i; - sendSignal(cownref, GSN_TAKE_OVERTCREQ, signal, 1, JBB); - - checkScanActiveInFailedLqh(signal, 0, hostptr.i); - checkWaitDropTabFailedLqh(signal, hostptr.i, 0); // nodeid, tableid - nodeFailCheckTransactions(signal, 0, hostptr.i); + // prevent race-condition with GSN_TAKE_OVERTCCONF from other node + //sendSignal(cownref, GSN_TAKE_OVERTCREQ, signal, 1, JBB); + execTAKE_OVERTCREQ(signal); + + checkScanActiveInFailedLqh(signal, 0, nodeId); + checkWaitDropTabFailedLqh(signal, nodeId, 0); // nodeid, tableid + nodeFailCheckTransactions(signal, 0, nodeId); } }//Dbtc::execNODE_FAILREP() @@ -7100,6 +7108,7 @@ Dbtc::checkNodeFailComplete(Signal* sign Uint32 failedNodeId, Uint32 bit) { + ndbrequire(failedNodeId >= 2 && failedNodeId <= 5); hostptr.i = failedNodeId; ptrCheckGuard(hostptr, chostFilesize, hostRecord); hostptr.p->m_nf_bits &= ~bit; @@ -7245,6 +7254,32 @@ void Dbtc::execTAKE_OVERTCCONF(Signal* s hostptr.i = tfailedNodeId; ptrCheckGuard(hostptr, chostFilesize, hostRecord); + printf("execTAKE_OVERTCCONF(%u) from %u(%s)", + tfailedNodeId, + refToNode(signal->getSendersBlockRef()), + getBlockName(refToBlock(signal->getSendersBlockRef()))); + + if (refToNode(signal->getSendersBlockRef()) != getOwnNodeId()) + { + jam(); + + ndbout_c(" -> serialize with Ndbcntr"); + + Uint32 len = signal->getLength() + NodeFailProgressRep::SignalLength; + ndbrequire(len <= 25); + memmove(signal->theData + NodeFailProgressRep::SignalLength, + signal->theData, + signal->getLength()); + NodeFailProgressRep* rep = (NodeFailProgressRep*)signal->getDataPtrSend(); + rep->dstRef = reference(); + rep->dstGsn = signal->header.theVerId_signalNumber; + rep->srcRef = signal->getSendersBlockRef(); + sendSignal(NDBCNTR_REF, GSN_NODE_FAIL_PROGRESS_REP, signal, len, JBB); + return; + } + + ndbout_c(" process"); + if (signal->getSendersBlockRef() != reference()) { jam(); @@ -7262,6 +7297,12 @@ void Dbtc::execTAKE_OVERTCCONF(Signal* s break; } } + if (unlikely(i == end)) + { + jam(); + ndbout_c("IGNORE"); + return; + } ndbrequire(i != end); tcNodeFailptr.p->queueList[i] = tcNodeFailptr.p->queueList[end-1]; tcNodeFailptr.p->queueIndex = end - 1; @@ -7276,10 +7317,14 @@ void Dbtc::execTAKE_OVERTCREQ(Signal* si tfailedNodeId = signal->theData[0]; tcNodeFailptr.i = 0; ptrAss(tcNodeFailptr, tcFailRecord); + + printf("execTAKE_OVERTCREQ(%u)", tfailedNodeId); + if (tcNodeFailptr.p->failStatus != FS_IDLE || cmasterNodeId != getOwnNodeId()) { jam(); + ndbout_c(" -> queue"); /*------------------------------------------------------------*/ /* WE CAN CURRENTLY ONLY HANDLE ONE TAKE OVER AT A TIME */ /*------------------------------------------------------------*/ @@ -7292,6 +7337,8 @@ void Dbtc::execTAKE_OVERTCREQ(Signal* si tcNodeFailptr.p->queueIndex = tcNodeFailptr.p->queueIndex + 1; return; }//if + + ndbout_c(" -> startTakeOverLab"); startTakeOverLab(signal); }//Dbtc::execTAKE_OVERTCREQ() === modified file 'storage/ndb/src/kernel/blocks/ndbcntr/Ndbcntr.hpp' --- storage/ndb/src/kernel/blocks/ndbcntr/Ndbcntr.hpp 2008-02-27 13:26:46 +0000 +++ storage/ndb/src/kernel/blocks/ndbcntr/Ndbcntr.hpp 2008-06-26 12:01:17 +0000 @@ -171,6 +171,7 @@ private: void execCNTR_START_REP(Signal* signal); void execCNTR_WAITREP(Signal* signal); void execNODE_FAILREP(Signal* signal); + void execNODE_FAIL_PROGRESS_REP(Signal* signal); void execSYSTEM_ERROR(Signal* signal); // Received signals === modified file 'storage/ndb/src/kernel/blocks/ndbcntr/NdbcntrInit.cpp' --- storage/ndb/src/kernel/blocks/ndbcntr/NdbcntrInit.cpp 2006-04-05 14:05:42 +0000 +++ storage/ndb/src/kernel/blocks/ndbcntr/NdbcntrInit.cpp 2008-06-26 11:58:40 +0000 @@ -104,6 +104,8 @@ Ndbcntr::Ndbcntr(const class Configurati addRecSignal(GSN_READ_CONFIG_CONF, &Ndbcntr::execREAD_CONFIG_CONF); addRecSignal(GSN_FSREMOVECONF, &Ndbcntr::execFSREMOVECONF); + addRecSignal(GSN_NODE_FAIL_PROGRESS_REP, + &Ndbcntr::execNODE_FAIL_PROGRESS_REP); initData(); ctypeOfStart = NodeState::ST_ILLEGAL_TYPE; === modified file 'storage/ndb/src/kernel/blocks/ndbcntr/NdbcntrMain.cpp' --- storage/ndb/src/kernel/blocks/ndbcntr/NdbcntrMain.cpp 2008-02-28 14:26:38 +0000 +++ storage/ndb/src/kernel/blocks/ndbcntr/NdbcntrMain.cpp 2008-06-27 07:39:38 +0000 @@ -1400,6 +1400,30 @@ void Ndbcntr::execCNTR_WAITREP(Signal* s }//switch }//Ndbcntr::execCNTR_WAITREP() +#ifdef ERROR_INSERTED +struct q_NodeFailProgressRep +{ + Uint32 len; + Uint32 gsn; + Uint32 theData[25]; +}; + +static Uint32 q_nfpr_len = 0; +static q_NodeFailProgressRep q_nfpr[25]; +static +void +add_queue(Signal* signal) +{ + Uint32 i = q_nfpr_len; + Uint32 len = q_nfpr[i].len = signal->getLength(); + memcpy(q_nfpr[i].theData, signal->theData, 4*len); + q_nfpr[i].gsn = signal->header.theVerId_signalNumber; + q_nfpr_len++; + + ndbout_c("adding %s pos: %u", getSignalName(q_nfpr[i].gsn), i); +} +#endif + /*******************************/ /* NODE_FAILREP */ /*******************************/ @@ -1407,12 +1431,13 @@ void Ndbcntr::execNODE_FAILREP(Signal* s { jamEntry(); +#ifdef ERROR_INSERT if (ERROR_INSERTED(1001)) { - sendSignalWithDelay(reference(), GSN_NODE_FAILREP, signal, 100, - signal->getLength()); + add_queue(signal); return; } +#endif const NodeFailRep * nodeFail = (NodeFailRep *)&signal->theData[0]; NdbNodeBitmask allFailed; @@ -1585,6 +1610,46 @@ void Ndbcntr::execNODE_FAILREP(Signal* s return; }//Ndbcntr::execNODE_FAILREP() +/** + * NodeFailureRep is to all (interested) blocks on nodes from node itself + * sometimes, the impl. of NodeFailureRep will perform syncronization + * in those cases there is a race-condition that a node might receive + * a syncronization message *before* it receives the actual NodeFailureRep + * + * To handle this in a consistent manner, NODE_FAIL_PROGRESS_REP + * is introduced...this serializes the syncronization messages + * so they are quaranteed to arrive in sensible order + * + * So instead of sending syncronization message directly to destination + * one should send NODE_FAIL_PROGRESS_REP to Ndbcntr on (interested) nodes + */ +void +Ndbcntr::execNODE_FAIL_PROGRESS_REP(Signal* signal) +{ + jamEntry(); +#ifdef ERROR_INSERT + if (ERROR_INSERTED(1001)) + { + add_queue(signal); + return; + } +#endif + + NodeFailProgressRep * rep = (NodeFailProgressRep*)signal->getDataPtr(); + Uint32 srcRef = rep->srcRef; + Uint32 dstRef = rep->dstRef; + Uint32 dstGsn = rep->dstGsn; + Uint32 dstLen = signal->getLength() - NodeFailProgressRep::SignalLength; + + ndbrequire(refToNode(dstRef) == getOwnNodeId()); + + memmove(signal->theData, + signal->theData+NodeFailProgressRep::SignalLength, + dstLen); + + sendSignal(dstRef, dstGsn, signal, dstLen, JBB); +} + /*******************************/ /* READ_NODESREQ */ /*******************************/ @@ -2081,6 +2146,23 @@ Ndbcntr::execDUMP_STATE_ORD(Signal* sign #endif } +#ifdef ERROR_INSERT + if (arg == 1001) + { + ndbrequire(ERROR_INSERTED(1001)); + CLEAR_ERROR_INSERT_VALUE; + for (Uint32 i = 0; itheData, q_nfpr[i].theData, 4*len); + sendSignal(reference(), q_nfpr[i].gsn, signal, len, JBB); + ndbout_c("sending %s to self from pos: %u", + getSignalName(q_nfpr[i].gsn), i); + } + q_nfpr_len = 0; + } +#endif + }//Ndbcntr::execDUMP_STATE_ORD() void Ndbcntr::execSET_VAR_REQ(Signal* signal) { === modified file 'storage/ndb/test/ndbapi/testNodeRestart.cpp' --- storage/ndb/test/ndbapi/testNodeRestart.cpp 2008-04-25 06:33:06 +0000 +++ storage/ndb/test/ndbapi/testNodeRestart.cpp 2008-06-27 07:52:19 +0000 @@ -1078,6 +1078,9 @@ int runBug25364(NDBT_Context* ctx, NDBT_ int master = restarter.getMasterNodeId(); int victim = restarter.getRandomNodeOtherNodeGroup(master, rand()); int second = restarter.getRandomNodeSameNodeGroup(victim, rand()); + + ndbout_c("master: %u victim: %u second: %u", + master, victim, second); int dump[] = { 935, victim } ; if (restarter.dumpStateOneNode(master, dump, 2)) @@ -1085,7 +1088,7 @@ int runBug25364(NDBT_Context* ctx, NDBT_ if (restarter.dumpStateOneNode(master, val2, 2)) return NDBT_FAILED; - + if (restarter.restartOneDbNode(second, false, true, true)) return NDBT_FAILED; @@ -1582,7 +1585,8 @@ runBug28717(NDBT_Context* ctx, NDBT_Step NdbSleep_SecSleep(3); - if (res.insertErrorInNode(node1, 0)) + int val3[] = { 1001 }; + if (res.dumpStateOneNode(node1, val3, 1)) return NDBT_FAILED; if (res.waitNodesNoStart(&node0, 1))