Description:
The stress test had been running for about a day. I added in 3 new scripts with the running tests and started a backup. On the start of the backup the cluster was gone. No core files were produced.
Errors were found in the MySQLd error log:
050714 1:49:04 [Note] NDB Binlog: drop table TESTER1/t1.
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
050714 1:49:05 [Note] NDB Binlog: cluster failure for cluster_replication/t1.
050714 1:49:05 [Note] NDB Binlog: cluster failure for atae/dcacache.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/ACCOUNT_TYPES.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/SYSTEM_VALUES.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/TRANSACTION.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/ACCOUNT.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/GL.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/ACCOUNT_TYPES.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/SYSTEM_VALUES.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/TRANSACTION.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/ACCOUNT.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/GL.
050714 1:49:05 [Note] NDB Binlog: cluster failure for cluster_replication/apply_status.
All NDBD error logs reported this same error: (6 NDBD in all)
Current byte-offset of file-pointer is: 468
Date/Time: Thursday 14 July 2005 - 01:49:04
Type of error: error
Message: Internal program error (failed ndbrequire)
Fault ID: 2341
Problem data: DbdihMain.cpp
Object of reference: DBDIH (Line: 7060) 0x0000000e
ProgramName: /home/ndbdev/jmiller/builds/libexec/ndbd
ProcessID: 25886
TraceFile: /space/run/ndb_4_trace.log.1
Version 5.1.0 (a_drop5p3)
All logs and trace files have been moved to ndb09: /space/bug##### where ##### = the number of this report.
How to repeat:
Setup cluster replication between 3 computer master and 2 computer slave. Start two instances of BANK, two instances of Sabre, the 3 scripts posted below and issue a cluster backup.
inout.pl
#!/user/bin/perl -w
#/* Copyright (C) 2000-2005 MySQL AB
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# Version 1.1
####################### Includes ############################
use DBI;
####################### Globals ############################
my $m_host='';
my $m_port='';
my $m_user='';
my $m_pass='';
my $s_host='';
my $s_port='';
my $s_user='';
my $s_pass='';
my $dbhM='';
my $dbhS='';
####################### Sub Pototypes ############################
sub CollectCommandPromptInfo;
sub ConnectToDatabases;
sub DisconnectFromDatabases;
######################## Program Main ###########################
while(1)
{
CollectCommandPromptInfo;
ConnectToDatabases;
DisconnectFromDatabases;
}
###################### Collect Command Prompt Info #############
sub CollectCommandPromptInfo
{
### Check that user has supplied correct number of command line args
die "Usage:\n
inout.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n
All 4 arguments must be passed. Use BLANK for NULL passwords\n"
unless @ARGV == 4;
$m_host =$ARGV[0];
$m_port = $ARGV[1];
$m_user = $ARGV[2];
$m_pass = $ARGV[3];
if ($m_pass eq "BLANK") { $m_pass = '';}
}
###################### Collect Command Prompt Info #############
sub ConnectToDatabases
{
### Connect to both master and slave clusters databases
### Connect to master
$dbhM = DBI->connect("dbi:mysql:database=BANK;host=$m_host;port=$m_port", "$m_user", "$m_pass")
or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n";
}
sub DisconnectFromDatabases
{
### Disconnect from master
$dbhM->disconnect
or warn " Disconnection failed: $DBI::errstr\n";
}
-------------------------------------------------------------------------------------------------------
cd_db.pl
#!/user/bin/perl -w
#/* Copyright (C) 2000-2005 MySQL AB
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# Version 1.1
####################### Includes ############################
use DBI;
####################### Globals ############################
my $m_host='';
my $m_port='';
my $m_user='';
my $m_pass='';
my $dbhM='';
####################### Sub Pototypes ############################
sub CollectCommandPromptInfo;
sub ConnectToDatabases;
sub DisconnectFromDatabases;
sub CreateDB;
sub DropDB;
######################## Program Main ###########################
while(1)
{
CollectCommandPromptInfo;
ConnectToDatabases;
CreateDB;
DropDB;
DisconnectFromDatabases;
}
###################### Collect Command Prompt Info #############
sub CollectCommandPromptInfo
{
### Check that user has supplied correct number of command line args
die "Usage:\n
cd_db.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n
All 4 arguments must be passed. Use BLANK for NULL passwords\n"
unless @ARGV == 4;
$m_host =$ARGV[0];
$m_port = $ARGV[1];
$m_user = $ARGV[2];
$m_pass = $ARGV[3];
if ($m_pass eq "BLANK") { $m_pass = '';}
}
###################### Collect Command Prompt Info #############
sub ConnectToDatabases
{
### Connect to both master and slave clusters databases
### Connect to master
$dbhM = DBI->connect("dbi:mysql:database=cluster_replication;host=$m_host;port=$m_port", "$m_user", "$m_pass")
or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n";
}
sub DisconnectFromDatabases
{
### Disconnect from master
$dbhM->disconnect
or warn " Disconnection failed: $DBI::errstr\n";
}
sub CreateDB
{
$sth = $dbhM->prepare("CREATE DATABASE TESTER;")
or die "Prepare error: ", $dbhM->errstr;
$sth->execute
or die "Create Error: ", $sth->errstr;
$sth->finish;
}
sub DropDB
{
$sth = $dbhM->prepare("DROP DATABASE TESTER;")
or die "Prepare drop error: ", $dbhM->errstr;
$sth->execute
or die "drop error: ", $sth->errstr;
$sth->finish;
}
-------------------------------------------------------------------------------------------------------------------
ccidd_db_tbl.pl
#!/user/bin/perl -w
#/* Copyright (C) 2000-2005 MySQL AB
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# Version 1.1
####################### Includes ############################
use DBI;
####################### Globals ############################
my $m_host='';
my $m_port='';
my $m_user='';
my $m_pass='';
my $dbhM='';
####################### Sub Pototypes ############################
sub CollectCommandPromptInfo;
sub ConnectToDatabases;
sub DisconnectFromDatabases;
sub CreateDB_Tbl_Insert;
sub Delete_DropDB;
######################## Program Main ###########################
while(1)
{
CollectCommandPromptInfo;
ConnectToDatabases;
CreateDB_Tbl_Insert;
Delete_DropDB;
DisconnectFromDatabases;
}
###################### Collect Command Prompt Info #############
sub CollectCommandPromptInfo
{
### Check that user has supplied correct number of command line args
die "Usage:\n
cd_db.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n
All 4 arguments must be passed. Use BLANK for NULL passwords\n"
unless @ARGV == 4;
$m_host =$ARGV[0];
$m_port = $ARGV[1];
$m_user = $ARGV[2];
$m_pass = $ARGV[3];
if ($m_pass eq "BLANK") { $m_pass = '';}
}
###################### Collect Command Prompt Info #############
sub ConnectToDatabases
{
### Connect to both master and slave clusters databases
### Connect to master
$dbhM = DBI->connect("dbi:mysql:database=cluster_replication;host=$m_host;port=$m_port", "$m_user", "$m_pass")
or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n";
}
sub DisconnectFromDatabases
{
### Disconnect from master
$dbhM->disconnect
or warn " Disconnection failed: $DBI::errstr\n";
}
sub CreateDB_Tbl_Insert
{
$sth = $dbhM->prepare("CREATE DATABASE TESTER1;")
or die "Prepare create db error: ", $dbhM->errstr;
$sth->execute
or die "Create DB Error: ", $sth->errstr;
$sth->finish;
$sth = $dbhM->prepare("USE TESTER1;")
or die "Prepare error: ", $dbhM->errstr;
$sth->execute
or die "Create DB Error: ", $sth->errstr;
$sth->finish;
$sth = $dbhM->prepare("CREATE TABLE t1 (c1 int, c2 CHAR(4), PRIMARY KEY (c1))ENGINE=NDB;")
or die "Prepare create table error: ", $dbhM->errstr;
$sth->execute
or die "Create Table Error: ", $sth->errstr;
$sth->finish;
$sth = $dbhM->prepare("INSERT INTO t1 VALUES(1,'txt'),(2,'txt'),(3,'TXT');")
or die "Prepare insert error: ", $dbhM->errstr;
$sth->execute
or die "Insert Error: ", $sth->errstr;
$sth->finish;
}
sub Delete_DropDB
{
$sth = $dbhM->prepare("DELETE FROM t1")
or die "Prepare drop error: ", $dbhM->errstr;
$sth->execute
or die "Delete Error: ", $sth->errstr;
$sth->finish;
$sth = $dbhM->prepare("DROP DATABASE TESTER1;")
or die "Prepare drop error: ", $dbhM->errstr;
$sth->execute
or die "drop error: ", $sth->errstr;
$sth->finish;
}