Description:
The stress test had been running for about a day. I added in 3 new scripts with the running tests and started a backup. On the start of the backup the cluster was gone. No core files were produced.
Errors were found in the MySQLd error log:
050714 1:49:04 [Note] NDB Binlog: drop table TESTER1/t1.
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
out of order bucket: 2 gci: 41774 m_latestGCI: 41770
050714 1:49:05 [Note] NDB Binlog: cluster failure for cluster_replication/t1.
050714 1:49:05 [Note] NDB Binlog: cluster failure for atae/dcacache.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/ACCOUNT_TYPES.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/SYSTEM_VALUES.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/TRANSACTION.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/ACCOUNT.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/GL.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/ACCOUNT_TYPES.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/SYSTEM_VALUES.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/TRANSACTION.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/ACCOUNT.
050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/GL.
050714 1:49:05 [Note] NDB Binlog: cluster failure for cluster_replication/apply_status.
All NDBD error logs reported this same error: (6 NDBD in all)
Current byte-offset of file-pointer is: 468
Date/Time: Thursday 14 July 2005 - 01:49:04
Type of error: error
Message: Internal program error (failed ndbrequire)
Fault ID: 2341
Problem data: DbdihMain.cpp
Object of reference: DBDIH (Line: 7060) 0x0000000e
ProgramName: /home/ndbdev/jmiller/builds/libexec/ndbd
ProcessID: 25886
TraceFile: /space/run/ndb_4_trace.log.1
Version 5.1.0 (a_drop5p3)
All logs and trace files have been moved to ndb09: /space/bug##### where ##### = the number of this report.
How to repeat:
Setup cluster replication between 3 computer master and 2 computer slave. Start two instances of BANK, two instances of Sabre, the 3 scripts posted below and issue a cluster backup.
inout.pl
#!/user/bin/perl -w
#/* Copyright (C) 2000-2005 MySQL AB
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# Version 1.1
####################### Includes ############################
use DBI;
####################### Globals ############################
my $m_host='';
my $m_port='';
my $m_user='';
my $m_pass='';
my $s_host='';
my $s_port='';
my $s_user='';
my $s_pass='';
my $dbhM='';
my $dbhS='';
####################### Sub Pototypes ############################
sub CollectCommandPromptInfo;
sub ConnectToDatabases;
sub DisconnectFromDatabases;
######################## Program Main ###########################
while(1)
{
CollectCommandPromptInfo;
ConnectToDatabases;
DisconnectFromDatabases;
}
###################### Collect Command Prompt Info #############
sub CollectCommandPromptInfo
{
### Check that user has supplied correct number of command line args
die "Usage:\n
inout.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n
All 4 arguments must be passed. Use BLANK for NULL passwords\n"
unless @ARGV == 4;
$m_host =$ARGV[0];
$m_port = $ARGV[1];
$m_user = $ARGV[2];
$m_pass = $ARGV[3];
if ($m_pass eq "BLANK") { $m_pass = '';}
}
###################### Collect Command Prompt Info #############
sub ConnectToDatabases
{
### Connect to both master and slave clusters databases
### Connect to master
$dbhM = DBI->connect("dbi:mysql:database=BANK;host=$m_host;port=$m_port", "$m_user", "$m_pass")
or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n";
}
sub DisconnectFromDatabases
{
### Disconnect from master
$dbhM->disconnect
or warn " Disconnection failed: $DBI::errstr\n";
}
-------------------------------------------------------------------------------------------------------
cd_db.pl
#!/user/bin/perl -w
#/* Copyright (C) 2000-2005 MySQL AB
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# Version 1.1
####################### Includes ############################
use DBI;
####################### Globals ############################
my $m_host='';
my $m_port='';
my $m_user='';
my $m_pass='';
my $dbhM='';
####################### Sub Pototypes ############################
sub CollectCommandPromptInfo;
sub ConnectToDatabases;
sub DisconnectFromDatabases;
sub CreateDB;
sub DropDB;
######################## Program Main ###########################
while(1)
{
CollectCommandPromptInfo;
ConnectToDatabases;
CreateDB;
DropDB;
DisconnectFromDatabases;
}
###################### Collect Command Prompt Info #############
sub CollectCommandPromptInfo
{
### Check that user has supplied correct number of command line args
die "Usage:\n
cd_db.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n
All 4 arguments must be passed. Use BLANK for NULL passwords\n"
unless @ARGV == 4;
$m_host =$ARGV[0];
$m_port = $ARGV[1];
$m_user = $ARGV[2];
$m_pass = $ARGV[3];
if ($m_pass eq "BLANK") { $m_pass = '';}
}
###################### Collect Command Prompt Info #############
sub ConnectToDatabases
{
### Connect to both master and slave clusters databases
### Connect to master
$dbhM = DBI->connect("dbi:mysql:database=cluster_replication;host=$m_host;port=$m_port", "$m_user", "$m_pass")
or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n";
}
sub DisconnectFromDatabases
{
### Disconnect from master
$dbhM->disconnect
or warn " Disconnection failed: $DBI::errstr\n";
}
sub CreateDB
{
$sth = $dbhM->prepare("CREATE DATABASE TESTER;")
or die "Prepare error: ", $dbhM->errstr;
$sth->execute
or die "Create Error: ", $sth->errstr;
$sth->finish;
}
sub DropDB
{
$sth = $dbhM->prepare("DROP DATABASE TESTER;")
or die "Prepare drop error: ", $dbhM->errstr;
$sth->execute
or die "drop error: ", $sth->errstr;
$sth->finish;
}
-------------------------------------------------------------------------------------------------------------------
ccidd_db_tbl.pl
#!/user/bin/perl -w
#/* Copyright (C) 2000-2005 MySQL AB
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# Version 1.1
####################### Includes ############################
use DBI;
####################### Globals ############################
my $m_host='';
my $m_port='';
my $m_user='';
my $m_pass='';
my $dbhM='';
####################### Sub Pototypes ############################
sub CollectCommandPromptInfo;
sub ConnectToDatabases;
sub DisconnectFromDatabases;
sub CreateDB_Tbl_Insert;
sub Delete_DropDB;
######################## Program Main ###########################
while(1)
{
CollectCommandPromptInfo;
ConnectToDatabases;
CreateDB_Tbl_Insert;
Delete_DropDB;
DisconnectFromDatabases;
}
###################### Collect Command Prompt Info #############
sub CollectCommandPromptInfo
{
### Check that user has supplied correct number of command line args
die "Usage:\n
cd_db.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n
All 4 arguments must be passed. Use BLANK for NULL passwords\n"
unless @ARGV == 4;
$m_host =$ARGV[0];
$m_port = $ARGV[1];
$m_user = $ARGV[2];
$m_pass = $ARGV[3];
if ($m_pass eq "BLANK") { $m_pass = '';}
}
###################### Collect Command Prompt Info #############
sub ConnectToDatabases
{
### Connect to both master and slave clusters databases
### Connect to master
$dbhM = DBI->connect("dbi:mysql:database=cluster_replication;host=$m_host;port=$m_port", "$m_user", "$m_pass")
or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n";
}
sub DisconnectFromDatabases
{
### Disconnect from master
$dbhM->disconnect
or warn " Disconnection failed: $DBI::errstr\n";
}
sub CreateDB_Tbl_Insert
{
$sth = $dbhM->prepare("CREATE DATABASE TESTER1;")
or die "Prepare create db error: ", $dbhM->errstr;
$sth->execute
or die "Create DB Error: ", $sth->errstr;
$sth->finish;
$sth = $dbhM->prepare("USE TESTER1;")
or die "Prepare error: ", $dbhM->errstr;
$sth->execute
or die "Create DB Error: ", $sth->errstr;
$sth->finish;
$sth = $dbhM->prepare("CREATE TABLE t1 (c1 int, c2 CHAR(4), PRIMARY KEY (c1))ENGINE=NDB;")
or die "Prepare create table error: ", $dbhM->errstr;
$sth->execute
or die "Create Table Error: ", $sth->errstr;
$sth->finish;
$sth = $dbhM->prepare("INSERT INTO t1 VALUES(1,'txt'),(2,'txt'),(3,'TXT');")
or die "Prepare insert error: ", $dbhM->errstr;
$sth->execute
or die "Insert Error: ", $sth->errstr;
$sth->finish;
}
sub Delete_DropDB
{
$sth = $dbhM->prepare("DELETE FROM t1")
or die "Prepare drop error: ", $dbhM->errstr;
$sth->execute
or die "Delete Error: ", $sth->errstr;
$sth->finish;
$sth = $dbhM->prepare("DROP DATABASE TESTER1;")
or die "Prepare drop error: ", $dbhM->errstr;
$sth->execute
or die "drop error: ", $sth->errstr;
$sth->finish;
}
Description: The stress test had been running for about a day. I added in 3 new scripts with the running tests and started a backup. On the start of the backup the cluster was gone. No core files were produced. Errors were found in the MySQLd error log: 050714 1:49:04 [Note] NDB Binlog: drop table TESTER1/t1. out of order bucket: 2 gci: 41774 m_latestGCI: 41770 out of order bucket: 2 gci: 41774 m_latestGCI: 41770 out of order bucket: 2 gci: 41774 m_latestGCI: 41770 out of order bucket: 2 gci: 41774 m_latestGCI: 41770 out of order bucket: 2 gci: 41774 m_latestGCI: 41770 050714 1:49:05 [Note] NDB Binlog: cluster failure for cluster_replication/t1. 050714 1:49:05 [Note] NDB Binlog: cluster failure for atae/dcacache. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/ACCOUNT_TYPES. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/SYSTEM_VALUES. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/TRANSACTION. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/ACCOUNT. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK2/GL. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/ACCOUNT_TYPES. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/SYSTEM_VALUES. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/TRANSACTION. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/ACCOUNT. 050714 1:49:05 [Note] NDB Binlog: cluster failure for BANK/GL. 050714 1:49:05 [Note] NDB Binlog: cluster failure for cluster_replication/apply_status. All NDBD error logs reported this same error: (6 NDBD in all) Current byte-offset of file-pointer is: 468 Date/Time: Thursday 14 July 2005 - 01:49:04 Type of error: error Message: Internal program error (failed ndbrequire) Fault ID: 2341 Problem data: DbdihMain.cpp Object of reference: DBDIH (Line: 7060) 0x0000000e ProgramName: /home/ndbdev/jmiller/builds/libexec/ndbd ProcessID: 25886 TraceFile: /space/run/ndb_4_trace.log.1 Version 5.1.0 (a_drop5p3) All logs and trace files have been moved to ndb09: /space/bug##### where ##### = the number of this report. How to repeat: Setup cluster replication between 3 computer master and 2 computer slave. Start two instances of BANK, two instances of Sabre, the 3 scripts posted below and issue a cluster backup. inout.pl #!/user/bin/perl -w #/* Copyright (C) 2000-2005 MySQL AB # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Version 1.1 ####################### Includes ############################ use DBI; ####################### Globals ############################ my $m_host=''; my $m_port=''; my $m_user=''; my $m_pass=''; my $s_host=''; my $s_port=''; my $s_user=''; my $s_pass=''; my $dbhM=''; my $dbhS=''; ####################### Sub Pototypes ############################ sub CollectCommandPromptInfo; sub ConnectToDatabases; sub DisconnectFromDatabases; ######################## Program Main ########################### while(1) { CollectCommandPromptInfo; ConnectToDatabases; DisconnectFromDatabases; } ###################### Collect Command Prompt Info ############# sub CollectCommandPromptInfo { ### Check that user has supplied correct number of command line args die "Usage:\n inout.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n All 4 arguments must be passed. Use BLANK for NULL passwords\n" unless @ARGV == 4; $m_host =$ARGV[0]; $m_port = $ARGV[1]; $m_user = $ARGV[2]; $m_pass = $ARGV[3]; if ($m_pass eq "BLANK") { $m_pass = '';} } ###################### Collect Command Prompt Info ############# sub ConnectToDatabases { ### Connect to both master and slave clusters databases ### Connect to master $dbhM = DBI->connect("dbi:mysql:database=BANK;host=$m_host;port=$m_port", "$m_user", "$m_pass") or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n"; } sub DisconnectFromDatabases { ### Disconnect from master $dbhM->disconnect or warn " Disconnection failed: $DBI::errstr\n"; } ------------------------------------------------------------------------------------------------------- cd_db.pl #!/user/bin/perl -w #/* Copyright (C) 2000-2005 MySQL AB # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Version 1.1 ####################### Includes ############################ use DBI; ####################### Globals ############################ my $m_host=''; my $m_port=''; my $m_user=''; my $m_pass=''; my $dbhM=''; ####################### Sub Pototypes ############################ sub CollectCommandPromptInfo; sub ConnectToDatabases; sub DisconnectFromDatabases; sub CreateDB; sub DropDB; ######################## Program Main ########################### while(1) { CollectCommandPromptInfo; ConnectToDatabases; CreateDB; DropDB; DisconnectFromDatabases; } ###################### Collect Command Prompt Info ############# sub CollectCommandPromptInfo { ### Check that user has supplied correct number of command line args die "Usage:\n cd_db.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n All 4 arguments must be passed. Use BLANK for NULL passwords\n" unless @ARGV == 4; $m_host =$ARGV[0]; $m_port = $ARGV[1]; $m_user = $ARGV[2]; $m_pass = $ARGV[3]; if ($m_pass eq "BLANK") { $m_pass = '';} } ###################### Collect Command Prompt Info ############# sub ConnectToDatabases { ### Connect to both master and slave clusters databases ### Connect to master $dbhM = DBI->connect("dbi:mysql:database=cluster_replication;host=$m_host;port=$m_port", "$m_user", "$m_pass") or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n"; } sub DisconnectFromDatabases { ### Disconnect from master $dbhM->disconnect or warn " Disconnection failed: $DBI::errstr\n"; } sub CreateDB { $sth = $dbhM->prepare("CREATE DATABASE TESTER;") or die "Prepare error: ", $dbhM->errstr; $sth->execute or die "Create Error: ", $sth->errstr; $sth->finish; } sub DropDB { $sth = $dbhM->prepare("DROP DATABASE TESTER;") or die "Prepare drop error: ", $dbhM->errstr; $sth->execute or die "drop error: ", $sth->errstr; $sth->finish; } ------------------------------------------------------------------------------------------------------------------- ccidd_db_tbl.pl #!/user/bin/perl -w #/* Copyright (C) 2000-2005 MySQL AB # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Version 1.1 ####################### Includes ############################ use DBI; ####################### Globals ############################ my $m_host=''; my $m_port=''; my $m_user=''; my $m_pass=''; my $dbhM=''; ####################### Sub Pototypes ############################ sub CollectCommandPromptInfo; sub ConnectToDatabases; sub DisconnectFromDatabases; sub CreateDB_Tbl_Insert; sub Delete_DropDB; ######################## Program Main ########################### while(1) { CollectCommandPromptInfo; ConnectToDatabases; CreateDB_Tbl_Insert; Delete_DropDB; DisconnectFromDatabases; } ###################### Collect Command Prompt Info ############# sub CollectCommandPromptInfo { ### Check that user has supplied correct number of command line args die "Usage:\n cd_db.pl <master MySQL host> <master MySQL port> <master user> < master pass>\n All 4 arguments must be passed. Use BLANK for NULL passwords\n" unless @ARGV == 4; $m_host =$ARGV[0]; $m_port = $ARGV[1]; $m_user = $ARGV[2]; $m_pass = $ARGV[3]; if ($m_pass eq "BLANK") { $m_pass = '';} } ###################### Collect Command Prompt Info ############# sub ConnectToDatabases { ### Connect to both master and slave clusters databases ### Connect to master $dbhM = DBI->connect("dbi:mysql:database=cluster_replication;host=$m_host;port=$m_port", "$m_user", "$m_pass") or die "Can't connect to Master Cluster MySQL process! Error: $DBI::errstr\n"; } sub DisconnectFromDatabases { ### Disconnect from master $dbhM->disconnect or warn " Disconnection failed: $DBI::errstr\n"; } sub CreateDB_Tbl_Insert { $sth = $dbhM->prepare("CREATE DATABASE TESTER1;") or die "Prepare create db error: ", $dbhM->errstr; $sth->execute or die "Create DB Error: ", $sth->errstr; $sth->finish; $sth = $dbhM->prepare("USE TESTER1;") or die "Prepare error: ", $dbhM->errstr; $sth->execute or die "Create DB Error: ", $sth->errstr; $sth->finish; $sth = $dbhM->prepare("CREATE TABLE t1 (c1 int, c2 CHAR(4), PRIMARY KEY (c1))ENGINE=NDB;") or die "Prepare create table error: ", $dbhM->errstr; $sth->execute or die "Create Table Error: ", $sth->errstr; $sth->finish; $sth = $dbhM->prepare("INSERT INTO t1 VALUES(1,'txt'),(2,'txt'),(3,'TXT');") or die "Prepare insert error: ", $dbhM->errstr; $sth->execute or die "Insert Error: ", $sth->errstr; $sth->finish; } sub Delete_DropDB { $sth = $dbhM->prepare("DELETE FROM t1") or die "Prepare drop error: ", $dbhM->errstr; $sth->execute or die "Delete Error: ", $sth->errstr; $sth->finish; $sth = $dbhM->prepare("DROP DATABASE TESTER1;") or die "Prepare drop error: ", $dbhM->errstr; $sth->execute or die "drop error: ", $sth->errstr; $sth->finish; }