#!/bin/sh
#
# Copyright (c) 2006 MySQL AB
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

echo "######################################################################"
echo "#"
echo "# Test Bug#17332 - changing key_buffer_size on a running server can crash under load"
echo "# Additional tests for code coverage testing"
echo "# - Small cache for higher eviction rate"
echo "# - Use temporary tables"
echo "# - Use LOAD INDEX INTO CACHE"
echo "# - Pause with resizing to give normal operation a chance"
echo "# - Shorter test for faster results"
echo "#"

#######################################################################
#
# Settings. CAUTION: Paths are not space safe.
#

# BASEDIR="install directory"
# DATADIR="databases directory e.g. $BASEDIR/var"
DATADIR="${DATADIR:-$BASEDIR/var}"
TREEROOT="${TREEROOT:-$HOME}"

MYSQLD="$BASEDIR/libexec/mysqld"
MYSQLC="$BASEDIR/bin/mysql"
MYSQLA="$BASEDIR/bin/mysqladmin"
MYSQLT="$BASEDIR/bin/mysqltest"

PORT_1="${MYSQL_TCP_PORT:+--port=$MYSQL_TCP_PORT}"
SOCK_1="${MYSQL_UNIX_PORT:+--socket=$MYSQL_UNIX_PORT}"
USER_1="-u root -D test"
CLNT_1= #"-A -v -f"
CLNT_1="-A -v --show-warnings"
DATA_1="--basedir=$BASEDIR --datadir=$DATADIR"
SERV_1="--log-error --core"
SERV_1="$SERV_1 --skip-innodb --skip-ndbcluster"
SERV_1="$SERV_1 --delay-key-write=all"
SERV_1="$SERV_1 --key_cache_block_size=1200" # force 1024 bytes cache blocks
SERV_1="$SERV_1 --key_buffer_size=36864" # small key cache
#SERV_1="$SERV_1 --key_buffer_size=12288" # force evictions with empty LRU ring
#SERV_1="$SERV_1 --key_buffer_size=0" # disable key cache
DBUG_1= #"--debug=t:d:i:O,$DATADIR/mysqld.trace"
DBUG_1= #"--debug=d,thrlock:i:O,$DATADIR/mysqld.trace"

HOSTNAM="`expr \`uname -n\` : '\([^\.]*\)'`"
ERRFILE="$HOSTNAM.err"
PIDFILE="$HOSTNAM.pid"

#######################################################################
#
# Go to DATADIR.
#
cd $DATADIR || exit $?

#######################################################################
#
# Cleanup.
#
rm -f *.trace *.err *.pid core* test/#sql* tmp/* bug17332*
test -d test || mkdir test
test -d tmp  || mkdir tmp

#######################################################################
#
# Functions.

sleep_until_pidfile_created ()
{
  file="$1"
  loop="$2"
  org_time="$2"

  while [ "$loop" -gt 0 ]
  do
    if [ -r "$file" ]
    then
      return
    fi
    sleep 1
    loop="`expr \"$loop\" - 1`"
  done
  echo "ERROR: $file was not created in $org_time seconds;  Aborting"
  exit 1;
}

sleep_until_pidfile_deleted ()
{
  file="$1"
  loop="$2"

  while [ "$loop" -gt 0 ]
  do
    if [ -r "$file" ]
    then
      if kill -0 "`cat \"$file\"`"
      then
        sleep 1
      else
        break
      fi
    else
      break
    fi
    loop="`expr \"$loop\" - 1`"
  done
}

######################################################################
#
# Preparing scripts.
#
cat > bug17332-01.sql <<'EOF'

      DROP TABLE IF EXISTS t1;
      CREATE TABLE t1 (
          id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
          v1 CHAR(150),
          v2 CHAR(150),
          v3 CHAR(150),
          KEY(id,v1,v2,v3),
          KEY(v2,v3,v1),
          KEY(v3,v1,v2)
          ) ENGINE=MyISAM;

      DROP TABLE IF EXISTS t2;
      CREATE TABLE t2 (
          id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
          v1 CHAR(150),
          v2 CHAR(150),
          v3 CHAR(150),
          KEY(id,v1,v2,v3),
          KEY(v2,v3,v1),
          KEY(v3,v1,v2)
          ) ENGINE=MyISAM;

      DROP TABLE IF EXISTS t3;
      CREATE TABLE t3 (
          id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
          v1 CHAR(150),
          v2 CHAR(150),
          v3 CHAR(150),
          KEY(id,v1,v2,v3),
          KEY(v2,v3,v1),
          KEY(v3,v1,v2)
          ) ENGINE=MyISAM;

EOF

cat > bug17332-02.pl <<'EOF'
#!/usr/bin/perl -w

    my $table= $ARGV[0];

    #
    # Loop on insert and flush.
    #
    for (my $insert= 50; $insert; $insert--)
    {
        print "SHOW VARIABLES LIKE 'key%';\n";
        print "SHOW STATUS LIKE 'key%';\n";
        #
        # Load keycache.
        #
        print "INSERT INTO ${table} (v1,v2,v3) VALUES 
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c'),
                ('a','b','c'),('a','b','c'),('a','b','c'),('a','b','c');\n";

        if ( $insert % int(rand(3) + 1) == 1 )
        {
            print "SHOW VARIABLES LIKE 'key%';\n";
            print "SHOW STATUS LIKE 'key%';\n";
            if ( int(rand(7)) == 0 )
            {
                print "UPDATE ${table} SET v1='a', v2='b', v3='c'".
                      "WHERE v2='e';\n";
            }
            else
            {
                print "UPDATE ${table} SET v1='d', v2='e', v3='f'".
                      "WHERE id MOD 13 = ". int(rand(13)) .";\n";
            }
        }
        if ( $insert % int(rand(10) + 1) == 1 )
        {
            print "SHOW VARIABLES LIKE 'key%';\n";
            print "SHOW STATUS LIKE 'key%';\n";
            if ( int(rand(4)) == 0 )
            {
                print "FLUSH TABLE;\n";
            }
            else
            {
                print "FLUSH TABLE ${table};\n";
            }
        }
        if ( $insert % int(rand(10) + 1) == 1 )
        {
            print "SHOW VARIABLES LIKE 'key%';\n";
            print "SHOW STATUS LIKE 'key%';\n";
            print "CHECK TABLE ${table};\n";
        }
        if ( $insert % int(rand(10) + 1) == 1 )
        {
            print "SHOW VARIABLES LIKE 'key%';\n";
            print "SHOW STATUS LIKE 'key%';\n";
            print "CREATE TEMPORARY TABLE tt1".
                  "(KEY(id), KEY(v1,v2,v3)) SELECT * FROM ${table};\n";
            print "DROP TABLE tt1;\n";
        }
        if ( $insert % int(rand(10) + 1) == 1 )
        {
            print "SHOW VARIABLES LIKE 'key%';\n";
            print "SHOW STATUS LIKE 'key%';\n";
            print "LOAD INDEX INTO CACHE ${table};\n";
        }
    }

    #
    # Check table.
    #
    if ( int(rand(2))  == 0 )
    {
        print "SHOW VARIABLES LIKE 'key%';\n";
        print "SHOW STATUS LIKE 'key%';\n";
        print "FLUSH TABLE ${table};\n";
    }
    print "SHOW VARIABLES LIKE 'key%';\n";
    print "SHOW STATUS LIKE 'key%';\n";
    print "CHECK TABLE ${table};\n";
    print "SELECT 'END OF TEST';\n";
EOF

cat > bug17332-03.pl <<'EOF'
#!/usr/bin/perl -w

    #
    # Loop on resize.
    #
    for (my $idx= 100 + int(rand(4000)); $idx; $idx--)
    {
        print "SHOW VARIABLES LIKE 'key%';\n";
        print "SHOW STATUS LIKE 'key%';\n";
        # Enable this for normal resizing
        print "SET GLOBAL key_buffer_size=". (12288 + int(rand(65536))) .";\n";
        # Enable this for no resizing with small key cache
        #print "SET GLOBAL key_buffer_size=". (36864) .";\n";
        # Enable this for resizing with disabled key cache
        #print "SET GLOBAL key_buffer_size=". (2000 + int(rand(8000))) .";\n";
        if ( $idx % int(rand(10) + 1) == 1 )
        {
            print "SHOW VARIABLES LIKE 'key%';\n";
            print "SHOW STATUS LIKE 'key%';\n";
            print "SELECT SLEEP(". int(rand(3) + 1) .");\n";
        }
    }
    print "SHOW VARIABLES LIKE 'key%';\n";
    print "SHOW STATUS LIKE 'key%';\n";
    print "SELECT 'END OF TEST';\n";
EOF

#######################################################################
#
# Run in a loop until it fails.
#
ROUND=1
while :
do

  echo "######################################################################"
  date '+%Y-%m-%d %H:%M:%S'
  echo "######################################################################"
  echo "#"
  echo "# Starting database server."
  nice "$MYSQLD" $PORT_1 $SOCK_1 $DATA_1 $SERV_1 $DBUG_1 &
  SERV_1_PID=$!
  echo "Process_id $SERV_1_PID"
  echo
  sleep_until_pidfile_created "$PIDFILE" 400


  echo "######################################################################"
  echo "#"
  echo "# Creating tables."
  "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 < bug17332-01.sql > bug17332-01.log
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 1.1 (inserter)."
  perl bug17332-02.pl t1 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-11.log 2>&1 &
  CLNT_11_PID=$!
  echo "Process_id $CLNT_11_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 1.2 (inserter)."
  perl bug17332-02.pl t1 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-12.log 2>&1 &
  CLNT_12_PID=$!
  echo "Process_id $CLNT_12_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 1.3 (inserter)."
  perl bug17332-02.pl t1 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-13.log 2>&1 &
  CLNT_13_PID=$!
  echo "Process_id $CLNT_13_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 1.4 (inserter)."
  perl bug17332-02.pl t1 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-14.log 2>&1 &
  CLNT_14_PID=$!
  echo "Process_id $CLNT_14_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 2.1 (inserter)."
  perl bug17332-02.pl t2 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-21.log 2>&1 &
  CLNT_21_PID=$!
  echo "Process_id $CLNT_21_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 2.2 (inserter)."
  perl bug17332-02.pl t2 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-22.log 2>&1 &
  CLNT_22_PID=$!
  echo "Process_id $CLNT_22_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 2.3 (inserter)."
  perl bug17332-02.pl t2 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-23.log 2>&1 &
  CLNT_23_PID=$!
  echo "Process_id $CLNT_23_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 2.4 (inserter)."
  perl bug17332-02.pl t2 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-24.log 2>&1 &
  CLNT_24_PID=$!
  echo "Process_id $CLNT_24_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 3.1 (inserter)."
  perl bug17332-02.pl t3 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-31.log 2>&1 &
  CLNT_31_PID=$!
  echo "Process_id $CLNT_31_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 3.2 (inserter)."
  perl bug17332-02.pl t3 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-32.log 2>&1 &
  CLNT_32_PID=$!
  echo "Process_id $CLNT_32_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 3.3 (inserter)."
  perl bug17332-02.pl t3 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-33.log 2>&1 &
  CLNT_33_PID=$!
  echo "Process_id $CLNT_33_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 3.4 (inserter)."
  perl bug17332-02.pl t3 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 \
          > bug17332-34.log 2>&1 &
  CLNT_34_PID=$!
  echo "Process_id $CLNT_34_PID"
  echo
  sleep 2


  echo "######################################################################"
  echo "#"
  echo "# Starting client 4.1 (key buffer changer)."
  perl bug17332-03.pl 2>/dev/null | \
      "$MYSQLC" $PORT_1 $SOCK_1 $USER_1 $CLNT_1 -vvv \
          > bug17332-41.log 2>&1 &
  CLNT_41_PID=$!
  echo "Process_id $CLNT_41_PID"
  echo
  sleep 2

  #
  # There is no use for more buffer changers as only one can be started 
  # from set_var.cc (see key_cache->in_init).
  #


  echo "######################################################################"
  echo "#"
  echo "`date '+%Y-%m-%d %H:%M:%S'` Waiting for completion of round $ROUND..."
  exec 6>&2 2>/dev/null
  # Loop until some of the processes finished
  REPORTED=""
  FINISHED=0
  while [ $FINISHED -lt 13 ]
  do
      sleep 10
      for PID in $CLNT_11_PID $CLNT_12_PID $CLNT_13_PID $CLNT_14_PID \
                 $CLNT_21_PID $CLNT_22_PID $CLNT_23_PID $CLNT_24_PID \
                 $CLNT_31_PID $CLNT_32_PID $CLNT_33_PID $CLNT_34_PID \
                 $CLNT_41_PID
      do
          if kill -0 $PID
          then
              :
          else
              case $REPORTED in
                  *" $PID "* )
                      :
                      ;;
                  * )
                      FINISHED=`expr $FINISHED + 1`
                      echo "`date '+%Y-%m-%d %H:%M:%S'` Process $PID finished"
                      REPORTED="$REPORTED $PID "
                      ;;
              esac
          fi
      done
  done
  # Kill all processes
  kill $CLNT_11_PID $CLNT_12_PID $CLNT_13_PID $CLNT_14_PID \
       $CLNT_21_PID $CLNT_22_PID $CLNT_23_PID $CLNT_24_PID \
       $CLNT_31_PID $CLNT_32_PID $CLNT_33_PID $CLNT_34_PID \
       $CLNT_41_PID || :
  sleep 2
  # Loop as long as any of the processes exists
  while kill $CLNT_11_PID || \
        kill $CLNT_12_PID || \
        kill $CLNT_13_PID || \
        kill $CLNT_14_PID || \
        kill $CLNT_21_PID || \
        kill $CLNT_22_PID || \
        kill $CLNT_23_PID || \
        kill $CLNT_24_PID || \
        kill $CLNT_31_PID || \
        kill $CLNT_32_PID || \
        kill $CLNT_33_PID || \
        kill $CLNT_34_PID || \
        kill $CLNT_41_PID
  do
      sleep 1
  done
  sleep 2
  echo
  exec 2>&6


  echo "######################################################################"
  echo "#"
  echo "# Stopping database server."
  "$MYSQLA" $PORT_1 $SOCK_1 -u root shutdown
  sleep_until_pidfile_deleted "$PIDFILE" 60


  echo
  if [ "`echo core*`" != "core*" ]
  then
      break
  fi
  if grep -Hi "corrupt|crash|incorrect key file" "$ERRFILE" \
     || egrep -Hi "corrupt|crash|incorrect key file" bug17332-*.log
  then
      break
  fi
  if grep -H "ERROR" "$ERRFILE" \
     || egrep -Hi "error" bug17332-*.log
  then
      : # break
  fi

  echo
  echo === $ROUND rounds complete `date '+%Y-%m-%d %H:%M:%S'`
  echo
  echo
  ROUND=`expr $ROUND + 1`
  echo

done


echo "# End of Test."
echo "#"
echo "######################################################################"