Summary: This fixes http://bugs.mysql.com/62100. The problem is that fil_rename_tablespace can timeout and return an error that is ignored. The ignored error is not reported to the user. The timeout occurs when there are pending writes for the renamed tablespace but the writer threads are not started. They are not started because the doublewrite buffer is not full and any attempt to request more writes and make it full are blocked by the stop_ios check in fil_mutex_enter_and_prepare_for_io. The stop_ios check is boosted before the check for too-many open file descriptors to make it deterministic. I modified fil0fil.c and then ran a reproduction case to confirm this reproduces and then fixes the problem. It is not possible to write an mtr test to reproduce this without such a hack as the race is hard to reproduce otherwise. --- .../suite/innodb_plugin/r/innodb_bug62100.result | 23 +++++++++++ .../innodb_plugin/r/innodb_bug62100_big.result | 30 +++++++++++++++ .../innodb_plugin/t/innodb_bug62100-master.opt | 1 + .../suite/innodb_plugin/t/innodb_bug62100.test | 32 ++++++++++++++++ .../innodb_plugin/t/innodb_bug62100_big-master.opt | 1 + .../suite/innodb_plugin/t/innodb_bug62100_big.test | 40 ++++++++++++++++++++ storage/innodb_plugin/fil/fil0fil.c | 39 ++++++++++++++++--- storage/innodb_plugin/row/row0mysql.c | 1 + 8 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/innodb_plugin/r/innodb_bug62100.result create mode 100644 mysql-test/suite/innodb_plugin/r/innodb_bug62100_big.result create mode 100644 mysql-test/suite/innodb_plugin/t/innodb_bug62100-master.opt create mode 100644 mysql-test/suite/innodb_plugin/t/innodb_bug62100.test create mode 100644 mysql-test/suite/innodb_plugin/t/innodb_bug62100_big-master.opt create mode 100644 mysql-test/suite/innodb_plugin/t/innodb_bug62100_big.test diff --git a/mysql-test/suite/innodb_plugin/r/innodb_bug62100.result b/mysql-test/suite/innodb_plugin/r/innodb_bug62100.result new file mode 100644 index 0000000..c420f43 --- /dev/null +++ b/mysql-test/suite/innodb_plugin/r/innodb_bug62100.result @@ -0,0 +1,23 @@ +drop table if exists t; +create table t (i int primary key auto_increment, c char(250), d char(250), e char(250)) engine=innodb; +create index cx on t(c); +create index dx on t(d); +create index ex on t(e); +insert into t values (NULL, 'a', 'b', 'c'); +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +select sum(i) from t; +sum(i) +214612 +rename table t to s; +select sum(i) from s; +sum(i) +214612 +drop table s; diff --git a/mysql-test/suite/innodb_plugin/r/innodb_bug62100_big.result b/mysql-test/suite/innodb_plugin/r/innodb_bug62100_big.result new file mode 100644 index 0000000..1e370f2 --- /dev/null +++ b/mysql-test/suite/innodb_plugin/r/innodb_bug62100_big.result @@ -0,0 +1,30 @@ +drop table if exists t; +create table t (i int primary key auto_increment, c char(250), d char(250), e char(250)) engine=innodb; +create index cx on t(c); +create index dx on t(d); +create index ex on t(e); +insert into t values (NULL, 'a', 'b', 'c'); +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +select sum(i) from t; +sum(i) +3578189140 +rename table t to s; +select sum(i) from s; +sum(i) +3578189140 +drop table s; diff --git a/mysql-test/suite/innodb_plugin/t/innodb_bug62100-master.opt b/mysql-test/suite/innodb_plugin/t/innodb_bug62100-master.opt new file mode 100644 index 0000000..1416d6a --- /dev/null +++ b/mysql-test/suite/innodb_plugin/t/innodb_bug62100-master.opt @@ -0,0 +1 @@ +--innodb_file_format=Barracuda --innodb_file_per_table=1 diff --git a/mysql-test/suite/innodb_plugin/t/innodb_bug62100.test b/mysql-test/suite/innodb_plugin/t/innodb_bug62100.test new file mode 100644 index 0000000..1d8aaae --- /dev/null +++ b/mysql-test/suite/innodb_plugin/t/innodb_bug62100.test @@ -0,0 +1,32 @@ +# Test for bug 62100. This is unlikely to reproduce the problem but is a +# sanity test to make sure nothing is obviously broken. + +--source include/have_innodb_plugin.inc + +--disable_warnings +drop table if exists t; +--enable_warnings + +create table t (i int primary key auto_increment, c char(250), d char(250), e char(250)) engine=innodb; +create index cx on t(c); +create index dx on t(d); +create index ex on t(e); + +insert into t values (NULL, 'a', 'b', 'c'); +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; + +select sum(i) from t; + +rename table t to s; + +select sum(i) from s; + +drop table s; diff --git a/mysql-test/suite/innodb_plugin/t/innodb_bug62100_big-master.opt b/mysql-test/suite/innodb_plugin/t/innodb_bug62100_big-master.opt new file mode 100644 index 0000000..1416d6a --- /dev/null +++ b/mysql-test/suite/innodb_plugin/t/innodb_bug62100_big-master.opt @@ -0,0 +1 @@ +--innodb_file_format=Barracuda --innodb_file_per_table=1 diff --git a/mysql-test/suite/innodb_plugin/t/innodb_bug62100_big.test b/mysql-test/suite/innodb_plugin/t/innodb_bug62100_big.test new file mode 100644 index 0000000..c152081 --- /dev/null +++ b/mysql-test/suite/innodb_plugin/t/innodb_bug62100_big.test @@ -0,0 +1,40 @@ +# Test for bug 62100. This is unlikely to reproduce the problem but is a +# sanity test to make sure nothing is obviously broken. + +--source include/have_innodb_plugin.inc +--source include/big_test.inc + +--disable_warnings +drop table if exists t; +--enable_warnings + +create table t (i int primary key auto_increment, c char(250), d char(250), e char(250)) engine=innodb; +create index cx on t(c); +create index dx on t(d); +create index ex on t(e); + +insert into t values (NULL, 'a', 'b', 'c'); +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; +insert into t select NULL, 'a', 'b', 'c' from t; + +select sum(i) from t; + +rename table t to s; + +select sum(i) from s; + +drop table s; diff --git a/storage/innodb_plugin/fil/fil0fil.c b/storage/innodb_plugin/fil/fil0fil.c index af83b45..539eb3f 100644 --- a/storage/innodb_plugin/fil/fil0fil.c +++ b/storage/innodb_plugin/fil/fil0fil.c @@ -954,11 +954,6 @@ retry: return; } - if (fil_system->n_open < fil_system->max_n_open) { - - return; - } - space = fil_space_get_by_id(space_id); if (space != NULL && space->stop_ios) { @@ -982,6 +977,11 @@ retry: goto retry; } + if (fil_system->n_open < fil_system->max_n_open) { + + return; + } + /* If the file is already open, no need to do anything; if the space does not exist, we handle the situation in the function which called this function */ @@ -2649,6 +2649,12 @@ fil_rename_tablespace( char* path; ibool old_name_was_specified = TRUE; char* old_path; + int pending_retry = 0; + int mod_retry = 0; + ulint last_pending = 0; + ulint last_pending_flushes = 0; + ulint last_modification = 0; + ulint last_flush_counter = 0; ut_a(id != 0); @@ -2665,7 +2671,13 @@ retry: ut_print_filename(stderr, old_name); fputs(" to ", stderr); ut_print_filename(stderr, new_name); - fprintf(stderr, ", %lu iterations\n", (ulong) count); + fprintf(stderr, ", %lu iterations, " + "retries: %d pending, %d modification, " + "%lu n_pending, %lu n_pending_flushes, " + "%lu modification_counter, %lu flush_counter\n", + count, pending_retry, mod_retry, + last_pending, last_pending_flushes, + last_modification, last_flush_counter); } mutex_enter(&fil_system->mutex); @@ -2700,14 +2712,28 @@ retry: ut_a(UT_LIST_GET_LEN(space->chain) == 1); node = UT_LIST_GET_FIRST(space->chain); + last_pending = node->n_pending; + last_pending_flushes = node->n_pending_flushes; + last_modification = node->modification_counter; + last_flush_counter = node->flush_counter; + if (node->n_pending > 0 || node->n_pending_flushes > 0) { /* There are pending i/o's or flushes, sleep for a while and retry */ mutex_exit(&fil_system->mutex); + /* For http://bugs.mysql.com/62100, if the doublewrite buffer + is not full then the writer threads might be sleeping and + things that would add writes to make it full can be blocked + by the stop_ios check in fil_mutex_enter_and_prepare_for_io. + So this call will wake them and make n_pending go to 0. */ + + os_aio_simulated_wake_handler_threads(); + os_thread_sleep(20000); + ++pending_retry; goto retry; } else if (node->modification_counter > node->flush_counter) { @@ -2719,6 +2745,7 @@ retry: fil_flush(id, FLUSH_FROM_OTHER); + ++mod_retry; goto retry; } else if (node->open) { diff --git a/storage/innodb_plugin/row/row0mysql.c b/storage/innodb_plugin/row/row0mysql.c index 82c21f4..160c14b 100644 --- a/storage/innodb_plugin/row/row0mysql.c +++ b/storage/innodb_plugin/row/row0mysql.c @@ -4042,6 +4042,7 @@ end: trx->error_state = DB_SUCCESS; trx_general_rollback_for_mysql(trx, NULL); trx->error_state = DB_SUCCESS; + err = DB_ERROR; goto funct_exit; } -- 1.7.4