Description:
innodb_force_recovery=6 skips redo log application and also any corrupted page recovery from the doublewrite buffer. But, instance started with this force recovery level may still write something, which will write to doublewrite - overwriting potentially not yet recovered data - further corrupting the instance.
How to repeat:
Assert that an instance does not write to doublewrite under innodb_force_recovery:
diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc
index a6b833a..28aad42 100644
--- a/storage/innobase/buf/buf0dblwr.cc
+++ b/storage/innobase/buf/buf0dblwr.cc
@@ -527,6 +527,8 @@ buf_dblwr_init_or_load_pages(
void
buf_dblwr_process(void)
{
+ ut_a(!high_level_read_only);
+
ulint page_no_dblwr = 0;
byte* read_buf;
byte* unaligned_read_buf;
@@ -730,7 +732,7 @@ buf_dblwr_update(
return;
}
- ut_ad(!srv_read_only_mode);
+ ut_ad(!high_level_read_only);
switch (flush_type) {
case BUF_FLUSH_LIST:
@@ -893,6 +895,7 @@ buf_dblwr_write_block_to_datafile(
bool sync) /*!< in: true if sync IO
is requested */
{
+ ut_a(!high_level_read_only);
ut_a(buf_page_in_file(bpage));
ulint type = IORequest::WRITE;
@@ -948,7 +951,7 @@ buf_dblwr_flush_buffered_writes(void)
return;
}
- ut_ad(!srv_read_only_mode);
+ ut_ad(!high_level_read_only);
try_again:
mutex_enter(&buf_dblwr->mutex);
@@ -1089,6 +1092,7 @@ buf_dblwr_add_to_batch(
/*====================*/
buf_page_t* bpage) /*!< in: buffer block to write */
{
+ ut_a(!high_level_read_only);
ut_a(buf_page_in_file(bpage));
try_again:
@@ -1178,6 +1182,7 @@ buf_dblwr_write_single_page(
ulint offset;
ulint i;
+ ut_a(!high_level_read_only);
ut_a(buf_page_in_file(bpage));
ut_a(srv_use_doublewrite_buf);
ut_a(buf_dblwr != NULL);
Run this MTR (non fully deterministic, it requires "skip if checkpointed" for GA):
--source include/have_innodb.inc
CREATE TABLE t(a INT PRIMARY KEY) ENGINE=InnoDB;
INSERT INTO t VALUES (1);
--let $restart_parameters=restart:--innodb-force-recovery=6
--source include/kill_and_restart_mysqld.inc
--let $restart_parameters=
--source include/restart_mysqld.inc
Check the error log:
...
2016-10-04T15:55:03.685401Z 0 [Note] InnoDB: !!! innodb_force_recovery is set to 6 !!!
...
2016-10-04T15:55:03.878928Z 0 [Note] InnoDB: Starting shutdown...
...
2016-10-04 18:55:04 0x700004f96000 InnoDB: Assertion failure in thread 123145385762816 in file buf0dblwr.cc line 954
InnoDB: Failing assertion: !high_level_read_only
Thus, the server tries to write to doublewrite, and it may overwrite an unrecovered data page there.
Suggested fix:
Do not touch doublewrite if it was not used for recovery yet.
Not sure if a) DROP TABLE is indeed supported with recover level 6; b) if it does, whether that results in the need to doublewrite-buffer the system tablespace, complicating the proposed fix.
Description: innodb_force_recovery=6 skips redo log application and also any corrupted page recovery from the doublewrite buffer. But, instance started with this force recovery level may still write something, which will write to doublewrite - overwriting potentially not yet recovered data - further corrupting the instance. How to repeat: Assert that an instance does not write to doublewrite under innodb_force_recovery: diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index a6b833a..28aad42 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -527,6 +527,8 @@ buf_dblwr_init_or_load_pages( void buf_dblwr_process(void) { + ut_a(!high_level_read_only); + ulint page_no_dblwr = 0; byte* read_buf; byte* unaligned_read_buf; @@ -730,7 +732,7 @@ buf_dblwr_update( return; } - ut_ad(!srv_read_only_mode); + ut_ad(!high_level_read_only); switch (flush_type) { case BUF_FLUSH_LIST: @@ -893,6 +895,7 @@ buf_dblwr_write_block_to_datafile( bool sync) /*!< in: true if sync IO is requested */ { + ut_a(!high_level_read_only); ut_a(buf_page_in_file(bpage)); ulint type = IORequest::WRITE; @@ -948,7 +951,7 @@ buf_dblwr_flush_buffered_writes(void) return; } - ut_ad(!srv_read_only_mode); + ut_ad(!high_level_read_only); try_again: mutex_enter(&buf_dblwr->mutex); @@ -1089,6 +1092,7 @@ buf_dblwr_add_to_batch( /*====================*/ buf_page_t* bpage) /*!< in: buffer block to write */ { + ut_a(!high_level_read_only); ut_a(buf_page_in_file(bpage)); try_again: @@ -1178,6 +1182,7 @@ buf_dblwr_write_single_page( ulint offset; ulint i; + ut_a(!high_level_read_only); ut_a(buf_page_in_file(bpage)); ut_a(srv_use_doublewrite_buf); ut_a(buf_dblwr != NULL); Run this MTR (non fully deterministic, it requires "skip if checkpointed" for GA): --source include/have_innodb.inc CREATE TABLE t(a INT PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t VALUES (1); --let $restart_parameters=restart:--innodb-force-recovery=6 --source include/kill_and_restart_mysqld.inc --let $restart_parameters= --source include/restart_mysqld.inc Check the error log: ... 2016-10-04T15:55:03.685401Z 0 [Note] InnoDB: !!! innodb_force_recovery is set to 6 !!! ... 2016-10-04T15:55:03.878928Z 0 [Note] InnoDB: Starting shutdown... ... 2016-10-04 18:55:04 0x700004f96000 InnoDB: Assertion failure in thread 123145385762816 in file buf0dblwr.cc line 954 InnoDB: Failing assertion: !high_level_read_only Thus, the server tries to write to doublewrite, and it may overwrite an unrecovered data page there. Suggested fix: Do not touch doublewrite if it was not used for recovery yet. Not sure if a) DROP TABLE is indeed supported with recover level 6; b) if it does, whether that results in the need to doublewrite-buffer the system tablespace, complicating the proposed fix.