diff --git a/mysql-test/include/mysqld_core_dump.inc b/mysql-test/include/mysqld_core_dump.inc new file mode 100644 index 0000000..f5e456d --- /dev/null +++ b/mysql-test/include/mysqld_core_dump.inc @@ -0,0 +1,103 @@ +--echo # Get the full path name of the PID file +--let $pid_file= query_get_value(SELECT @@pid_file, @@pid_file, 1) +--let PIDFILE= $pid_file + +--echo # Expecting a "crash", but don't restart the server until it is told to +--echo # Expected max core size is $expected_max_core_size MB +--let MAXCORESIZE= $expected_max_core_size + +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +perl; + +my $pid_file = $ENV{'PIDFILE'} or die "PIDFILE not set"; +my $expected_max_core_size = $ENV{'MAXCORESIZE'} or die "MAXCORESIZE not set"; + +# The argument is in MB +$expected_max_core_size = $expected_max_core_size * 1024 * 1024; + +# Get PID of mysqld +open(my $fh, '<', $pid_file) || die "Cannot open pid file $pid_file\n"; +my $pid = <$fh>; +$pid =~ s/\s//g; +close($fh); + +if ($pid eq "") { + die "Couldn't retrieve PID from PID file.\n"; +} + +# The current time in seconds since epoch +$cur_time = time; + +# Kill mysqld to dump a core +system("kill", "-s", "SIGSEGV", "$pid"); +print "# Perl: Sent a SIGSEGV to mysqld to dump a core.\n"; + +# Get the core file pattern, e.g. /var/tmp/cores/%e.%p +$core_pattern = `cat /proc/sys/kernel/core_pattern`; + +$last_slash = rindex($core_pattern, '/'); + +# The core file directory, e.g. /var/tmp/cores +$core_dir = substr($core_pattern, 0, $last_slash); + +$found_core = 0; +$core_size = 0; +$core_size_good = 0; + +# Check the files in the core file directory +$wait_sec = 60; +while ($wait_sec > 0) { + opendir(my $dir, $core_dir) or die "Failed to open dir $core_dir: $!\n"; + while (my $file = readdir($dir)) { + # If the core file name contains the PID + if (index($file, $pid) != -1) { + # The last write time in seconds since epoch + $full_path = $core_dir . '/' . $file; + @stat = stat($full_path); + $core_size = $stat[7]; + $write_secs = $stat[9]; + + # If the file was written within a minute + if ($cur_time <= $write_secs && $write_secs - $cur_time < 60) { + $found_core = 1; + if ($core_size < $expected_max_core_size) { + $core_size_good = 1; + } + # Remove the core file to avoid it get accumulated over time + unlink $full_path; + last; + } + } + } + closedir($dir); + + if ($found_core) { + last; + } + # Sleep 1 second and try again + --$wait_sec; + sleep 1; +} + +if ($found_core) { + if ($core_size_good) { + print "# Perl: OK! Found the core file and it's small!\n"; + } else { + print "# Perl: Failed! Found the core file but it's too big ($core_size)!\n"; + } +} else { + print "# Perl: Failed! Didn't find the core file!\n"; +} + +EOF + +--echo # Make server restart +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +--enable_reconnect + +--echo # Wait for server to be back online +--source include/wait_until_connected_again.inc + +--disable_reconnect diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 0707b55..bc11b22 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -5283,6 +5283,7 @@ sub mysqld_arguments ($$$) { # override defaults above. my $found_skip_core= 0; + my $innodb_dump_core_without_large_mem_buf= 0; my $found_no_console= 0; my $found_log_error= 0; foreach my $arg ( @$extra_opts ) @@ -5301,6 +5302,10 @@ sub mysqld_arguments ($$$) { { $found_skip_core= 1; } + elsif ($arg eq "--innodb-dump-core-without-large-mem-buf") + { + $innodb_dump_core_without_large_mem_buf= 1; + } elsif ($arg eq "--no-console") { $found_no_console= 1; @@ -5335,6 +5340,11 @@ sub mysqld_arguments ($$$) { mtr_add_arg($args, "%s", "--core-file"); } + # Set the default value to false so that the full core will be dumped + if ( !$innodb_dump_core_without_large_mem_buf) + { + mtr_add_arg($args, "--skip-innodb-dump-core-without-large-mem-buf"); + } return $args; } diff --git a/mysql-test/suite/innodb/r/mysqld_core_dump_without_large_mem_buf.result b/mysql-test/suite/innodb/r/mysqld_core_dump_without_large_mem_buf.result new file mode 100644 index 0000000..507f628 --- /dev/null +++ b/mysql-test/suite/innodb/r/mysqld_core_dump_without_large_mem_buf.result @@ -0,0 +1,9 @@ +# Shutdown server +# Restart server with --log-error +# Get the full path name of the PID file +# Expecting a "crash", but don't restart the server until it is told to +# Expected max core size is 3584 MB +# Perl: Sent a SIGSEGV to mysqld to dump a core. +# Perl: OK! Found the core file and it's small! +# Make server restart +# Wait for server to be back online diff --git a/mysql-test/suite/innodb/r/mysqld_core_dump_without_large_mem_buf_with_resizing.result b/mysql-test/suite/innodb/r/mysqld_core_dump_without_large_mem_buf_with_resizing.result new file mode 100644 index 0000000..5547de2 --- /dev/null +++ b/mysql-test/suite/innodb/r/mysqld_core_dump_without_large_mem_buf_with_resizing.result @@ -0,0 +1,15 @@ +# Shutdown server +# Restart server with --log-error +set global innodb_file_format=`Barracuda`; +set global innodb_file_per_table=ON; +set global innodb_adaptive_hash_index=ON; +set global innodb_buffer_pool_size = 21474836480; +set global innodb_adaptive_hash_index=OFF; +set global innodb_buffer_pool_size = 64424509440; +# Get the full path name of the PID file +# Expecting a "crash", but don't restart the server until it is told to +# Expected max core size is 5632 MB +# Perl: Sent a SIGSEGV to mysqld to dump a core. +# Perl: OK! Found the core file and it's small! +# Make server restart +# Wait for server to be back online diff --git a/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf-master.opt b/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf-master.opt new file mode 100644 index 0000000..a91b3b9 --- /dev/null +++ b/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf-master.opt @@ -0,0 +1,2 @@ +--innodb-dump-core-without-large-mem-buf +--innodb-buffer-pool-size=60G diff --git a/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf.test b/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf.test new file mode 100644 index 0000000..c4c42bc --- /dev/null +++ b/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf.test @@ -0,0 +1,25 @@ +################################################################################ +# This test is to test if mysqld can dump a core without large memory buffers. +# See opt file for the config: +# (1) --innodb-dump-core-without-large-mem-buf is set +# (2) the buffer pool is set to be large so that without dropping the large +# memory buffers the core size will be much greater than 3.5GB (the actual +# core size is less than 3GB now but set the limit to 3.5GB here in case +# the memory footprint increases in the future) + +--source include/not_valgrind.inc +--source include/have_innodb.inc + +# Embedded mode doesn't support restart +--source include/not_embedded.inc + +--echo # Shutdown server +--source include/shutdown_mysqld.inc + +--echo # Restart server with --log-error +--exec echo "restart:--log-error=$MYSQLTEST_VARDIR/log/core_dump.err" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc + +--let $expected_max_core_size = 3584 +--source include/mysqld_core_dump.inc diff --git a/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf_with_resizing-master.opt b/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf_with_resizing-master.opt new file mode 100644 index 0000000..bd2b435 --- /dev/null +++ b/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf_with_resizing-master.opt @@ -0,0 +1,4 @@ +--innodb-dump-core-without-large-mem-buf +--innodb-buffer-pool-size=60G +--innodb-buffer-pool-chunk-size=64M +--innodb-buffer-pool-resizing-timeout=60 diff --git a/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf_with_resizing.test b/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf_with_resizing.test new file mode 100644 index 0000000..fbbce9f --- /dev/null +++ b/mysql-test/suite/innodb/t/mysqld_core_dump_without_large_mem_buf_with_resizing.test @@ -0,0 +1,46 @@ +################################################################################ +# This test is to test if mysqld can dump a core without large memory buffers. +# See opt file for the config: +# (1) --innodb-dump-core-without-large-mem-buf is set +# (2) the buffer pool is set to be large initially, shrink it, then expand +# it back to the original large size, without dropping the large memory +# buffers the core size will be much greater than 5.5GB (the actual +# core size is less than 5GB now but set the limit to 5.5GB here in case +# the memory footprint increases in the future) + +--source include/not_valgrind.inc +--source include/have_innodb.inc + +# Embedded mode doesn't support restart +--source include/not_embedded.inc + +--echo # Shutdown server +--source include/shutdown_mysqld.inc + +--echo # Restart server with --log-error +--exec echo "restart:--log-error=$MYSQLTEST_VARDIR/log/core_dump_with_resizing.err" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source suite/innodb/t/innodb-buffer-pool-resize-setup.inc + +# Shrink buffer pool to 20GB +set global innodb_buffer_pool_size = 21474836480; +--source include/wait_condition.inc + +set global innodb_adaptive_hash_index=OFF; + +# Expand buffer pool back to 60GB +set global innodb_buffer_pool_size = 64424509440; +--source include/wait_condition.inc + +--disable_query_log +set global innodb_buffer_pool_size = @old_innodb_buffer_pool_size; +set global innodb_file_format = @old_innodb_file_format; +set global innodb_file_per_table = @old_innodb_file_per_table; +set global innodb_adaptive_hash_index = @old_innodb_adaptive_hash_index; +--enable_query_log +--source include/wait_condition.inc + +--let $expected_max_core_size = 5632 +--source include/mysqld_core_dump.inc diff --git a/mysql-test/suite/sys_vars/r/core_file_basic.result b/mysql-test/suite/sys_vars/r/core_file_basic.result index aa8b3e1..eef0d2c 100644 --- a/mysql-test/suite/sys_vars/r/core_file_basic.result +++ b/mysql-test/suite/sys_vars/r/core_file_basic.result @@ -1,31 +1,23 @@ -SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE '%core%'; -VARIABLE_NAME VARIABLE_VALUE -CORE_FILE ON -SHOW GLOBAL VARIABLES LIKE '%core%'; -Variable_name Value -core_file ON -SET @old_val = @@global.core_file; -SELECT @old_val; -@old_val -1 -SET GLOBAL core_file = FALSE; -SELECT @@core_file; -@@core_file -0 -SHOW GLOBAL VARIABLES LIKE '%core%'; -Variable_name Value -core_file OFF -SET GLOBAL core_file = TRUE; -SELECT @@core_file; -@@core_file +select @@global.core_file; +@@global.core_file 1 -SHOW GLOBAL VARIABLES LIKE '%core%'; +select @@session.core_file; +ERROR HY000: Variable 'core_file' is a GLOBAL variable +show global variables like 'core_file'; Variable_name Value core_file ON -SET @@global.core_file = @old_val; -SELECT @@global.core_file; -@@global.core_file -1 -SHOW GLOBAL VARIABLES LIKE '%core%'; +show session variables like 'core_file'; Variable_name Value core_file ON +select * from information_schema.global_variables where variable_name='core_file'; +VARIABLE_NAME VARIABLE_VALUE +CORE_FILE ON +select * from information_schema.session_variables where variable_name='core_file'; +VARIABLE_NAME VARIABLE_VALUE +CORE_FILE ON +set global core_file = default; +ERROR HY000: Variable 'core_file' is a read only variable +Expected error 'Read only variable' +set global core_file = true; +ERROR HY000: Variable 'core_file' is a read only variable +Expected error 'Read only variable' diff --git a/mysql-test/suite/sys_vars/r/innodb_dump_core_without_large_mem_buf_basic.result b/mysql-test/suite/sys_vars/r/innodb_dump_core_without_large_mem_buf_basic.result new file mode 100644 index 0000000..55140de --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_dump_core_without_large_mem_buf_basic.result @@ -0,0 +1,23 @@ +select @@global.innodb_dump_core_without_large_mem_buf; +@@global.innodb_dump_core_without_large_mem_buf +0 +select @@session.innodb_dump_core_without_large_mem_buf; +ERROR HY000: Variable 'innodb_dump_core_without_large_mem_buf' is a GLOBAL variable +show global variables like 'innodb_dump_core_without_large_mem_buf'; +Variable_name Value +innodb_dump_core_without_large_mem_buf OFF +show session variables like 'innodb_dump_core_without_large_mem_buf'; +Variable_name Value +innodb_dump_core_without_large_mem_buf OFF +select * from information_schema.global_variables where variable_name='innodb_dump_core_without_large_mem_buf'; +VARIABLE_NAME VARIABLE_VALUE +INNODB_DUMP_CORE_WITHOUT_LARGE_MEM_BUF OFF +select * from information_schema.session_variables where variable_name='innodb_dump_core_without_large_mem_buf'; +VARIABLE_NAME VARIABLE_VALUE +INNODB_DUMP_CORE_WITHOUT_LARGE_MEM_BUF OFF +set global innodb_dump_core_without_large_mem_buf = default; +ERROR HY000: Variable 'innodb_dump_core_without_large_mem_buf' is a read only variable +Expected error 'Read only variable' +set global innodb_dump_core_without_large_mem_buf = true; +ERROR HY000: Variable 'innodb_dump_core_without_large_mem_buf' is a read only variable +Expected error 'Read only variable' diff --git a/mysql-test/suite/sys_vars/t/core_file_basic.test b/mysql-test/suite/sys_vars/t/core_file_basic.test index 4b25111..db265a4 100644 --- a/mysql-test/suite/sys_vars/t/core_file_basic.test +++ b/mysql-test/suite/sys_vars/t/core_file_basic.test @@ -1,20 +1,20 @@ -SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE '%core%'; -SHOW GLOBAL VARIABLES LIKE '%core%'; +--source include/load_sysvars.inc -SET @old_val = @@global.core_file; -SELECT @old_val; +select @@global.core_file; -SET GLOBAL core_file = FALSE; -SELECT @@core_file; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +select @@session.core_file; -SHOW GLOBAL VARIABLES LIKE '%core%'; +show global variables like 'core_file'; +show session variables like 'core_file'; -SET GLOBAL core_file = TRUE; -SELECT @@core_file; +select * from information_schema.global_variables where variable_name='core_file'; +select * from information_schema.session_variables where variable_name='core_file'; -SHOW GLOBAL VARIABLES LIKE '%core%'; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +set global core_file = default; +--echo Expected error 'Read only variable' -SET @@global.core_file = @old_val; -SELECT @@global.core_file; - -SHOW GLOBAL VARIABLES LIKE '%core%'; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +set global core_file = true; +--echo Expected error 'Read only variable' diff --git a/mysql-test/suite/sys_vars/t/innodb_dump_core_without_large_mem_buf_basic.test b/mysql-test/suite/sys_vars/t/innodb_dump_core_without_large_mem_buf_basic.test new file mode 100644 index 0000000..da4e85e --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_dump_core_without_large_mem_buf_basic.test @@ -0,0 +1,21 @@ +--source include/have_innodb.inc +--source include/load_sysvars.inc + +select @@global.innodb_dump_core_without_large_mem_buf; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +select @@session.innodb_dump_core_without_large_mem_buf; + +show global variables like 'innodb_dump_core_without_large_mem_buf'; +show session variables like 'innodb_dump_core_without_large_mem_buf'; + +select * from information_schema.global_variables where variable_name='innodb_dump_core_without_large_mem_buf'; +select * from information_schema.session_variables where variable_name='innodb_dump_core_without_large_mem_buf'; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +set global innodb_dump_core_without_large_mem_buf = default; +--echo Expected error 'Read only variable' + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +set global innodb_dump_core_without_large_mem_buf = true; +--echo Expected error 'Read only variable' diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 8a2ec88..bbfd15b 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -796,7 +796,7 @@ static bool fix_binlog_format_after_update(sys_var *self, THD *thd, my_bool opt_core_file = FALSE; static Sys_var_mybool Sys_core_file( "core_file", "write a core-file on crashes", - GLOBAL_VAR(opt_core_file), NO_CMD_LINE, DEFAULT(FALSE)); + READ_ONLY GLOBAL_VAR(opt_core_file), NO_CMD_LINE, DEFAULT(FALSE)); static Sys_var_enum Sys_binlog_format( "binlog_format", "What form of binary logging the master will " diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 2cf5eaf..ab18665 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -142,6 +142,10 @@ static ulong innobase_read_io_threads; static ulong innobase_write_io_threads; static long innobase_buffer_pool_instances = 1; +/* Boolean config knobs that tell Linux kernel whether to dump core without +large memory buffer, e.g. InnoDB buffer pool, when core-file is enabled. */ +my_bool srv_dump_core_without_large_mem_buf = TRUE; + long long innobase_buffer_pool_size; static long long innobase_log_file_size; static unsigned long innobase_sync_pool_size; @@ -17967,6 +17971,13 @@ static MYSQL_SYSVAR_BOOL(buffer_pool_dump_at_shutdown, srv_buffer_pool_dump_at_s "Dump the buffer pool into a file named @@innodb_buffer_pool_filename", NULL, NULL, FALSE); +static MYSQL_SYSVAR_BOOL(dump_core_without_large_mem_buf, + srv_dump_core_without_large_mem_buf, + PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, + "Dump core without large memory buffer. Default value is TRUE. " + "Disable with --skip-innodb-dump-core-without-large-mem-buf.", + NULL, NULL, TRUE); + #ifdef UNIV_DEBUG static MYSQL_SYSVAR_STR(buffer_pool_evict, srv_buffer_pool_evict, PLUGIN_VAR_RQCMDARG, @@ -18640,6 +18651,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(buffer_pool_filename), MYSQL_SYSVAR(buffer_pool_dump_now), MYSQL_SYSVAR(buffer_pool_dump_at_shutdown), + MYSQL_SYSVAR(dump_core_without_large_mem_buf), MYSQL_SYSVAR(evicted_pages_sampling_ratio), MYSQL_SYSVAR(buffer_pool_resizing_timeout), MYSQL_SYSVAR(histogram_step_size_async_read), diff --git a/storage/innobase/os/os0proc.cc b/storage/innobase/os/os0proc.cc index df532bb..561ba0f 100644 --- a/storage/innobase/os/os0proc.cc +++ b/storage/innobase/os/os0proc.cc @@ -57,6 +57,13 @@ UNIV_INTERN ibool os_use_large_pages; /* Large page size. This may be a boot-time option on some platforms */ UNIV_INTERN ulint os_large_page_size; +/* If core dump is enabled */ +extern my_bool opt_core_file; + +/* Boolean config knobs that tell Linux kernel whether to dump core without +large memory buffer, e.g. InnoDB buffer pool, when core-file is enabled. */ +extern my_bool srv_dump_core_without_large_mem_buf; + /****************************************************************//** Converts the current process id to a number. It is not guaranteed that the number is unique. In Linux returns the 'process number' of the current @@ -197,6 +204,9 @@ skip: os_fast_mutex_lock(&ut_list_mutex); ut_total_allocated_memory += size; os_fast_mutex_unlock(&ut_list_mutex); + /* Dump core without large memory buffers */ + if (opt_core_file && srv_dump_core_without_large_mem_buf) + madvise(ptr, size, MADV_DONTDUMP); UNIV_MEM_ALLOC(ptr, size); } #endif @@ -277,6 +287,9 @@ os_mem_free_large( ut_a(ut_total_allocated_memory >= size); ut_total_allocated_memory -= size; os_fast_mutex_unlock(&ut_list_mutex); + /* Undo the effect of the earlier MADV_DONTDUMP */ + if (opt_core_file && srv_dump_core_without_large_mem_buf) + madvise(ptr, size, MADV_DODUMP); UNIV_MEM_FREE(ptr, size); } #endif