############## rm -rf data cmake . -DDOWNLOAD_BOOST=1 -DWITH_BOOST=./boost make -j32 ## my.cnf (set plugin_dir if required) [mysqld] early-plugin-load=keyring_file.so keyring_file_data=/export/umesh/server/source/bugs/mysql-5.7.14/mysql-keyring/mykeyring ./sql/mysqld --initialize-insecure --basedir=/export/umesh/server/source/bugs/mysql-5.7.14 --datadir=/export/umesh/server/source/bugs/mysql-5.7.14/data ./sql/mysqld --defaults-file=./my.cnf --basedir=/export/umesh/server/source/bugs/mysql-5.7.14 --datadir=/export/umesh/server/source/bugs/mysql-5.7.14/data --core-file --socket=/tmp/mysql.sock --port=3306 --log-error=/export/umesh/server/source/bugs/mysql-5.7.14/data/log.err 2 -- CLI session [umshastr@hod03]/export/umesh/server/source/bugs/mysql-5.7.14: client/mysql -uroot -p Enter password: Welcome to the MySQL monitor. Commands end with ; or \g. Your MySQL connection id is 2 Server version: 5.7.14 Source distribution Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective owners. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. mysql> drop database test; Query OK, 1 row affected (0.00 sec) mysql> create database test; Query OK, 1 row affected (0.00 sec) -- -- provision data using sysbench ( no need to use 'run' to trigger this issue) [umshastr@hod03]~/bugs/sysbench: sysbench/sysbench --test=sysbench/tests/db/oltp.lua --oltp-table-size=1000000 --mysql-table-engine=innodb --mysql-db=test --mysql-user=root --mysql-socket=/tmp/mysql.sock prepare sysbench 0.5: multi-threaded system evaluation benchmark Creating table 'sbtest1'... Inserting 1000000 records into 'sbtest1' -- mysql> use test Database changed mysql> alter table sbtest1 ENCRYPTION='Y'; Query OK, 1000000 rows affected (15.86 sec) ### run python script from the bug report and move on to ALTER... [umshastr@hod03]/export/umesh/server/binaries/2.1.3: cat bug.py import mysql.connector cnx = mysql.connector.connect(user='root', password='', host='localhost', database='test', port=3306) cursor = cnx.cursor() alter_stmt = "ALTER INSTANCE ROTATE INNODB MASTER KEY" while(True): print "Master key rotated" cursor.execute(alter_stmt) cursor.close() cnx.close() [umshastr@hod03]/export/umesh/server/binaries/2.1.3: PYTHONPATH="./lib/python2.7/site-packages/" python bug.py . . Master key rotated Master key rotated Master key rotated Traceback (most recent call last): File "bug.py", line 13, in cursor.execute(alter_stmt) File "/export/umesh/server/binaries/2.1.3/lib/python2.7/site-packages/mysql/connector/cursor.py", line 515, in execute self._handle_result(self._connection.cmd_query(stmt)) File "/export/umesh/server/binaries/2.1.3/lib/python2.7/site-packages/mysql/connector/connection.py", line 488, in cmd_query result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query)) File "/export/umesh/server/binaries/2.1.3/lib/python2.7/site-packages/mysql/connector/connection.py", line 267, in _send_cmd return self._socket.recv() File "/export/umesh/server/binaries/2.1.3/lib/python2.7/site-packages/mysql/connector/network.py", line 228, in recv_plain raise errors.InterfaceError(errno=2013) mysql.connector.errors.InterfaceError: 2013: Lost connection to MySQL server during query thon bug.py -- CLI mysql> alter table sbtest1 discard tablespace; Query OK, 0 rows affected (0.09 sec) mysql> drop table sbtest1; Query OK, 0 rows affected (0.00 sec) mysql> \q Bye [umshastr@hod03]/export/umesh/server/source/bugs/mysql-5.7.14: ps aux|grep mysqld umshastr 8285 70.6 0.1 1896900 324436 pts/4 Sl 10:11 1:28 ./sql/mysqld --defaults-file=./my.cnf --basedir=/export/umesh/server/source/bugs/mysql-5.7.14 --datadir=/export/umesh/server/source/bugs/mysql-5.7.14/data --core-file --socket=/tmp/mysql.sock --port=3306 --log-error=/export/umesh/server/source/bugs/mysql-5.7.14/data/log.err umshastr 8334 0.0 0.0 112632 968 pts/4 S+ 10:13 0:00 grep --color=auto mysqld [umshastr@hod03]/export/umesh/server/source/bugs/mysql-5.7.14: kill -9 8285 [umshastr@hod03]/export/umesh/server/source/bugs/mysql-5.7.14: ./sql/mysqld --defaults-file=./my.cnf --basedir=/export/umesh/server/source/bugs/mysql-5.7.14 --datadir=/export/umesh/server/source/bugs/mysql-5.7.14/data --core-file --socket=/tmp/mysql.sock --port=3306 --log-error=/export/umesh/server/source/bugs/mysql-5.7.14/data/log.err 2>&1 & [2] 8335 [1] Killed ./sql/mysqld --defaults-file=./my.cnf --basedir=/export/umesh/server/source/bugs/mysql-5.7.14 --datadir=/export/umesh/server/source/bugs/mysql-5.7.14/data --core-file --socket=/tmp/mysql.sock --port=3306 --log-error=/export/umesh/server/source/bugs/mysql-5.7.14/data/log.err 2>&1 [umshastr@hod03]/export/umesh/server/source/bugs/mysql-5.7.14: [2]+ Aborted (core dumped) ./sql/mysqld --defaults-file=./my.cnf --basedir=/export/umesh/server/source/bugs/mysql-5.7.14 --datadir=/export/umesh/server/source/bugs/mysql-5.7.14/data --core-file --socket=/tmp/mysql.sock --port=3306 --log-error=/export/umesh/server/source/bugs/mysql-5.7.14/data/log.err 2>&1 -- bt -- extract from error log 2016-08-02T08:13:26.961095Z 0 [Note] InnoDB: PUNCH HOLE support available 2016-08-02T08:13:26.961133Z 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins 2016-08-02T08:13:26.961141Z 0 [Note] InnoDB: Uses event mutexes 2016-08-02T08:13:26.961145Z 0 [Note] InnoDB: GCC builtin __atomic_thread_fence() is used for memory barrier 2016-08-02T08:13:26.961149Z 0 [Note] InnoDB: Compressed tables use zlib 1.2.3 2016-08-02T08:13:26.961153Z 0 [Note] InnoDB: Using Linux native AIO 2016-08-02T08:13:26.961397Z 0 [Note] InnoDB: Number of pools: 1 2016-08-02T08:13:26.961503Z 0 [Note] InnoDB: Using CPU crc32 instructions 2016-08-02T08:13:26.962779Z 0 [Note] InnoDB: Initializing buffer pool, total size = 128M, instances = 1, chunk size = 128M 2016-08-02T08:13:26.972280Z 0 [Note] InnoDB: Completed initialization of buffer pool 2016-08-02T08:13:26.974924Z 0 [Note] InnoDB: If the mysqld execution user is authorized, page cleaner thread priority can be changed. See the man page of setpriority(). 2016-08-02T08:13:26.987461Z 0 [Note] InnoDB: Highest supported file format is Barracuda. 2016-08-02T08:13:26.988175Z 0 [Note] InnoDB: Log scan progressed past the checkpoint lsn 1647230745 2016-08-02T08:13:26.988190Z 0 [Note] InnoDB: Doing recovery: scanned up to log sequence number 1647235252 2016-08-02T08:13:26.989026Z 0 [Note] InnoDB: Doing recovery: scanned up to log sequence number 1647235252 2016-08-02T08:13:26.989055Z 0 [ERROR] [FATAL] InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE for redo log record 30 (page 35:0) at 1647230745. 2016-08-02 10:13:26 0x7f1cfcfe7740 InnoDB: Assertion failure in thread 139762480346944 in file ut0ut.cc line 916 InnoDB: We intentionally generate a memory trap. InnoDB: Submit a detailed bug report to http://bugs.mysql.com. InnoDB: If you get repeated assertion failures or crashes, even InnoDB: immediately after the mysqld startup, there may be InnoDB: corruption in the InnoDB tablespace. Please refer to InnoDB: http://dev.mysql.com/doc/refman/5.7/en/forcing-innodb-recovery.html InnoDB: about forcing recovery. 08:13:26 UTC - mysqld got signal 6 ; This could be because you hit a bug. It is also possible that this binary or one of the libraries it was linked against is corrupt, improperly built, or misconfigured. This error can also be caused by malfunctioning hardware. Attempting to collect some information that could help diagnose the problem. As this is a crash and something is definitely wrong, the information collection process might fail. key_buffer_size=8388608 read_buffer_size=131072 max_used_connections=0 max_threads=151 thread_count=0 connection_count=0 It is possible that mysqld could use up to key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 68190 K bytes of memory Hope that's ok; if not, decrease some variables in the equation. Thread pointer: 0x0 Attempting backtrace. You can use the following information to find out where mysqld died. If you see no messages after this, something went terribly wrong... stack_bottom = 0 thread_stack 0x40000 ./sql/mysqld(my_print_stacktrace+0x2c)[0xeac4dc] ./sql/mysqld(handle_fatal_signal+0x451)[0x7a3e21] /lib64/libpthread.so.0(+0xf130)[0x7f1cfcbe1130] /lib64/libc.so.6(gsignal+0x37)[0x7f1cfb5d85d7] /lib64/libc.so.6(abort+0x148)[0x7f1cfb5d9cc8] ./sql/mysqld[0x774808] ./sql/mysqld(_ZN2ib5fatalD1Ev+0x4d)[0x107558d] ./sql/mysqld[0xf4fe3b] ./sql/mysqld[0xf54aad] ./sql/mysqld[0xf55141] ./sql/mysqld(_Z35recv_recovery_from_checkpoint_startm+0x14f6)[0xf56716] ./sql/mysqld(_Z34innobase_start_or_create_for_mysqlv+0x3e4a)[0x102392a] ./sql/mysqld[0xef806c] ./sql/mysqld(_Z24ha_initialize_handlertonP13st_plugin_int+0x51)[0x7ecb91] ./sql/mysqld[0xcada76] ./sql/mysqld(_Z11plugin_initPiPPci+0x5f0)[0xcb4590] ./sql/mysqld[0x79cfad] ./sql/mysqld(_Z11mysqld_mainiPPc+0x821)[0x79e471] /lib64/libc.so.6(__libc_start_main+0xf5)[0x7f1cfb5c4af5] ./sql/mysqld[0x79479d] The manual page at http://dev.mysql.com/doc/mysql/en/crashing.html contains information that should help you find out what is causing the crash. Writing a core file [umshastr@hod03]/export/umesh/server/source/bugs/mysql-5.7.14: gdb ./sql/mysqld data/core.8335 GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-64.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /export/umesh/server/source/bugs/mysql-5.7.14/sql/mysqld...done. [New LWP 8335] [New LWP 8338] [New LWP 8341] [New LWP 8340] [New LWP 8343] [New LWP 8345] [New LWP 8336] [New LWP 8339] [New LWP 8337] [New LWP 8342] [New LWP 8344] [New LWP 8346] [New LWP 8347] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `./sql/mysqld --defaults-file=./my.cnf --basedir=/export/umesh/server/source/bug'. Program terminated with signal 6, Aborted. #0 0x00007f1cfcbde771 in pthread_kill () from /lib64/libpthread.so.0 Missing separate debuginfos, use: debuginfo-install glibc-2.17-78.0.1.el7.x86_64 libaio-0.3.109-12.el7.x86_64 libgcc-4.8.3-9.el7.x86_64 libstdc++-4.8.3-9.el7.x86_64 nss-softokn-freebl-3.16.2.3-13.el7_1.x86_64 numactl-libs-2.0.9-5.el7_1.x86_64 (gdb) bt #0 0x00007f1cfcbde771 in pthread_kill () from /lib64/libpthread.so.0 #1 0x00000000007a3d95 in handle_fatal_signal (sig=6) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/signal_handler.cc:220 #2 #3 0x00007f1cfb5d85d7 in raise () from /lib64/libc.so.6 #4 0x00007f1cfb5d9cc8 in abort () from /lib64/libc.so.6 #5 0x0000000000774808 in ut_dbg_assertion_failed (expr=expr@entry=0x0, file=file@entry=0x15df740 "/export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/ut/ut0ut.cc", line=line@entry=916) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/ut/ut0dbg.cc:67 #6 0x000000000107558d in ib::fatal::~fatal (this=0x7ffd2c6b05c0, __in_chrg=) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/ut/ut0ut.cc:916 #7 0x0000000000f4fe3b in recv_parse_or_apply_log_rec_body (type=MLOG_WRITE_STRING, ptr=ptr@entry=0x4621dbb "(\226", end_ptr=end_ptr@entry=0x4622ec3 "", space_id=35, page_no=0, apply=apply@entry=true, block=block@entry=0x0, mtr=mtr@entry=0x0) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/log/log0recv.cc:1804 #8 0x0000000000f54aad in recv_parse_log_rec (body=, apply=true, page_no=0x7ffd2c6b07d0, space=0x7ffd2c6b07c0, end_ptr=0x4622ec3 "", ptr=0x4621db8 "\236#", type=0x7ffd2c6b07b0) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/log/log0recv.cc:3053 #9 recv_parse_log_recs (apply=true, store=STORE_YES, checkpoint_lsn=1647230745) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/log/log0recv.cc:3214 #10 recv_scan_log_recs (available_memory=available_memory@entry=125812736, store_to_hash=store_to_hash@entry=0x7ffd2c6b09a0, buf=, len=, checkpoint_lsn=checkpoint_lsn@entry=1647230745, start_lsn=start_lsn@entry=1647230464, contiguous_lsn=contiguous_lsn@entry=0x7ffd2c6b0a40, group_scanned_lsn=group_scanned_lsn@entry=0x4b71b78) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/log/log0recv.cc:3755 #11 0x0000000000f55141 in recv_group_scan_log_recs (group=group@entry=0x4b71b28, contiguous_lsn=contiguous_lsn@entry=0x7ffd2c6b0a40, last_phase=last_phase@entry=false) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/log/log0recv.cc:3846 #12 0x0000000000f56716 in recv_recovery_from_checkpoint_start (flush_lsn=2524971) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/log/log0recv.cc:4157 #13 0x000000000102392a in innobase_start_or_create_for_mysql () at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/srv/srv0start.cc:2181 #14 0x0000000000ef806c in innobase_init (p=) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/handler/ha_innodb.cc:3984 #15 0x00000000007ecb91 in ha_initialize_handlerton (plugin=0x3bcc728) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/handler.cc:837 #16 0x0000000000cada76 in plugin_initialize (plugin=plugin@entry=0x3bcc728) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_plugin.cc:1193 #17 0x0000000000cb4590 in plugin_init (argc=argc@entry=0x1e0d6e0 , argv=0x33bab08, flags=flags@entry=0) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_plugin.cc:1535 #18 0x000000000079cfad in init_server_components () at /export/umesh/server/source/bugs/mysql-5.7.14/sql/mysqld.cc:3998 #19 0x000000000079e471 in mysqld_main (argc=11, argv=0x33bab08) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/mysqld.cc:4632 #20 0x00007f1cfb5c4af5 in __libc_start_main () from /lib64/libc.so.6 #21 0x000000000079479d in _start () (gdb) -- with sysbench run then stacke traces are diff [umshastr@hod03]~/bugs/sysbench: sysbench/sysbench --test=sysbench/tests/db/oltp.lua --oltp-table-size=1000000 --mysql-table-engine=innodb --mysql-db=test --mysql-user=root --mysql-socket=/tmp/mysql.sock --max-requests=0 run WARNING: Both max-requests and max-time are 0, running endless test sysbench 0.5: multi-threaded system evaluation benchmark Running the test with following options: Number of threads: 1 Random number generator seed is 0 and will be ignored Threads started! ALERT: failed to execute MySQL query: `SELECT c FROM sbtest1 WHERE id=497647`: ALERT: Error 2013 Lost connection to MySQL server during query FATAL: failed to execute function `event': (null) 2016-08-02 10:02:14 0x7f87640b0700 InnoDB: Assertion failure in thread 140219475756800 in file ha_innodb.cc line 20710 InnoDB: Failing assertion: format != 0 (gdb) bt #0 0x00007f878159b771 in pthread_kill () from /lib64/libpthread.so.0 #1 0x00000000007a3d95 in handle_fatal_signal (sig=6) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/signal_handler.cc:220 #2 #3 0x00007f877ff955d7 in raise () from /lib64/libc.so.6 #4 0x00007f877ff96cc8 in abort () from /lib64/libc.so.6 #5 0x0000000000774808 in ut_dbg_assertion_failed (expr=expr@entry=0x15b8dbd "format != 0", file=file@entry=0x15ba698 "/export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/handler/ha_innodb.cc", line=line@entry=20710) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/ut/ut0dbg.cc:67 #6 0x0000000000ee0c4d in ib_senderrf (thd=0x7f87200120b0, level=IB_LOG_LEVEL_ERROR, code=code@entry=1814, level=IB_LOG_LEVEL_ERROR) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/handler/ha_innodb.cc:20710 #7 0x0000000000ef1ffa in ha_innobase::index_read (this=0x7f8720018280, buf=, key_ptr=, key_len=, find_flag=) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/innobase/handler/ha_innodb.cc:8667 #8 0x00000000007e87d8 in handler::index_read_idx_map (this=0x7f8720018280, buf=0x7f8720018690 "\377=\245\a", index=, key=0x7f8720002890 "\033'\b", keypart_map=1, find_flag=HA_READ_KEY_EXACT) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/handler.cc:7417 #9 0x00000000007ef314 in handler::ha_index_read_idx_map (this=0x7f8720018280, buf=0x7f8720018690 "\377=\245\a", index=0, key=0x7f8720002890 "\033'\b", keypart_map=1, find_flag=find_flag@entry=HA_READ_KEY_EXACT) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/handler.cc:3050 #10 0x0000000000c5f42a in read_const (ref=0x7f8720009cf0, table=0x7f8720015640) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_executor.cc:2007 #11 join_read_const_table (tab=tab@entry=0x7f87200027d8, pos=0x7f8720009d98) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_executor.cc:1892 #12 0x0000000000c7cde3 in JOIN::extract_func_dependent_tables (this=this@entry=0x7f8720002380) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_optimizer.cc:5531 #13 0x0000000000c846b2 in JOIN::make_join_plan (this=this@entry=0x7f8720002380) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_optimizer.cc:4995 #14 0x0000000000c84d3c in JOIN::optimize (this=0x7f8720002380) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_optimizer.cc:368 #15 0x0000000000cc8935 in st_select_lex::optimize (this=this@entry=0x7f8720000940, thd=thd@entry=0x7f87200120b0) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_select.cc:1009 #16 0x0000000000cc8b15 in handle_query (thd=thd@entry=0x7f87200120b0, lex=lex@entry=0x7f87200141f8, result=result@entry=0x7f8720002260, added_options=added_options@entry=0, removed_options=removed_options@entry=0) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_select.cc:164 #17 0x00000000007663ae in execute_sqlcom_select (thd=thd@entry=0x7f87200120b0, all_tables=) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_parse.cc:5143 #18 0x0000000000c8eb8c in mysql_execute_command (thd=thd@entry=0x7f87200120b0, first_level=first_level@entry=true) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_parse.cc:2756 #19 0x0000000000c910cd in mysql_parse (thd=thd@entry=0x7f87200120b0, parser_state=parser_state@entry=0x7f87640af7a0) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_parse.cc:5559 #20 0x0000000000c91a28 in dispatch_command (thd=thd@entry=0x7f87200120b0, com_data=com_data@entry=0x7f87640afe00, command=COM_QUERY) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_parse.cc:1427 #21 0x0000000000c93347 in do_command (thd=thd@entry=0x7f87200120b0) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/sql_parse.cc:995 #22 0x0000000000d4c030 in handle_connection (arg=arg@entry=0x2a13be0) at /export/umesh/server/source/bugs/mysql-5.7.14/sql/conn_handler/connection_handler_per_thread.cc:300 #23 0x0000000001217011 in pfs_spawn_thread (arg=0x2aa76e0) at /export/umesh/server/source/bugs/mysql-5.7.14/storage/perfschema/pfs.cc:2188 #24 0x00007f8781596df5 in start_thread () from /lib64/libpthread.so.0 #25 0x00007f878005660d in clone () from /lib64/libc.so.6 (gdb)