Description:
On Gentoo Linux server I cannot start mysql-debug-4.0.18 binaries:
0x80732df handle_segfault + 423
0x82c3688 pthread_sighandler + 184
(nil)
0x809324f open_table__FP3THDPCcN21Pb + 1507
0x8094346 open_tables__FP3THDP13st_table_list + 126
0x80bc206 acl_init__FP3THDb + 590
0x80745f3 main + 2639
0x82c70b4 __libc_start_main + 212
0x8048101 _start + 33
* sys-libs/glibc
Latest version available: 2.3.2-r9
Latest version installed: 2.3.2-r9
Size of downloaded files: 14,946 kB
Homepage: http://www.gnu.org/software/libc/libc.html
Description: GNU libc6 (also called glibc2) C library
License: LGPL-2
How to repeat:
When I have limited number of files using ulimit(1) to 1024, I get:
# cd /usr/local
# /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld --defaults-file=/etc/mysql/my.cnf
040306 23:32:34 Warning: setrlimit couldn't increase number of open files to more than 1024 (request: 1006633070)
/usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld: Out of memory (Needed 4026531968 bytes)
#
# cat /proc/sys/fs/file-max
104774
# cat /proc/sys/fs/file-nr
1350 220 104774
#
When I use ulimit(1) to adjust number of maxfiles to system maximum - 104774, I get:
# /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld --defaults-file=/etc/mysql/my.cnf
040306 23:35:24 Warning: setrlimit couldn't increase number of open files to more than 104774 (request: 1006633070)
/usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld: Out of memory (Needed 4026531968 bytes)
#
/etc/mysql/my.cnf contains:
[client]
port = 3306
socket = /var/run/mysqld/mysqld.sock
[safe_mysqld]
err-log = /var/log/mysql/mysql.err
max_allowed_packet=128M
[mysqld]
skip-innodb
user = mysql
pid-file = /var/run/mysqld/mysqld.pid
socket = /var/run/mysqld/mysqld.sock
log-error = /var/log/mysql/mysqld.err
datadir = /var/lib/mysql
tmpdir = /tmp
skip-locking
set-variable = max_user_connections=50
set-variable = key_buffer=128M
set-variable = max_allowed_packet=128M
set-variable = table_cache=480M
set-variable = query_cache_size=32M
set-variable = thread_stack=128K
set-variable = interactive_timeout=60000
set-variable = wait_timeout=60000
set-variable = connect_timeout=60000
set-variable = net_read_timeout=60000
set-variable = open_files_limit=30000
bind-address = 127.0.0.1
port = 3306
[mysqldump]
quick
set-variable = max_allowed_packet=1M
[mysql]
max_allowed_packet = 128M
[isamchk]
set-variable = key_buffer=16M
# strace /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld --defaults-file=/etc/mysql/my.cnf
execve("/usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld", ["/usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld", "--defaults-file=/etc/mysql/my.cnf"], [/* 50 vars */]) = 0
fcntl64(0, F_GETFD) = 0
fcntl64(1, F_GETFD) = 0
fcntl64(2, F_GETFD) = 0
geteuid32() = 0
getuid32() = 0
getegid32() = 0
getgid32() = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
setrlimit(RLIMIT_STACK, {rlim_cur=124*1024, rlim_max=RLIM_INFINITY}) = 0
<cut>
read(3, "# /etc/mysql/my.cnf: The global "..., 4096) = 3164
<cut>
open("/usr/local/mysql-debug-4.0.18-pc-linux-i686/share/mysql/charsets/Index", O_RDONLY|O_LARGEFILE) = 3
<cut>
getrlimit(RLIMIT_NOFILE, {rlim_cur=104774, rlim_max=104774}) = 0
setrlimit(RLIMIT_NOFILE, {rlim_cur=1006633070, rlim_max=1006633070}) = -1 EPERM (Operation not permitted)
time(NULL) = 1078612743
write(2, "040306 23:39:03 ", 17040306 23:39:03 ) = 17
write(2, "Warning: setrlimit couldn\'t incr"..., 99Warning: setrlimit couldn't increase number of open files to more than 104774 (request: 1006633070)) = 99
<cut>
--- SIGRTMIN (Unknown signal 32) @ 0 (0) ---
<... rt_sigsuspend resumed> ) = -1 EINTR (Interrupted system call)
sigreturn() = ? (mask now [HUP INT QUIT PIPE TERM TSTP RTMIN])
rt_sigprocmask(SIG_SETMASK, NULL, [HUP INT QUIT PIPE TERM TSTP RTMIN], 8) = 0
rt_sigsuspend([HUP INT QUIT PIPE TERM TSTP] <unfinished ...>
--- SIGRTMIN (Unknown signal 32) @ 0 (0) ---
<... rt_sigsuspend resumed> ) = -1 EINTR (Interrupted system call)
sigreturn() = ? (mask now [HUP INT QUIT PIPE TERM TSTP RTMIN])
open("./mysql/host.frm", O_RDONLY|O_LARGEFILE) = 7
read(7, "\376\1\7\t\3\0\0\20\1\0\0000\0\0\347\0i\0\0\0\0\0\0\0\0"..., 64) = 64
_llseek(7, 64, [64], SEEK_SET) = 0
read(7, "//\0\0 \0\0", 7) = 7
_llseek(7, 4096, [4096], SEEK_SET) = 0
read(7, "\1\2\34\0\n\0\2\\\0\2\1\200\2\0\0\0\0<\0\2\200>\0\0\0\0"..., 39) = 39
pread(7, "\377 "..., 105, 4327) = 105
_llseek(7, 8192, [8192], SEEK_SET) = 0
read(7, "h\3\0\20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 288) = 288
read(7, "\0\1\17\16\2\24) "..., 584) = 584
close(7) = 0
getcwd("/var/lib/mysql", 4095) = 15
lstat64("/var/lib/mysql/mysql", {st_mode=S_IFDIR|0755, st_size=536, ...}) = 0
lstat64("/var/lib/mysql/mysql/host.MYI", {st_mode=S_IFREG|0660, st_size=1024, ...}) = 0
open("/var/lib/mysql/mysql/host.MYI", O_RDWR|O_LARGEFILE) = 7
read(7, "\376\376\7\1\0\2\1\305\0\260\0d\0\310\0\2\0\0\1\0\10\1"..., 24) = 24
readlink("./mysql/host.MYI", 0xbfffe2ac, 511) = -1 EINVAL (Invalid argument)
readlink("./mysql/host.MYD", 0xbfffe0ac, 511) = -1 EINVAL (Invalid argument)
_llseek(7, 0, [0], SEEK_SET) = 0
read(7, "\376\376\7\1\0\2\1\305\0\260\0d\0\310\0\2\0\0\1\0\10\1"..., 453) = 453
open("./mysql/host.MYD", O_RDWR|O_LARGEFILE) = 8
getpid() = 3291
brk(0x849a000) = 0x849a000
brk(0x849d000) = 0x849d000
--- SIGSEGV (Segmentation fault) @ 0 (0) ---
write(2, "mysqld got signal 11;\nThis could"..., 1693) = 1693
write(6, "\200\223C\10\2\0\0\0\1\0\0\0x\331D\10\1\0\0\0H\353\377"..., 148) = 148
rt_sigprocmask(SIG_SETMASK, NULL, [HUP INT QUIT PIPE TERM TSTP RTMIN], 8) = 0
rt_sigsuspend([HUP INT QUIT PIPE TERM TSTP] <unfinished ...>
--- SIGRTMIN (Unknown signal 32) @ 0 (0) ---
<... rt_sigsuspend resumed> ) = -1 EINTR (Interrupted system call)
sigreturn() = ? (mask now [HUP INT QUIT PIPE TERM TSTP RTMIN])
wait4(3292, NULL, __WCLONE, NULL) = 3292
munmap(0x41fff000, 4096) = 0
_exit(1) = ?
#
(gdb)
2161 set_if_bigger(wanted_files, open_files_limit);
(gdb)
2163 uint files=set_maximum_open_files(wanted_files);
(gdb)
040306 23:58:00 Warning: setrlimit couldn't increase number of open files to more than 104774 (request: 1006633070)
2164 if (files && files < wanted_files && ! open_files_limit)
(gdb)
<cut>
(gdb)
2204 server_init();
(gdb)
2205 table_cache_init();
(gdb)
/usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld: Out of memory (Needed 4026531968 bytes)
2206 hostname_cache_init();
(gdb)
<cut>
(gdb)
safe_cond_wait (cond=0x844f26c, mp=0x844ef60, file=0x83544ce "mysqld.cc", line=1702) at thr_mutex.c:191
191 thr_mutex.c: No such file or directory.
in thr_mutex.c
(gdb)
192 in thr_mutex.c
(gdb)
198 in thr_mutex.c
(gdb)
206 in thr_mutex.c
(gdb)
207 in thr_mutex.c
(gdb)
206 in thr_mutex.c
(gdb)
207 in thr_mutex.c
(gdb)
208 in thr_mutex.c
(gdb)
209 in thr_mutex.c
(gdb)
210 in thr_mutex.c
(gdb)
start_signal_handler () at mysqld.cc:1703
1703 pthread_mutex_unlock(&LOCK_thread_count);
Current language: auto; currently c++
(gdb)
1705 (void) pthread_attr_destroy(&thr_attr);
(gdb)
1706 DBUG_VOID_RETURN;
(gdb)
main (argc=22, argv=0x8486060) at mysqld.cc:2293
2293 if (acl_init((THD*) 0, opt_noacl))
(gdb)
Program received signal SIGSEGV, Segmentation fault.
0x00000000 in ?? ()
(gdb) bt full
#0 0x00000000 in ?? ()
No symbol table info available.
(gdb) where
#0 0x00000000 in ?? ()
(gdb) bt full
#0 0x00000000 in ?? ()
No symbol table info available.
(gdb) l
2288 if (unix_sock != INVALID_SOCKET)
2289 unlink(mysql_unix_port);
2290 unireg_abort(1);
2291 }
2292 start_signal_handler(); // Creates pidfile
2293 if (acl_init((THD*) 0, opt_noacl))
2294 {
2295 abort_loop=1;
2296 select_thread_in_use=0;
2297 #ifndef __NETWARE__
(gdb)
From another gdb session:
(gdb) c
Continuing.
Breakpoint 1, acl_init (org_thd=0x0, dont_read_acl_tables=false) at sql_acl.cc:142
142 DBUG_ENTER("acl_init");
Current language: auto; currently c++
(gdb) l
137 TABLE_LIST tables[3];
138 TABLE *table;
139 READ_RECORD read_record_info;
140 MYSQL_LOCK *lock;
141 my_bool return_val=1;
142 DBUG_ENTER("acl_init");
143
144 if (!acl_cache)
145 acl_cache=new hash_filo(ACL_CACHE_SIZE,0,0,
146 (hash_get_key) acl_entry_get_key,
(gdb) bt full
#0 acl_init (org_thd=0x0, dont_read_acl_tables=false) at sql_acl.cc:142
thd = (THD *) 0xbffff36c
tables = {{next = 0x0, db = 0x6a6 <Address 0x6a6 out of bounds>, alias = 0x0, real_name = 0x844ef78 "",
on_expr = 0x0, natural_join = 0xbffff42c, use_index = 0x82c1f16, ignore_index = 0x844ef70, table = 0x0, grant = {
grant_table = 0xbffff43c, version = 137109270, privilege = 138735456, want_privilege = 0}, lock_type = TL_UNLOCK,
outer_join = 138736236, shared = 138735456, db_length = 0, real_name_length = 3221222492, straight = 139,
updating = 128, force_index = 42}, {next = 0x844ef60, db = 0x1000 <Address 0x1000 out of bounds>,
alias = 0xbffff46c "Źô˙ż°ô˙ż¸ô˙ż`ďD\bÎD5\bŚ\006",
real_name = 0x82a823a "1Ŕ\215eč[^_\211ě]Ă\211öU\211ĺ\203ě\fWVS\213]\f\213}\030\203ÄôSčĐ\230\001", on_expr = 0xbffff4b8,
natural_join = 0x0, use_index = 0xbffff4a8, ignore_index = 0x82a8225, table = 0xbffff4b8, grant = {grant_table = 0x0,
version = 3221222620, privilege = 134690663, want_privilege = 1706}, lock_type = -1073744728,
outer_join = 3221222572, shared = 3221222576, db_length = 3221222584, real_name_length = 138735456, straight = 206,
updating = 68, force_index = 53}, {next = 0x6a6, db = 0x844e9c0 "/var/log/mysql/mysqld.err",
alias = 0x19946 <Address 0x19946 out of bounds>, real_name = 0x2 <Address 0x2 out of bounds>, on_expr = 0x8289ae0,
natural_join = 0x83e5dd8, use_index = 0x0, ignore_index = 0xbffff4cc, table = 0xbffff4ac, grant = {
grant_table = 0xbffff4b0, version = 104774, privilege = 2, want_privilege = 135076739}, lock_type = 138734016,
outer_join = 1, shared = 0, db_length = 0, real_name_length = 1, straight = false, updating = false,
force_index = false}}
table = (TABLE *) 0x0
read_record_info = {table = 0x844ef78, file = 0x844f26c, forms = 0x844ef78,
read_record = 0x82df961 <__libc_malloc+209>, thd = 0x8439380, select = 0x844ef78, cache_records = 138735480,
ref_length = 3220700160, struct_length = 3220700160, reclength = 0, rec_cache_size = 3221222380,
error_offset = 137109156, index = 138646400, ref_pos = 0x0, record = 0xbffff3ec "Ś\006", cache = 0x82c1b6e "1Ŕé\002\001",
cache_pos = 0x844ef60 "", cache_end = 0x0, read_positions = 0x6a7 <Address 0x6a7 out of bounds>, io_cache = 0xbffff4b8,
print_error = 96, ignore_not_found_rows = 239}
lock = (MYSQL_LOCK *) 0x0
return_val = 1 '\001'
_db_func_ = 0x8439380 "\200\223C\b"
_db_file_ = 0xbffff33c "ěó˙ż 0,\bló˙ż"
_db_level_ = 4
_db_framep_ = (char **) 0xbffff36c
ptr = {0x2, 0xbffff3ac, 0x0}
#1 0x080745f3 in main (argc=22, argv=0x8486060) at mysqld.cc:2293
No locals.
(gdb) p tables
$1 = {{next = 0x0, db = 0x6a6 <Address 0x6a6 out of bounds>, alias = 0x0, real_name = 0x844ef78 "", on_expr = 0x0,
natural_join = 0xbffff42c, use_index = 0x82c1f16, ignore_index = 0x844ef70, table = 0x0, grant = {
grant_table = 0xbffff43c, version = 137109270, privilege = 138735456, want_privilege = 0}, lock_type = TL_UNLOCK,
outer_join = 138736236, shared = 138735456, db_length = 0, real_name_length = 3221222492, straight = 139,
updating = 128, force_index = 42}, {next = 0x844ef60, db = 0x1000 <Address 0x1000 out of bounds>,
alias = 0xbffff46c "Źô˙ż°ô˙ż¸ô˙ż`ďD\bÎD5\bŚ\006",
real_name = 0x82a823a "1Ŕ\215eč[^_\211ě]Ă\211öU\211ĺ\203ě\fWVS\213]\f\213}\030\203ÄôSčĐ\230\001", on_expr = 0xbffff4b8,
natural_join = 0x0, use_index = 0xbffff4a8, ignore_index = 0x82a8225, table = 0xbffff4b8, grant = {grant_table = 0x0,
version = 3221222620, privilege = 134690663, want_privilege = 1706}, lock_type = -1073744728,
outer_join = 3221222572, shared = 3221222576, db_length = 3221222584, real_name_length = 138735456, straight = 206,
updating = 68, force_index = 53}, {next = 0x6a6, db = 0x844e9c0 "/var/log/mysql/mysqld.err",
alias = 0x19946 <Address 0x19946 out of bounds>, real_name = 0x2 <Address 0x2 out of bounds>, on_expr = 0x8289ae0,
natural_join = 0x83e5dd8, use_index = 0x0, ignore_index = 0xbffff4cc, table = 0xbffff4ac, grant = {
grant_table = 0xbffff4b0, version = 104774, privilege = 2, want_privilege = 135076739}, lock_type = 138734016,
outer_join = 1, shared = 0, db_length = 0, real_name_length = 1, straight = false, updating = false,
force_index = false}}
(gdb) p *tables
$2 = {next = 0x0, db = 0x6a6 <Address 0x6a6 out of bounds>, alias = 0x0, real_name = 0x844ef78 "", on_expr = 0x0,
natural_join = 0xbffff42c, use_index = 0x82c1f16, ignore_index = 0x844ef70, table = 0x0, grant = {
grant_table = 0xbffff43c, version = 137109270, privilege = 138735456, want_privilege = 0}, lock_type = TL_UNLOCK,
outer_join = 138736236, shared = 138735456, db_length = 0, real_name_length = 3221222492, straight = 139, updating = 128,
force_index = 42}
(gdb) p tables[3]
$4 = {next = 0xbffff52c, db = 0x80745f3 "\203Ä\020\204ŔtcĄ\210ôD\bĆ\005\221îD\b\001Ć\005ŹôD\b", alias = 0x0,
real_name = 0x2f400 <Address 0x2f400 out of bounds>, on_expr = 0xbffff52c, natural_join = 0x80745da,
use_index = 0xbffff5a0, ignore_index = 0xbffff594, table = 0x805c39d, grant = {grant_table = 0x1, version = 65535,
privilege = 2, want_privilege = 131072}, lock_type = TL_WRITE_CONCURRENT_INSERT, outer_join = 10, shared = 0,
db_length = 7, real_name_length = 2, straight = 104, updating = false, force_index = false}
(gdb) frame
#0 acl_init (org_thd=0x0, dont_read_acl_tables=false) at sql_acl.cc:142
142 DBUG_ENTER("acl_init");
(gdb) up
#1 0x080745f3 in main (argc=22, argv=0x8486060) at mysqld.cc:2293
2293 if (acl_init((THD*) 0, opt_noacl))
(gdb) ptype tables[3]
type = class st_table_list {
public:
st_table_list *next;
char *db;
char *alias;
char *real_name;
Item *on_expr;
st_table_list *natural_join;
List<String> *use_index;
List<String> *ignore_index;
TABLE *table;
GRANT_INFO grant;
thr_lock_type lock_type;
uint outer_join;
uint shared;
uint32 db_length;
uint32 real_name_length;
bool straight;
bool updating;
bool force_index;
st_table_list & operator=(st_table_list const &);
st_table_list(st_table_list const &);
st_table_list(void);
}
(gdb)
(gdb) ptype THD*
type = class THD : public ilink {
public:
NET net;
LEX lex;
MEM_ROOT mem_root;
HASH user_vars;
class String packet;
sockaddr_in remote;
rand_struct rand;
system_variables variables;
safe_mutex_t LOCK_delete;
char *query;
char *thread_stack;
char *host;
char *user;
char *priv_user;
char *db;
char *ip;
char priv_host[61];
uint16 peer_port;
char *proc_info;
char *host_or_ip;
uint client_capabilities;
uint sql_mode;
ulong max_client_packet_length;
ulong master_access;
ulong db_access;
TABLE *open_tables;
TABLE *temporary_tables;
TABLE *handler_tables;
MYSQL_LOCK *lock;
MYSQL_LOCK *locked_tables;
class ULL *ull;
uint dbug_sentry;
st_my_thread_var *mysys_var;
enum_server_command command;
uint32 server_id;
uint32 file_id;
char *where;
time_t start_time;
time_t time_after_lock;
time_t user_time;
time_t connect_time;
time_t thr_create_time;
thr_lock_type update_lock_default;
class delayed_insert *di;
st_transactions transaction;
Item *free_list;
---Type <return> to continue, or q <return> to quit---
Item *handler_items;
class Field *dupp_field;
sigset_t signals;
sigset_t block_signals;
ulonglong next_insert_id;
ulonglong last_insert_id;
ulonglong current_insert_id;
ulonglong limit_found_rows;
ha_rows select_limit;
ha_rows offset_limit;
ha_rows cuted_fields;
ha_rows sent_row_count;
ha_rows examined_row_count;
table_map used_tables;
USER_CONN *user_connect;
ulong query_id;
ulong version;
ulong options;
ulong thread_id;
ulong col_access;
ulong rand_saved_seed1;
ulong rand_saved_seed2;
long int dbug_thread_id;
pthread_t real_id;
uint current_tablenr;
uint tmp_table;
uint cond_count;
uint server_status;
uint open_options;
uint system_thread;
uint32 query_length;
uint32 db_length;
enum_tx_isolation session_tx_isolation;
char scramble[9];
bool slave_thread;
bool set_query_id;
bool locked;
bool count_cuted_fields;
bool some_tables_deleted;
bool no_errors;
bool allow_sum_func;
bool password;
bool fatal_error;
bool query_start_used;
bool last_insert_id_used;
bool insert_id_used;
bool rand_used;
bool in_lock_tables;
---Type <return> to continue, or q <return> to quit---
bool global_read_lock;
bool query_error;
bool bootstrap;
bool cleanup_done;
bool safe_to_cache_query;
bool killed;
LOG_INFO *current_linfo;
ulong slave_proxy_id;
NET *slave_net;
union {
public:
my_bool my_bool_value;
long int long_value;
union {...} & operator=(THD::._83 const &);
._83(THD::._83 const &);
._83(void);
} sys_var_tmp;
THD & operator=(THD const &);
virtual ~THD(void);
THD(THD const &);
THD(void);
void init(void);
void change_user(void);
void cleanup(void);
bool store_globals(void);
void awake(bool);
char * enter_cond(pthread_cond_t *, st_safe_mutex_t *, char const *);
void exit_cond(char const *);
time_t query_start(void);
void set_time(void);
void end_time(void);
void set_time(long);
void lock_time(void);
void insert_id(unsigned long long);
ulonglong insert_id(void);
ulonglong found_rows(void);
bool active_transaction(void);
gptr alloc(unsigned int);
gptr calloc(unsigned int);
char * strdup(char const *);
char * strmake(char const *, unsigned int);
char * memdup(char const *, unsigned int);
char * memdup_w_gap(char const *, unsigned int, unsigned int);
gptr trans_alloc(unsigned int);
void add_changed_table(st_table *);
void add_changed_table(char const *, long);
---Type <return> to continue, or q <return> to quit---
CHANGED_TABLE_LIST * changed_table_dup(char const *, long);
void clear_error(void);
} *
(gdb)
(gdb) ptype opt_noacl
type = char
(gdb) p opt_noacl
$1 = 0 '\0'
(gdb) p acl_init((THD*) 0, opt_noacl)
Program received signal SIGSEGV, Segmentation fault.
0x00000000 in ?? ()
The program being debugged was signaled while in a function called from GDB.
GDB remains in the frame where the signal was received.
To change this behavior use "set unwindonsignal on"
Evaluation of the expression containing the function (acl_init(THD *, bool)) will be abandoned.
(gdb)
Description: On Gentoo Linux server I cannot start mysql-debug-4.0.18 binaries: 0x80732df handle_segfault + 423 0x82c3688 pthread_sighandler + 184 (nil) 0x809324f open_table__FP3THDPCcN21Pb + 1507 0x8094346 open_tables__FP3THDP13st_table_list + 126 0x80bc206 acl_init__FP3THDb + 590 0x80745f3 main + 2639 0x82c70b4 __libc_start_main + 212 0x8048101 _start + 33 * sys-libs/glibc Latest version available: 2.3.2-r9 Latest version installed: 2.3.2-r9 Size of downloaded files: 14,946 kB Homepage: http://www.gnu.org/software/libc/libc.html Description: GNU libc6 (also called glibc2) C library License: LGPL-2 How to repeat: When I have limited number of files using ulimit(1) to 1024, I get: # cd /usr/local # /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld --defaults-file=/etc/mysql/my.cnf 040306 23:32:34 Warning: setrlimit couldn't increase number of open files to more than 1024 (request: 1006633070) /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld: Out of memory (Needed 4026531968 bytes) # # cat /proc/sys/fs/file-max 104774 # cat /proc/sys/fs/file-nr 1350 220 104774 # When I use ulimit(1) to adjust number of maxfiles to system maximum - 104774, I get: # /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld --defaults-file=/etc/mysql/my.cnf 040306 23:35:24 Warning: setrlimit couldn't increase number of open files to more than 104774 (request: 1006633070) /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld: Out of memory (Needed 4026531968 bytes) # /etc/mysql/my.cnf contains: [client] port = 3306 socket = /var/run/mysqld/mysqld.sock [safe_mysqld] err-log = /var/log/mysql/mysql.err max_allowed_packet=128M [mysqld] skip-innodb user = mysql pid-file = /var/run/mysqld/mysqld.pid socket = /var/run/mysqld/mysqld.sock log-error = /var/log/mysql/mysqld.err datadir = /var/lib/mysql tmpdir = /tmp skip-locking set-variable = max_user_connections=50 set-variable = key_buffer=128M set-variable = max_allowed_packet=128M set-variable = table_cache=480M set-variable = query_cache_size=32M set-variable = thread_stack=128K set-variable = interactive_timeout=60000 set-variable = wait_timeout=60000 set-variable = connect_timeout=60000 set-variable = net_read_timeout=60000 set-variable = open_files_limit=30000 bind-address = 127.0.0.1 port = 3306 [mysqldump] quick set-variable = max_allowed_packet=1M [mysql] max_allowed_packet = 128M [isamchk] set-variable = key_buffer=16M # strace /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld --defaults-file=/etc/mysql/my.cnf execve("/usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld", ["/usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld", "--defaults-file=/etc/mysql/my.cnf"], [/* 50 vars */]) = 0 fcntl64(0, F_GETFD) = 0 fcntl64(1, F_GETFD) = 0 fcntl64(2, F_GETFD) = 0 geteuid32() = 0 getuid32() = 0 getegid32() = 0 getgid32() = 0 getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0 setrlimit(RLIMIT_STACK, {rlim_cur=124*1024, rlim_max=RLIM_INFINITY}) = 0 <cut> read(3, "# /etc/mysql/my.cnf: The global "..., 4096) = 3164 <cut> open("/usr/local/mysql-debug-4.0.18-pc-linux-i686/share/mysql/charsets/Index", O_RDONLY|O_LARGEFILE) = 3 <cut> getrlimit(RLIMIT_NOFILE, {rlim_cur=104774, rlim_max=104774}) = 0 setrlimit(RLIMIT_NOFILE, {rlim_cur=1006633070, rlim_max=1006633070}) = -1 EPERM (Operation not permitted) time(NULL) = 1078612743 write(2, "040306 23:39:03 ", 17040306 23:39:03 ) = 17 write(2, "Warning: setrlimit couldn\'t incr"..., 99Warning: setrlimit couldn't increase number of open files to more than 104774 (request: 1006633070)) = 99 <cut> --- SIGRTMIN (Unknown signal 32) @ 0 (0) --- <... rt_sigsuspend resumed> ) = -1 EINTR (Interrupted system call) sigreturn() = ? (mask now [HUP INT QUIT PIPE TERM TSTP RTMIN]) rt_sigprocmask(SIG_SETMASK, NULL, [HUP INT QUIT PIPE TERM TSTP RTMIN], 8) = 0 rt_sigsuspend([HUP INT QUIT PIPE TERM TSTP] <unfinished ...> --- SIGRTMIN (Unknown signal 32) @ 0 (0) --- <... rt_sigsuspend resumed> ) = -1 EINTR (Interrupted system call) sigreturn() = ? (mask now [HUP INT QUIT PIPE TERM TSTP RTMIN]) open("./mysql/host.frm", O_RDONLY|O_LARGEFILE) = 7 read(7, "\376\1\7\t\3\0\0\20\1\0\0000\0\0\347\0i\0\0\0\0\0\0\0\0"..., 64) = 64 _llseek(7, 64, [64], SEEK_SET) = 0 read(7, "//\0\0 \0\0", 7) = 7 _llseek(7, 4096, [4096], SEEK_SET) = 0 read(7, "\1\2\34\0\n\0\2\\\0\2\1\200\2\0\0\0\0<\0\2\200>\0\0\0\0"..., 39) = 39 pread(7, "\377 "..., 105, 4327) = 105 _llseek(7, 8192, [8192], SEEK_SET) = 0 read(7, "h\3\0\20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 288) = 288 read(7, "\0\1\17\16\2\24) "..., 584) = 584 close(7) = 0 getcwd("/var/lib/mysql", 4095) = 15 lstat64("/var/lib/mysql/mysql", {st_mode=S_IFDIR|0755, st_size=536, ...}) = 0 lstat64("/var/lib/mysql/mysql/host.MYI", {st_mode=S_IFREG|0660, st_size=1024, ...}) = 0 open("/var/lib/mysql/mysql/host.MYI", O_RDWR|O_LARGEFILE) = 7 read(7, "\376\376\7\1\0\2\1\305\0\260\0d\0\310\0\2\0\0\1\0\10\1"..., 24) = 24 readlink("./mysql/host.MYI", 0xbfffe2ac, 511) = -1 EINVAL (Invalid argument) readlink("./mysql/host.MYD", 0xbfffe0ac, 511) = -1 EINVAL (Invalid argument) _llseek(7, 0, [0], SEEK_SET) = 0 read(7, "\376\376\7\1\0\2\1\305\0\260\0d\0\310\0\2\0\0\1\0\10\1"..., 453) = 453 open("./mysql/host.MYD", O_RDWR|O_LARGEFILE) = 8 getpid() = 3291 brk(0x849a000) = 0x849a000 brk(0x849d000) = 0x849d000 --- SIGSEGV (Segmentation fault) @ 0 (0) --- write(2, "mysqld got signal 11;\nThis could"..., 1693) = 1693 write(6, "\200\223C\10\2\0\0\0\1\0\0\0x\331D\10\1\0\0\0H\353\377"..., 148) = 148 rt_sigprocmask(SIG_SETMASK, NULL, [HUP INT QUIT PIPE TERM TSTP RTMIN], 8) = 0 rt_sigsuspend([HUP INT QUIT PIPE TERM TSTP] <unfinished ...> --- SIGRTMIN (Unknown signal 32) @ 0 (0) --- <... rt_sigsuspend resumed> ) = -1 EINTR (Interrupted system call) sigreturn() = ? (mask now [HUP INT QUIT PIPE TERM TSTP RTMIN]) wait4(3292, NULL, __WCLONE, NULL) = 3292 munmap(0x41fff000, 4096) = 0 _exit(1) = ? # (gdb) 2161 set_if_bigger(wanted_files, open_files_limit); (gdb) 2163 uint files=set_maximum_open_files(wanted_files); (gdb) 040306 23:58:00 Warning: setrlimit couldn't increase number of open files to more than 104774 (request: 1006633070) 2164 if (files && files < wanted_files && ! open_files_limit) (gdb) <cut> (gdb) 2204 server_init(); (gdb) 2205 table_cache_init(); (gdb) /usr/local/mysql-debug-4.0.18-pc-linux-i686/bin/mysqld: Out of memory (Needed 4026531968 bytes) 2206 hostname_cache_init(); (gdb) <cut> (gdb) safe_cond_wait (cond=0x844f26c, mp=0x844ef60, file=0x83544ce "mysqld.cc", line=1702) at thr_mutex.c:191 191 thr_mutex.c: No such file or directory. in thr_mutex.c (gdb) 192 in thr_mutex.c (gdb) 198 in thr_mutex.c (gdb) 206 in thr_mutex.c (gdb) 207 in thr_mutex.c (gdb) 206 in thr_mutex.c (gdb) 207 in thr_mutex.c (gdb) 208 in thr_mutex.c (gdb) 209 in thr_mutex.c (gdb) 210 in thr_mutex.c (gdb) start_signal_handler () at mysqld.cc:1703 1703 pthread_mutex_unlock(&LOCK_thread_count); Current language: auto; currently c++ (gdb) 1705 (void) pthread_attr_destroy(&thr_attr); (gdb) 1706 DBUG_VOID_RETURN; (gdb) main (argc=22, argv=0x8486060) at mysqld.cc:2293 2293 if (acl_init((THD*) 0, opt_noacl)) (gdb) Program received signal SIGSEGV, Segmentation fault. 0x00000000 in ?? () (gdb) bt full #0 0x00000000 in ?? () No symbol table info available. (gdb) where #0 0x00000000 in ?? () (gdb) bt full #0 0x00000000 in ?? () No symbol table info available. (gdb) l 2288 if (unix_sock != INVALID_SOCKET) 2289 unlink(mysql_unix_port); 2290 unireg_abort(1); 2291 } 2292 start_signal_handler(); // Creates pidfile 2293 if (acl_init((THD*) 0, opt_noacl)) 2294 { 2295 abort_loop=1; 2296 select_thread_in_use=0; 2297 #ifndef __NETWARE__ (gdb) From another gdb session: (gdb) c Continuing. Breakpoint 1, acl_init (org_thd=0x0, dont_read_acl_tables=false) at sql_acl.cc:142 142 DBUG_ENTER("acl_init"); Current language: auto; currently c++ (gdb) l 137 TABLE_LIST tables[3]; 138 TABLE *table; 139 READ_RECORD read_record_info; 140 MYSQL_LOCK *lock; 141 my_bool return_val=1; 142 DBUG_ENTER("acl_init"); 143 144 if (!acl_cache) 145 acl_cache=new hash_filo(ACL_CACHE_SIZE,0,0, 146 (hash_get_key) acl_entry_get_key, (gdb) bt full #0 acl_init (org_thd=0x0, dont_read_acl_tables=false) at sql_acl.cc:142 thd = (THD *) 0xbffff36c tables = {{next = 0x0, db = 0x6a6 <Address 0x6a6 out of bounds>, alias = 0x0, real_name = 0x844ef78 "", on_expr = 0x0, natural_join = 0xbffff42c, use_index = 0x82c1f16, ignore_index = 0x844ef70, table = 0x0, grant = { grant_table = 0xbffff43c, version = 137109270, privilege = 138735456, want_privilege = 0}, lock_type = TL_UNLOCK, outer_join = 138736236, shared = 138735456, db_length = 0, real_name_length = 3221222492, straight = 139, updating = 128, force_index = 42}, {next = 0x844ef60, db = 0x1000 <Address 0x1000 out of bounds>, alias = 0xbffff46c "Źô˙ż°ô˙ż¸ô˙ż`ďD\bÎD5\bŚ\006", real_name = 0x82a823a "1Ŕ\215eč[^_\211ě]Ă\211öU\211ĺ\203ě\fWVS\213]\f\213}\030\203ÄôSčĐ\230\001", on_expr = 0xbffff4b8, natural_join = 0x0, use_index = 0xbffff4a8, ignore_index = 0x82a8225, table = 0xbffff4b8, grant = {grant_table = 0x0, version = 3221222620, privilege = 134690663, want_privilege = 1706}, lock_type = -1073744728, outer_join = 3221222572, shared = 3221222576, db_length = 3221222584, real_name_length = 138735456, straight = 206, updating = 68, force_index = 53}, {next = 0x6a6, db = 0x844e9c0 "/var/log/mysql/mysqld.err", alias = 0x19946 <Address 0x19946 out of bounds>, real_name = 0x2 <Address 0x2 out of bounds>, on_expr = 0x8289ae0, natural_join = 0x83e5dd8, use_index = 0x0, ignore_index = 0xbffff4cc, table = 0xbffff4ac, grant = { grant_table = 0xbffff4b0, version = 104774, privilege = 2, want_privilege = 135076739}, lock_type = 138734016, outer_join = 1, shared = 0, db_length = 0, real_name_length = 1, straight = false, updating = false, force_index = false}} table = (TABLE *) 0x0 read_record_info = {table = 0x844ef78, file = 0x844f26c, forms = 0x844ef78, read_record = 0x82df961 <__libc_malloc+209>, thd = 0x8439380, select = 0x844ef78, cache_records = 138735480, ref_length = 3220700160, struct_length = 3220700160, reclength = 0, rec_cache_size = 3221222380, error_offset = 137109156, index = 138646400, ref_pos = 0x0, record = 0xbffff3ec "Ś\006", cache = 0x82c1b6e "1Ŕé\002\001", cache_pos = 0x844ef60 "", cache_end = 0x0, read_positions = 0x6a7 <Address 0x6a7 out of bounds>, io_cache = 0xbffff4b8, print_error = 96, ignore_not_found_rows = 239} lock = (MYSQL_LOCK *) 0x0 return_val = 1 '\001' _db_func_ = 0x8439380 "\200\223C\b" _db_file_ = 0xbffff33c "ěó˙ż 0,\bló˙ż" _db_level_ = 4 _db_framep_ = (char **) 0xbffff36c ptr = {0x2, 0xbffff3ac, 0x0} #1 0x080745f3 in main (argc=22, argv=0x8486060) at mysqld.cc:2293 No locals. (gdb) p tables $1 = {{next = 0x0, db = 0x6a6 <Address 0x6a6 out of bounds>, alias = 0x0, real_name = 0x844ef78 "", on_expr = 0x0, natural_join = 0xbffff42c, use_index = 0x82c1f16, ignore_index = 0x844ef70, table = 0x0, grant = { grant_table = 0xbffff43c, version = 137109270, privilege = 138735456, want_privilege = 0}, lock_type = TL_UNLOCK, outer_join = 138736236, shared = 138735456, db_length = 0, real_name_length = 3221222492, straight = 139, updating = 128, force_index = 42}, {next = 0x844ef60, db = 0x1000 <Address 0x1000 out of bounds>, alias = 0xbffff46c "Źô˙ż°ô˙ż¸ô˙ż`ďD\bÎD5\bŚ\006", real_name = 0x82a823a "1Ŕ\215eč[^_\211ě]Ă\211öU\211ĺ\203ě\fWVS\213]\f\213}\030\203ÄôSčĐ\230\001", on_expr = 0xbffff4b8, natural_join = 0x0, use_index = 0xbffff4a8, ignore_index = 0x82a8225, table = 0xbffff4b8, grant = {grant_table = 0x0, version = 3221222620, privilege = 134690663, want_privilege = 1706}, lock_type = -1073744728, outer_join = 3221222572, shared = 3221222576, db_length = 3221222584, real_name_length = 138735456, straight = 206, updating = 68, force_index = 53}, {next = 0x6a6, db = 0x844e9c0 "/var/log/mysql/mysqld.err", alias = 0x19946 <Address 0x19946 out of bounds>, real_name = 0x2 <Address 0x2 out of bounds>, on_expr = 0x8289ae0, natural_join = 0x83e5dd8, use_index = 0x0, ignore_index = 0xbffff4cc, table = 0xbffff4ac, grant = { grant_table = 0xbffff4b0, version = 104774, privilege = 2, want_privilege = 135076739}, lock_type = 138734016, outer_join = 1, shared = 0, db_length = 0, real_name_length = 1, straight = false, updating = false, force_index = false}} (gdb) p *tables $2 = {next = 0x0, db = 0x6a6 <Address 0x6a6 out of bounds>, alias = 0x0, real_name = 0x844ef78 "", on_expr = 0x0, natural_join = 0xbffff42c, use_index = 0x82c1f16, ignore_index = 0x844ef70, table = 0x0, grant = { grant_table = 0xbffff43c, version = 137109270, privilege = 138735456, want_privilege = 0}, lock_type = TL_UNLOCK, outer_join = 138736236, shared = 138735456, db_length = 0, real_name_length = 3221222492, straight = 139, updating = 128, force_index = 42} (gdb) p tables[3] $4 = {next = 0xbffff52c, db = 0x80745f3 "\203Ä\020\204ŔtcĄ\210ôD\bĆ\005\221îD\b\001Ć\005ŹôD\b", alias = 0x0, real_name = 0x2f400 <Address 0x2f400 out of bounds>, on_expr = 0xbffff52c, natural_join = 0x80745da, use_index = 0xbffff5a0, ignore_index = 0xbffff594, table = 0x805c39d, grant = {grant_table = 0x1, version = 65535, privilege = 2, want_privilege = 131072}, lock_type = TL_WRITE_CONCURRENT_INSERT, outer_join = 10, shared = 0, db_length = 7, real_name_length = 2, straight = 104, updating = false, force_index = false} (gdb) frame #0 acl_init (org_thd=0x0, dont_read_acl_tables=false) at sql_acl.cc:142 142 DBUG_ENTER("acl_init"); (gdb) up #1 0x080745f3 in main (argc=22, argv=0x8486060) at mysqld.cc:2293 2293 if (acl_init((THD*) 0, opt_noacl)) (gdb) ptype tables[3] type = class st_table_list { public: st_table_list *next; char *db; char *alias; char *real_name; Item *on_expr; st_table_list *natural_join; List<String> *use_index; List<String> *ignore_index; TABLE *table; GRANT_INFO grant; thr_lock_type lock_type; uint outer_join; uint shared; uint32 db_length; uint32 real_name_length; bool straight; bool updating; bool force_index; st_table_list & operator=(st_table_list const &); st_table_list(st_table_list const &); st_table_list(void); } (gdb) (gdb) ptype THD* type = class THD : public ilink { public: NET net; LEX lex; MEM_ROOT mem_root; HASH user_vars; class String packet; sockaddr_in remote; rand_struct rand; system_variables variables; safe_mutex_t LOCK_delete; char *query; char *thread_stack; char *host; char *user; char *priv_user; char *db; char *ip; char priv_host[61]; uint16 peer_port; char *proc_info; char *host_or_ip; uint client_capabilities; uint sql_mode; ulong max_client_packet_length; ulong master_access; ulong db_access; TABLE *open_tables; TABLE *temporary_tables; TABLE *handler_tables; MYSQL_LOCK *lock; MYSQL_LOCK *locked_tables; class ULL *ull; uint dbug_sentry; st_my_thread_var *mysys_var; enum_server_command command; uint32 server_id; uint32 file_id; char *where; time_t start_time; time_t time_after_lock; time_t user_time; time_t connect_time; time_t thr_create_time; thr_lock_type update_lock_default; class delayed_insert *di; st_transactions transaction; Item *free_list; ---Type <return> to continue, or q <return> to quit--- Item *handler_items; class Field *dupp_field; sigset_t signals; sigset_t block_signals; ulonglong next_insert_id; ulonglong last_insert_id; ulonglong current_insert_id; ulonglong limit_found_rows; ha_rows select_limit; ha_rows offset_limit; ha_rows cuted_fields; ha_rows sent_row_count; ha_rows examined_row_count; table_map used_tables; USER_CONN *user_connect; ulong query_id; ulong version; ulong options; ulong thread_id; ulong col_access; ulong rand_saved_seed1; ulong rand_saved_seed2; long int dbug_thread_id; pthread_t real_id; uint current_tablenr; uint tmp_table; uint cond_count; uint server_status; uint open_options; uint system_thread; uint32 query_length; uint32 db_length; enum_tx_isolation session_tx_isolation; char scramble[9]; bool slave_thread; bool set_query_id; bool locked; bool count_cuted_fields; bool some_tables_deleted; bool no_errors; bool allow_sum_func; bool password; bool fatal_error; bool query_start_used; bool last_insert_id_used; bool insert_id_used; bool rand_used; bool in_lock_tables; ---Type <return> to continue, or q <return> to quit--- bool global_read_lock; bool query_error; bool bootstrap; bool cleanup_done; bool safe_to_cache_query; bool killed; LOG_INFO *current_linfo; ulong slave_proxy_id; NET *slave_net; union { public: my_bool my_bool_value; long int long_value; union {...} & operator=(THD::._83 const &); ._83(THD::._83 const &); ._83(void); } sys_var_tmp; THD & operator=(THD const &); virtual ~THD(void); THD(THD const &); THD(void); void init(void); void change_user(void); void cleanup(void); bool store_globals(void); void awake(bool); char * enter_cond(pthread_cond_t *, st_safe_mutex_t *, char const *); void exit_cond(char const *); time_t query_start(void); void set_time(void); void end_time(void); void set_time(long); void lock_time(void); void insert_id(unsigned long long); ulonglong insert_id(void); ulonglong found_rows(void); bool active_transaction(void); gptr alloc(unsigned int); gptr calloc(unsigned int); char * strdup(char const *); char * strmake(char const *, unsigned int); char * memdup(char const *, unsigned int); char * memdup_w_gap(char const *, unsigned int, unsigned int); gptr trans_alloc(unsigned int); void add_changed_table(st_table *); void add_changed_table(char const *, long); ---Type <return> to continue, or q <return> to quit--- CHANGED_TABLE_LIST * changed_table_dup(char const *, long); void clear_error(void); } * (gdb) (gdb) ptype opt_noacl type = char (gdb) p opt_noacl $1 = 0 '\0' (gdb) p acl_init((THD*) 0, opt_noacl) Program received signal SIGSEGV, Segmentation fault. 0x00000000 in ?? () The program being debugged was signaled while in a function called from GDB. GDB remains in the frame where the signal was received. To change this behavior use "set unwindonsignal on" Evaluation of the expression containing the function (acl_init(THD *, bool)) will be abandoned. (gdb)