Description:
We found on windows when MySQL server is started, there is a separate alarm thread. On solaris there isn't such thread. The consequence is all other user threads have to call pthread_sigmask very frequently causing very prominent effect with lwp_sigmask. We modified the solaris mysql code to make it also use a separate alarm thread and saw 20% performance gain with sysbench read-write test.
We use lockstat to observe the kernel level mutex
This is before we are using separate thread.
Adaptive mutex spin: 65663 events in 5.031 seconds (13050 events/sec)
Count indv cuml rcnt nsec Lock Caller
-------------------------------------------------------------------------------
40822 62% 62% 0.00 2147 0xffffff09051b3f80 lwp_sigmask+0x41
8669 13% 75% 0.00 2559 0xffffff09051b3f80 trap+0xfc6
2602 4% 79% 0.00 2387 0xffffff09051b3f80 lwp_unpark+0x32
After the change , so you can see lwp_sigmask is gone but lwp_unpark
inches up meaning we are contending more on user level locks.
Adaptive mutex spin: 14490 events in 5.032 seconds (2880 events/sec)
Count indv cuml rcnt nsec Lock Caller
-------------------------------------------------------------------------------
5756 40% 40% 0.00 2201 0xffffff09051b42c0 trap+0xfc6
4536 31% 71% 0.00 2412 0xffffff09051b42c0 lwp_unpark+0x32
443 3% 74% 0.00 2988 0xffffff09051b3f00 lwp_unpark+0x32
I attached the difference of the code in the "suggested fix" section
How to repeat:
This can be easily reproduced with running 1M row sysbench read-write test
Suggested fix:
This is the diff of the code
diff -cw thr_alarm.c.save thr_alarm.c
*** thr_alarm.c.save Fri Aug 1 12:23:09 2008
--- thr_alarm.c Mon Aug 4 10:23:25 2008
***************
*** 51,56 ****
--- 51,60 ----
static uint max_used_alarms=0;
pthread_t alarm_thread;
+ #if !defined(USE_ALARM_THREAD)
+ #define USE_ALARM_THREAD
+ #endif
+
#ifdef USE_ALARM_THREAD
static void *alarm_handler(void *arg);
#define reschedule_alarms() pthread_cond_signal(&COND_alarm)
***************
*** 158,164 ****
--- 162,170 ----
DBUG_PRINT("enter",("thread: %s sec: %d",my_thread_name(),sec));
now=(ulong) time((time_t*) 0);
+ #if !defined(USE_ALARM_THREAD)
pthread_sigmask(SIG_BLOCK,&full_signal_set,&old_mask);
+ #endif
pthread_mutex_lock(&LOCK_alarm); /* Lock from threads & alarms */
if (alarm_aborted > 0)
{ /* No signal thread */
***************
*** 165,171 ****
--- 171,179 ----
DBUG_PRINT("info", ("alarm aborted"));
*alrm= 0; /* No alarm */
pthread_mutex_unlock(&LOCK_alarm);
+ #if !defined(USE_ALARM_THREAD)
pthread_sigmask(SIG_SETMASK,&old_mask,NULL);
+ #endif
DBUG_RETURN(1);
}
if (alarm_aborted < 0)
***************
*** 179,185 ****
--- 187,195 ----
fprintf(stderr,"Warning: thr_alarm queue is full\n");
*alrm= 0; /* No alarm */
pthread_mutex_unlock(&LOCK_alarm);
+ #if !defined(USE_ALARM_THREAD)
pthread_sigmask(SIG_SETMASK,&old_mask,NULL);
+ #endif
DBUG_RETURN(1);
}
max_used_alarms=alarm_queue.elements+1;
***************
*** 194,200 ****
--- 204,212 ----
DBUG_PRINT("info", ("failed my_malloc()"));
*alrm= 0; /* No alarm */
pthread_mutex_unlock(&LOCK_alarm);
+ #if !defined(USE_ALARM_THREAD)
pthread_sigmask(SIG_SETMASK,&old_mask,NULL);
+ #endif
DBUG_RETURN(1);
}
alarm_data->malloced=1;
***************
*** 216,222 ****
--- 228,236 ----
reschedule_alarms(); /* Reschedule alarms */
}
pthread_mutex_unlock(&LOCK_alarm);
+ #if !defined(USE_ALARM_THREAD)
pthread_sigmask(SIG_SETMASK,&old_mask,NULL);
+ #endif
(*alrm)= &alarm_data->alarmed;
DBUG_RETURN(0);
}
***************
*** 233,239 ****
--- 247,255 ----
uint i, found=0;
DBUG_ENTER("thr_end_alarm");
+ #if !defined(USE_ALARM_THREAD)
pthread_sigmask(SIG_BLOCK,&full_signal_set,&old_mask);
+ #endif
pthread_mutex_lock(&LOCK_alarm);
alarm_data= (ALARM*) ((byte*) *alarmed - offsetof(ALARM,alarmed));
***************
*** 260,266 ****
--- 276,284 ----
(long) *alarmed));
}
pthread_mutex_unlock(&LOCK_alarm);
+ #if !defined(USE_ALARM_THREAD)
pthread_sigmask(SIG_SETMASK,&old_mask,NULL);
+ #endif
DBUG_VOID_RETURN;
}
Description: We found on windows when MySQL server is started, there is a separate alarm thread. On solaris there isn't such thread. The consequence is all other user threads have to call pthread_sigmask very frequently causing very prominent effect with lwp_sigmask. We modified the solaris mysql code to make it also use a separate alarm thread and saw 20% performance gain with sysbench read-write test. We use lockstat to observe the kernel level mutex This is before we are using separate thread. Adaptive mutex spin: 65663 events in 5.031 seconds (13050 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 40822 62% 62% 0.00 2147 0xffffff09051b3f80 lwp_sigmask+0x41 8669 13% 75% 0.00 2559 0xffffff09051b3f80 trap+0xfc6 2602 4% 79% 0.00 2387 0xffffff09051b3f80 lwp_unpark+0x32 After the change , so you can see lwp_sigmask is gone but lwp_unpark inches up meaning we are contending more on user level locks. Adaptive mutex spin: 14490 events in 5.032 seconds (2880 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 5756 40% 40% 0.00 2201 0xffffff09051b42c0 trap+0xfc6 4536 31% 71% 0.00 2412 0xffffff09051b42c0 lwp_unpark+0x32 443 3% 74% 0.00 2988 0xffffff09051b3f00 lwp_unpark+0x32 I attached the difference of the code in the "suggested fix" section How to repeat: This can be easily reproduced with running 1M row sysbench read-write test Suggested fix: This is the diff of the code diff -cw thr_alarm.c.save thr_alarm.c *** thr_alarm.c.save Fri Aug 1 12:23:09 2008 --- thr_alarm.c Mon Aug 4 10:23:25 2008 *************** *** 51,56 **** --- 51,60 ---- static uint max_used_alarms=0; pthread_t alarm_thread; + #if !defined(USE_ALARM_THREAD) + #define USE_ALARM_THREAD + #endif + #ifdef USE_ALARM_THREAD static void *alarm_handler(void *arg); #define reschedule_alarms() pthread_cond_signal(&COND_alarm) *************** *** 158,164 **** --- 162,170 ---- DBUG_PRINT("enter",("thread: %s sec: %d",my_thread_name(),sec)); now=(ulong) time((time_t*) 0); + #if !defined(USE_ALARM_THREAD) pthread_sigmask(SIG_BLOCK,&full_signal_set,&old_mask); + #endif pthread_mutex_lock(&LOCK_alarm); /* Lock from threads & alarms */ if (alarm_aborted > 0) { /* No signal thread */ *************** *** 165,171 **** --- 171,179 ---- DBUG_PRINT("info", ("alarm aborted")); *alrm= 0; /* No alarm */ pthread_mutex_unlock(&LOCK_alarm); + #if !defined(USE_ALARM_THREAD) pthread_sigmask(SIG_SETMASK,&old_mask,NULL); + #endif DBUG_RETURN(1); } if (alarm_aborted < 0) *************** *** 179,185 **** --- 187,195 ---- fprintf(stderr,"Warning: thr_alarm queue is full\n"); *alrm= 0; /* No alarm */ pthread_mutex_unlock(&LOCK_alarm); + #if !defined(USE_ALARM_THREAD) pthread_sigmask(SIG_SETMASK,&old_mask,NULL); + #endif DBUG_RETURN(1); } max_used_alarms=alarm_queue.elements+1; *************** *** 194,200 **** --- 204,212 ---- DBUG_PRINT("info", ("failed my_malloc()")); *alrm= 0; /* No alarm */ pthread_mutex_unlock(&LOCK_alarm); + #if !defined(USE_ALARM_THREAD) pthread_sigmask(SIG_SETMASK,&old_mask,NULL); + #endif DBUG_RETURN(1); } alarm_data->malloced=1; *************** *** 216,222 **** --- 228,236 ---- reschedule_alarms(); /* Reschedule alarms */ } pthread_mutex_unlock(&LOCK_alarm); + #if !defined(USE_ALARM_THREAD) pthread_sigmask(SIG_SETMASK,&old_mask,NULL); + #endif (*alrm)= &alarm_data->alarmed; DBUG_RETURN(0); } *************** *** 233,239 **** --- 247,255 ---- uint i, found=0; DBUG_ENTER("thr_end_alarm"); + #if !defined(USE_ALARM_THREAD) pthread_sigmask(SIG_BLOCK,&full_signal_set,&old_mask); + #endif pthread_mutex_lock(&LOCK_alarm); alarm_data= (ALARM*) ((byte*) *alarmed - offsetof(ALARM,alarmed)); *************** *** 260,266 **** --- 276,284 ---- (long) *alarmed)); } pthread_mutex_unlock(&LOCK_alarm); + #if !defined(USE_ALARM_THREAD) pthread_sigmask(SIG_SETMASK,&old_mask,NULL); + #endif DBUG_VOID_RETURN; }