diff -ru mysql-5.0.54_orig/innobase/configure mysql-5.0.54/innobase/configure --- mysql-5.0.54_orig/innobase/configure 2007-12-25 00:20:37.000000000 +0900 +++ mysql-5.0.54/innobase/configure 2008-01-26 12:41:00.000000000 +0900 @@ -21058,6 +21058,88 @@ fi done + +# as http://lists.mysql.com/commits/40686 does +{ echo "$as_me:$LINENO: checking whether the compiler provides atomic builtins" >&5 +echo $ECHO_N "checking whether the compiler provides atomic builtins... $ECHO_C" >&6; } +if test "${mysql_cv_atomic_builtins+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test "$cross_compiling" = yes; then + { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling +See \`config.log' for more details." >&5 +echo "$as_me: error: cannot run test program while cross compiling +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; } +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + + int main() + { + int foo= -10; int bar= 10; + __sync_fetch_and_add(&foo, bar); + if (foo) + return -1; + bar= __sync_lock_test_and_set(&foo, bar); + if (bar || foo != 10) + return -1; + bar= __sync_val_compare_and_swap(&bar, foo, 15); + if (bar) + return -1; + return 0; + } + +_ACEOF +rm -f conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' + { (case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + mysql_cv_atomic_builtins=yes +else + echo "$as_me: program exited with status $ac_status" >&5 +echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +( exit $ac_status ) +mysql_cv_atomic_builtins=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext +fi + + +fi +{ echo "$as_me:$LINENO: result: $mysql_cv_atomic_builtins" >&5 +echo "${ECHO_T}$mysql_cv_atomic_builtins" >&6; } + +if test "x$mysql_cv_atomic_builtins" = xyes; then + +cat >>confdefs.h <<\_ACEOF +#define HAVE_ATOMIC_BUILTINS 1 +_ACEOF + +fi + #AC_CHECK_FUNCS(readdir_r) MySQL checks that it has also the right args. # Some versions of Unix only take 2 arguments. #AC_C_INLINE Already checked in MySQL diff -ru mysql-5.0.54_orig/innobase/configure.in mysql-5.0.54/innobase/configure.in --- mysql-5.0.54_orig/innobase/configure.in 2007-04-19 10:36:16.000000000 +0900 +++ mysql-5.0.54/innobase/configure.in 2008-01-26 12:41:00.000000000 +0900 @@ -42,6 +42,31 @@ AC_CHECK_FUNCS(sched_yield) AC_CHECK_FUNCS(fdatasync) AC_CHECK_FUNCS(localtime_r) + +# as http://lists.mysql.com/commits/40686 does +AC_CACHE_CHECK([whether the compiler provides atomic builtins], + [mysql_cv_atomic_builtins], [AC_TRY_RUN([ + int main() + { + int foo= -10; int bar= 10; + __sync_fetch_and_add(&foo, bar); + if (foo) + return -1; + bar= __sync_lock_test_and_set(&foo, bar); + if (bar || foo != 10) + return -1; + bar= __sync_val_compare_and_swap(&bar, foo, 15); + if (bar) + return -1; + return 0; + } +], [mysql_cv_atomic_builtins=yes], [mysql_cv_atomic_builtins=no])]) + +if test "x$mysql_cv_atomic_builtins" = xyes; then + AC_DEFINE(HAVE_ATOMIC_BUILTINS, 1, + [Define to 1 if compiler provides atomic builtins.]) +fi + #AC_CHECK_FUNCS(readdir_r) MySQL checks that it has also the right args. # Some versions of Unix only take 2 arguments. #AC_C_INLINE Already checked in MySQL diff -ru mysql-5.0.54_orig/innobase/ib_config.h mysql-5.0.54/innobase/ib_config.h --- mysql-5.0.54_orig/innobase/ib_config.h 2007-12-25 00:22:15.000000000 +0900 +++ mysql-5.0.54/innobase/ib_config.h 2008-01-26 12:41:00.000000000 +0900 @@ -4,6 +4,9 @@ /* Define to 1 if you have the header file. */ #define HAVE_AIO_H 1 +/* Define to 1 if compiler provides atomic builtins. */ +#define HAVE_ATOMIC_BUILTINS 1 + /* Define to 1 if you have the header file. */ #define HAVE_DLFCN_H 1 diff -ru mysql-5.0.54_orig/innobase/ib_config.h.in mysql-5.0.54/innobase/ib_config.h.in --- mysql-5.0.54_orig/innobase/ib_config.h.in 2007-12-25 00:20:28.000000000 +0900 +++ mysql-5.0.54/innobase/ib_config.h.in 2008-01-26 12:41:00.000000000 +0900 @@ -3,6 +3,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_AIO_H +/* Define to 1 if compiler provides atomic builtins. */ +#undef HAVE_ATOMIC_BUILTINS + /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H diff -ru mysql-5.0.54_orig/innobase/include/sync0rw.h mysql-5.0.54/innobase/include/sync0rw.h --- mysql-5.0.54_orig/innobase/include/sync0rw.h 2007-11-21 02:38:06.000000000 +0900 +++ mysql-5.0.54/innobase/include/sync0rw.h 2008-01-26 13:35:20.000000000 +0900 @@ -323,11 +323,24 @@ rw_lock_t* lock); /* in: rw-lock */ /************************************************************************ Accessor functions for rw lock. */ +#ifdef HAVE_ATOMIC_BUILTINS +UNIV_INLINE +ulint +rw_lock_get_s_waiters( +/*==================*/ + rw_lock_t* lock); +UNIV_INLINE +ulint +rw_lock_get_x_waiters( +/*==================*/ + rw_lock_t* lock); +#else UNIV_INLINE ulint rw_lock_get_waiters( /*================*/ rw_lock_t* lock); +#endif UNIV_INLINE ulint rw_lock_get_writer( @@ -408,6 +421,11 @@ rw_lock_debug_t* info); /* in: debug struct */ #endif /* UNIV_SYNC_DEBUG */ +#ifdef HAVE_ATOMIC_BUILTINS +/* This value means NOT_LOCKED */ +#define RW_LOCK_BIAS 0x00100000 +#endif + /* NOTE! The structure appears here only for the compiler to know its size. Do not use its fields directly! The structure used in the spin lock implementation of a read-write lock. Several threads may have a shared lock @@ -417,7 +435,12 @@ field. Then no new readers are allowed in. */ struct rw_lock_struct { +#ifdef HAVE_ATOMIC_BUILTINS + os_event_t s_event; /* Used by s_lock */ + os_event_t x_event; /* Used by x_lock */ +#else os_event_t event; /* Used by sync0arr.c for thread queueing */ +#endif #ifdef __WIN__ os_event_t wait_ex_event; /* This windows specific event is @@ -429,29 +452,40 @@ signalled. See LEMMA 2 in sync0sync.c */ #endif - ulint reader_count; /* Number of readers who have locked this +#ifdef HAVE_ATOMIC_BUILTINS + volatile lint lock_word; /* Used by using atomic builtin */ +#endif + + volatile ulint reader_count; /* Number of readers who have locked this lock in the shared mode */ - ulint writer; /* This field is set to RW_LOCK_EX if there + volatile ulint writer; /* This field is set to RW_LOCK_EX if there is a writer owning the lock (in exclusive mode), RW_LOCK_WAIT_EX if a writer is queueing for the lock, and RW_LOCK_NOT_LOCKED, otherwise. */ - os_thread_id_t writer_thread; + volatile os_thread_id_t writer_thread; /* Thread id of a possible writer thread */ - ulint writer_count; /* Number of times the same thread has + volatile ulint writer_count; /* Number of times the same thread has recursively locked the lock in the exclusive mode */ +#ifndef HAVE_ATOMIC_BUILTINS mutex_t mutex; /* The mutex protecting rw_lock_struct */ +#endif ulint pass; /* Default value 0. This is set to some value != 0 given by the caller of an x-lock operation, if the x-lock is to be passed to another thread to unlock (which happens in asynchronous i/o). */ - ulint waiters; /* This ulint is set to 1 if there are +#ifdef HAVE_ATOMIC_BUILTINS + volatile ulint s_waiters; /* 1: there are waiters (readers) */ + volatile ulint x_waiters; /* 1: there are waiters (writers) */ +#else + volatile ulint waiters; /* This ulint is set to 1 if there are waiters (readers or writers) in the global wait array, waiting for this rw_lock. Otherwise, == 0. */ - ibool writer_is_wait_ex; +#endif + volatile ibool writer_is_wait_ex; /* This is TRUE if the writer field is RW_LOCK_WAIT_EX; this field is located far from the memory update hotspot fields which diff -ru mysql-5.0.54_orig/innobase/include/sync0rw.ic mysql-5.0.54/innobase/include/sync0rw.ic --- mysql-5.0.54_orig/innobase/include/sync0rw.ic 2007-11-21 02:38:06.000000000 +0900 +++ mysql-5.0.54/innobase/include/sync0rw.ic 2008-01-26 20:14:43.000000000 +0900 @@ -45,6 +45,40 @@ /************************************************************************ Accessor functions for rw lock. */ +#ifdef HAVE_ATOMIC_BUILTINS +UNIV_INLINE +ulint +rw_lock_get_s_waiters( +/*================*/ + rw_lock_t* lock) +{ + return(lock->s_waiters); +} +UNIV_INLINE +ulint +rw_lock_get_x_waiters( +/*================*/ + rw_lock_t* lock) +{ + return(lock->x_waiters); +} +UNIV_INLINE +void +rw_lock_set_s_waiters( + rw_lock_t* lock, + ulint flag) +{ + lock->s_waiters = flag; +} +UNIV_INLINE +void +rw_lock_set_x_waiters( + rw_lock_t* lock, + ulint flag) +{ + lock->x_waiters = flag; +} +#else UNIV_INLINE ulint rw_lock_get_waiters( @@ -62,13 +96,24 @@ { lock->waiters = flag; } +#endif UNIV_INLINE ulint rw_lock_get_writer( /*===============*/ rw_lock_t* lock) { +#ifdef HAVE_ATOMIC_BUILTINS + /* lock->writer is used for contention of x_lock. */ + /* We shouldn't pick up inconsistent value. */ + ulint ret; + do { + ret = lock->writer; + } while(ret > RW_LOCK_WAIT_EX || ret < RW_LOCK_NOT_LOCKED); + return(ret); +#else return(lock->writer); +#endif } UNIV_INLINE void @@ -96,6 +141,7 @@ { lock->reader_count = count; } +#ifndef HAVE_ATOMIC_BUILTINS UNIV_INLINE mutex_t* rw_lock_get_mutex( @@ -104,6 +150,7 @@ { return(&(lock->mutex)); } +#endif /********************************************************************** Returns the value of writer_count for the lock. Does not reserve the lock @@ -133,14 +180,26 @@ const char* file_name, /* in: file name where lock requested */ ulint line) /* in: line where requested */ { -#ifdef UNIV_SYNC_DEBUG +#if defined(UNIV_SYNC_DEBUG) && !defined(HAVE_ATOMIC_BUILTINS) ut_ad(mutex_own(rw_lock_get_mutex(lock))); #endif /* UNIV_SYNC_DEBUG */ /* Check if the writer field is free */ +#ifdef HAVE_ATOMIC_BUILTINS + if (UNIV_LIKELY(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED)) { + /* try s-lock */ + if(__sync_sub_and_fetch(&(lock->lock_word),1) <= 0) { + /* fail */ + __sync_fetch_and_add(&(lock->lock_word),1); + return(FALSE); /* locking did not succeed */ + } + /* success */ + __sync_fetch_and_add(&(lock->reader_count),1); +#else if (UNIV_LIKELY(lock->writer == RW_LOCK_NOT_LOCKED)) { /* Set the shared lock by incrementing the reader count */ lock->reader_count++; +#endif #ifdef UNIV_SYNC_DEBUG rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name, @@ -167,11 +226,15 @@ const char* file_name, /* in: file name where requested */ ulint line) /* in: line where lock requested */ { - ut_ad(lock->writer == RW_LOCK_NOT_LOCKED); + ut_ad(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED); ut_ad(rw_lock_get_reader_count(lock) == 0); /* Set the shared lock by incrementing the reader count */ +#ifdef HAVE_ATOMIC_BUILTINS + __sync_fetch_and_add(&(lock->reader_count),1); +#else lock->reader_count++; +#endif lock->last_s_file_name = file_name; lock->last_s_line = line; @@ -199,7 +262,11 @@ rw_lock_set_writer(lock, RW_LOCK_EX); lock->writer_thread = os_thread_get_curr_id(); +#ifdef HAVE_ATOMIC_BUILTINS + __sync_fetch_and_add(&(lock->writer_count),1); +#else lock->writer_count++; +#endif lock->pass = 0; lock->last_x_file_name = file_name; @@ -241,15 +308,21 @@ ut_ad(!rw_lock_own(lock, RW_LOCK_SHARED)); /* see NOTE above */ #endif /* UNIV_SYNC_DEBUG */ +#ifndef HAVE_ATOMIC_BUILTINS mutex_enter(rw_lock_get_mutex(lock)); +#endif if (UNIV_LIKELY(rw_lock_s_lock_low(lock, pass, file_name, line))) { +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(rw_lock_get_mutex(lock)); +#endif return; /* Success */ } else { /* Did not succeed, try spin wait */ +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(rw_lock_get_mutex(lock)); +#endif rw_lock_s_lock_spin(lock, pass, file_name, line); @@ -272,11 +345,23 @@ { ibool success = FALSE; +#ifdef HAVE_ATOMIC_BUILTINS + if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) { + /* try s-lock */ + if(__sync_sub_and_fetch(&(lock->lock_word),1) <= 0) { + /* fail */ + __sync_fetch_and_add(&(lock->lock_word),1); + return(FALSE); /* locking did not succeed */ + } + /* success */ + __sync_fetch_and_add(&(lock->reader_count),1); +#else mutex_enter(rw_lock_get_mutex(lock)); if (lock->writer == RW_LOCK_NOT_LOCKED) { /* Set the shared lock by incrementing the reader count */ lock->reader_count++; +#endif #ifdef UNIV_SYNC_DEBUG rw_lock_add_debug_info(lock, 0, RW_LOCK_SHARED, file_name, @@ -289,7 +374,9 @@ success = TRUE; } +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(rw_lock_get_mutex(lock)); +#endif return(success); } @@ -309,6 +396,59 @@ { ibool success = FALSE; os_thread_id_t curr_thread = os_thread_get_curr_id(); +#ifdef HAVE_ATOMIC_BUILTINS + if ((lock->lock_word == RW_LOCK_BIAS) + && rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) { + /* try x-lock */ + if(__sync_sub_and_fetch(&(lock->lock_word), + RW_LOCK_BIAS) == 0) { + /* success */ + /* try to lock writer */ + if(__sync_sub_and_fetch(&(lock->writer), + RW_LOCK_NOT_LOCKED) == 0) { + /* success */ + lock->writer_thread = curr_thread; + lock->pass = 0; + /* 0 -> RW_LOCK_EX */ + __sync_fetch_and_add(&(lock->writer), + RW_LOCK_EX); + relock: + __sync_fetch_and_add(&(lock->writer_count),1); + +#ifdef UNIV_SYNC_DEBUG + rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line); +#endif + + lock->last_x_file_name = file_name; + lock->last_x_line = line; + + ut_ad(rw_lock_validate(lock)); + + return(TRUE); + } else { + /* fail (lock writer) */ + __sync_fetch_and_add(&(lock->writer), + RW_LOCK_NOT_LOCKED); + /* x-unlock */ + __sync_fetch_and_add(&(lock->lock_word), + RW_LOCK_BIAS); + } + } else { + /* fail (x-lock) */ + __sync_fetch_and_add(&(lock->lock_word),RW_LOCK_BIAS); + } + } + + if (lock->pass == 0 + && os_thread_eq(lock->writer_thread, curr_thread) + && rw_lock_get_writer(lock) == RW_LOCK_EX) { + goto relock; + } + + ut_ad(rw_lock_validate(lock)); + + return(FALSE); +#else mutex_enter(rw_lock_get_mutex(lock)); if (UNIV_UNLIKELY(rw_lock_get_reader_count(lock) != 0)) { @@ -339,6 +479,7 @@ ut_ad(rw_lock_validate(lock)); return(success); +#endif } /********************************************************************** @@ -354,16 +495,32 @@ #endif ) { +#ifndef HAVE_ATOMIC_BUILTINS mutex_t* mutex = &(lock->mutex); +#endif ibool sg = FALSE; +#ifdef HAVE_ATOMIC_BUILTINS + ibool last = FALSE; +#endif +#ifndef HAVE_ATOMIC_BUILTINS /* Acquire the mutex protecting the rw-lock fields */ mutex_enter(mutex); +#endif /* Reset the shared lock by decrementing the reader count */ ut_a(lock->reader_count > 0); +#ifdef HAVE_ATOMIC_BUILTINS + /* unlock lock_word */ + __sync_fetch_and_add(&(lock->lock_word),1); + + if(__sync_sub_and_fetch(&(lock->reader_count),1) == 0) { + last = TRUE; + } +#else lock->reader_count--; +#endif #ifdef UNIV_SYNC_DEBUG rw_lock_remove_debug_info(lock, pass, RW_LOCK_SHARED); @@ -372,20 +529,34 @@ /* If there may be waiters and this was the last s-lock, signal the object */ +#ifdef HAVE_ATOMIC_BUILTINS + if (UNIV_UNLIKELY(last && lock->x_waiters)) { +#else if (UNIV_UNLIKELY(lock->waiters) && lock->reader_count == 0) { +#endif sg = TRUE; +#ifdef HAVE_ATOMIC_BUILTINS + rw_lock_set_x_waiters(lock, 0); +#else rw_lock_set_waiters(lock, 0); +#endif } +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(mutex); +#endif if (UNIV_UNLIKELY(sg)) { #ifdef __WIN__ os_event_set(lock->wait_ex_event); #endif +#ifdef HAVE_ATOMIC_BUILTINS + os_event_set(lock->x_event); +#else os_event_set(lock->event); +#endif sync_array_object_signalled(sync_primary_wait_array); } @@ -409,13 +580,22 @@ ut_ad(lock->reader_count > 0); +#ifdef HAVE_ATOMIC_BUILTINS + __sync_sub_and_fetch(&(lock->reader_count),1); +#else lock->reader_count--; +#endif #ifdef UNIV_SYNC_DEBUG rw_lock_remove_debug_info(lock, 0, RW_LOCK_SHARED); #endif +#ifdef HAVE_ATOMIC_BUILTINS + ut_ad(!lock->s_waiters); + ut_ad(!lock->x_waiters); +#else ut_ad(!lock->waiters); +#endif ut_ad(rw_lock_validate(lock)); #ifdef UNIV_SYNC_PERF_STAT rw_s_exit_count++; @@ -435,36 +615,89 @@ #endif ) { +#ifdef HAVE_ATOMIC_BUILTINS + ibool s_sg = FALSE; + ibool x_sg = FALSE; + ibool last = FALSE; +#else ibool sg = FALSE; /* Acquire the mutex protecting the rw-lock fields */ mutex_enter(&(lock->mutex)); +#endif /* Reset the exclusive lock if this thread no longer has an x-mode lock */ ut_ad(lock->writer_count > 0); +#ifdef HAVE_ATOMIC_BUILTINS + if(__sync_sub_and_fetch(&(lock->writer_count),1) == 0) { + last = TRUE; + } + + if (last) { + /* unlock lock_word */ + __sync_fetch_and_add(&(lock->lock_word),RW_LOCK_BIAS); + + /* FIXME: It is a value of bad manners for pthread. + But we shouldn't keep an ID of not-owner. */ + lock->writer_thread = -1; + + /* RW_LOCK_EX -> RW_LOCK_NOT_LOCKED */ + __sync_fetch_and_sub(&(lock->writer), + (RW_LOCK_EX - RW_LOCK_NOT_LOCKED)); + } +#else lock->writer_count--; if (lock->writer_count == 0) { rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); } +#endif #ifdef UNIV_SYNC_DEBUG rw_lock_remove_debug_info(lock, pass, RW_LOCK_EX); #endif /* If there may be waiters, signal the lock */ +#ifdef HAVE_ATOMIC_BUILTINS + if (last) { + if(lock->s_waiters){ + s_sg = TRUE; + rw_lock_set_s_waiters(lock, 0); + } + if(lock->x_waiters){ + x_sg = TRUE; + rw_lock_set_x_waiters(lock, 0); + } + } +#else if (UNIV_UNLIKELY(lock->waiters) && lock->writer_count == 0) { sg = TRUE; rw_lock_set_waiters(lock, 0); } +#endif +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(&(lock->mutex)); +#endif +#ifdef HAVE_ATOMIC_BUILTINS + if (UNIV_UNLIKELY(s_sg)) { + os_event_set(lock->s_event); + sync_array_object_signalled(sync_primary_wait_array); + } + if (UNIV_UNLIKELY(x_sg)) { +#ifdef __WIN__ + os_event_set(lock->wait_ex_event); +#endif + os_event_set(lock->x_event); + sync_array_object_signalled(sync_primary_wait_array); + } +#else if (UNIV_UNLIKELY(sg)) { #ifdef __WIN__ os_event_set(lock->wait_ex_event); @@ -472,6 +705,7 @@ os_event_set(lock->event); sync_array_object_signalled(sync_primary_wait_array); } +#endif ut_ad(rw_lock_validate(lock)); @@ -494,9 +728,13 @@ ut_ad(lock->writer_count > 0); +#ifdef HAVE_ATOMIC_BUILTINS + if(__sync_sub_and_fetch(&(lock->writer_count),1) == 0) { +#else lock->writer_count--; if (lock->writer_count == 0) { +#endif rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); } @@ -504,7 +742,12 @@ rw_lock_remove_debug_info(lock, 0, RW_LOCK_EX); #endif +#ifdef HAVE_ATOMIC_BUILTINS + ut_ad(!lock->s_waiters); + ut_ad(!lock->x_waiters); +#else ut_ad(!lock->waiters); +#endif ut_ad(rw_lock_validate(lock)); #ifdef UNIV_SYNC_PERF_STAT diff -ru mysql-5.0.54_orig/innobase/sync/sync0arr.c mysql-5.0.54/innobase/sync/sync0arr.c --- mysql-5.0.54_orig/innobase/sync/sync0arr.c 2007-11-21 02:38:06.000000000 +0900 +++ mysql-5.0.54/innobase/sync/sync0arr.c 2008-01-26 17:41:20.000000000 +0900 @@ -314,9 +314,17 @@ return(os_event_reset( ((rw_lock_t *) object)->wait_ex_event)); #endif +#ifdef HAVE_ATOMIC_BUILTINS + } else if (type == RW_LOCK_SHARED) { + return(os_event_reset(((rw_lock_t *) object)->s_event)); + } else { /* RW_LOCK_EX */ + return(os_event_reset(((rw_lock_t *) object)->x_event)); + } +#else } else { return(os_event_reset(((rw_lock_t *) object)->event)); } +#endif } /********************************************************************** @@ -422,9 +430,17 @@ } else if (cell->request_type == RW_LOCK_WAIT_EX) { event = ((rw_lock_t*) cell->wait_object)->wait_ex_event; #endif +#ifdef HAVE_ATOMIC_BUILTINS + } else if (cell->request_type == RW_LOCK_SHARED) { + event = ((rw_lock_t*) cell->wait_object)->s_event; + } else { + event = ((rw_lock_t*) cell->wait_object)->x_event; + } +#else } else { event = ((rw_lock_t*) cell->wait_object)->event; } +#endif cell->waiting = TRUE; @@ -464,6 +480,7 @@ mutex_t* mutex; rw_lock_t* rwlock; ulint type; + ulint writer; type = cell->request_type; @@ -505,21 +522,31 @@ " RW-latch at %p created in file %s line %lu\n", rwlock, rwlock->cfile_name, (ulong) rwlock->cline); - if (rwlock->writer != RW_LOCK_NOT_LOCKED) { + writer = rw_lock_get_writer(rwlock); + if (writer != RW_LOCK_NOT_LOCKED) { fprintf(file, "a writer (thread id %lu) has reserved it in mode %s", (ulong) os_thread_pf(rwlock->writer_thread), - rwlock->writer == RW_LOCK_EX + writer == RW_LOCK_EX ? " exclusive\n" : " wait exclusive\n"); } fprintf(file, +#ifdef HAVE_ATOMIC_BUILTINS + "number of readers %lu, s_waiters flag %lu, x_waiters flag %lu\n" +#else "number of readers %lu, waiters flag %lu\n" +#endif "Last time read locked in file %s line %lu\n" "Last time write locked in file %s line %lu\n", (ulong) rwlock->reader_count, +#ifdef HAVE_ATOMIC_BUILTINS + (ulong) rwlock->s_waiters, + (ulong) rwlock->x_waiters, +#else (ulong) rwlock->waiters, +#endif rwlock->last_s_file_name, (ulong) rwlock->last_s_line, rwlock->last_x_file_name, @@ -839,11 +866,15 @@ /*========================*/ sync_array_t* arr) /* in: wait array */ { +#ifdef HAVE_ATOMIC_BUILTINS + __sync_fetch_and_add(&(arr->sg_count),1); +#else sync_array_enter(arr); arr->sg_count++; sync_array_exit(arr); +#endif } /************************************************************************** @@ -888,12 +919,27 @@ lock = cell->wait_object; os_event_set(lock->wait_ex_event); #endif +#ifdef HAVE_ATOMIC_BUILTINS + } else if (cell->request_type + == RW_LOCK_SHARED) { + rw_lock_t* lock; + + lock = cell->wait_object; + os_event_set(lock->s_event); + } else { + rw_lock_t* lock; + + lock = cell->wait_object; + os_event_set(lock->x_event); + } +#else } else { rw_lock_t* lock; lock = cell->wait_object; os_event_set(lock->event); } +#endif } } diff -ru mysql-5.0.54_orig/innobase/sync/sync0rw.c mysql-5.0.54/innobase/sync/sync0rw.c --- mysql-5.0.54_orig/innobase/sync/sync0rw.c 2007-11-21 02:38:06.000000000 +0900 +++ mysql-5.0.54/innobase/sync/sync0rw.c 2008-01-26 21:39:02.000000000 +0900 @@ -99,6 +99,7 @@ object is created, then the following call initializes the sync system. */ +#ifndef HAVE_ATOMIC_BUILTINS mutex_create(rw_lock_get_mutex(lock)); mutex_set_level(rw_lock_get_mutex(lock), SYNC_NO_ORDER_CHECK); @@ -108,8 +109,15 @@ lock->mutex.cmutex_name = cmutex_name; lock->mutex.mutex_type = 1; #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */ +#endif /* !HAVE_ATOMIC_BUILTINS */ +#ifdef HAVE_ATOMIC_BUILTINS + lock->lock_word = RW_LOCK_BIAS; + rw_lock_set_s_waiters(lock, 0); + rw_lock_set_x_waiters(lock, 0); +#else rw_lock_set_waiters(lock, 0); +#endif rw_lock_set_writer(lock, RW_LOCK_NOT_LOCKED); lock->writer_count = 0; rw_lock_set_reader_count(lock, 0); @@ -130,7 +138,12 @@ lock->last_x_file_name = "not yet reserved"; lock->last_s_line = 0; lock->last_x_line = 0; +#ifdef HAVE_ATOMIC_BUILTINS + lock->s_event = os_event_create(NULL); + lock->x_event = os_event_create(NULL); +#else lock->event = os_event_create(NULL); +#endif #ifdef __WIN__ lock->wait_ex_event = os_event_create(NULL); @@ -162,15 +175,27 @@ ut_a(rw_lock_validate(lock)); #endif /* UNIV_DEBUG */ ut_a(rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED); +#ifdef HAVE_ATOMIC_BUILTINS + ut_a(rw_lock_get_s_waiters(lock) == 0); + ut_a(rw_lock_get_x_waiters(lock) == 0); +#else ut_a(rw_lock_get_waiters(lock) == 0); +#endif ut_a(rw_lock_get_reader_count(lock) == 0); lock->magic_n = 0; +#ifndef HAVE_ATOMIC_BUILTINS mutex_free(rw_lock_get_mutex(lock)); +#endif mutex_enter(&rw_lock_list_mutex); +#ifdef HAVE_ATOMIC_BUILTINS + os_event_free(lock->s_event); + os_event_free(lock->x_event); +#else os_event_free(lock->event); +#endif #ifdef __WIN__ os_event_free(lock->wait_ex_event); @@ -191,6 +216,7 @@ /********************************************************************** Checks that the rw-lock has been initialized and that there are no simultaneous shared and exclusive locks. */ +/* If HAVE_ATOMIC_BUILTINS, we should use this function statically. */ ibool rw_lock_validate( @@ -199,7 +225,9 @@ { ut_a(lock); +#ifndef HAVE_ATOMIC_BUILTINS mutex_enter(rw_lock_get_mutex(lock)); +#endif ut_a(lock->magic_n == RW_LOCK_MAGIC_N); ut_a((rw_lock_get_reader_count(lock) == 0) @@ -207,11 +235,20 @@ ut_a((rw_lock_get_writer(lock) == RW_LOCK_EX) || (rw_lock_get_writer(lock) == RW_LOCK_WAIT_EX) || (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED)); +#ifdef HAVE_ATOMIC_BUILTINS + ut_a((rw_lock_get_s_waiters(lock) == 0) + || (rw_lock_get_s_waiters(lock) == 1)); + ut_a((rw_lock_get_x_waiters(lock) == 0) + || (rw_lock_get_x_waiters(lock) == 1)); +#else ut_a((rw_lock_get_waiters(lock) == 0) || (rw_lock_get_waiters(lock) == 1)); +#endif ut_a((lock->writer != RW_LOCK_EX) || (lock->writer_count > 0)); +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(rw_lock_get_mutex(lock)); +#endif return(TRUE); } @@ -237,13 +274,14 @@ ut_ad(rw_lock_validate(lock)); lock_loop: + i = 0; +spin_loop: rw_s_spin_wait_count++; /* Spin waiting for the writer field to become free */ - i = 0; - while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED - && i < SYNC_SPIN_ROUNDS) { + while (i < SYNC_SPIN_ROUNDS + && rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); } @@ -262,15 +300,27 @@ lock->cfile_name, (ulong) lock->cline, (ulong) i); } +#ifndef HAVE_ATOMIC_BUILTINS mutex_enter(rw_lock_get_mutex(lock)); +#endif /* We try once again to obtain the lock */ if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) { +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(rw_lock_get_mutex(lock)); +#endif return; /* Success */ } else { +#ifdef HAVE_ATOMIC_BUILTINS + /* like sync0sync.c doing */ + i++; + + if (i < SYNC_SPIN_ROUNDS) { + goto spin_loop; + } +#endif /* If we get here, locking did not succeed, we may suspend the thread to wait in the wait array */ @@ -281,9 +331,21 @@ file_name, line, &index); +#ifdef HAVE_ATOMIC_BUILTINS + rw_lock_set_s_waiters(lock, 1); + + /* like sync0sync.c doing */ + for (i = 0; i < 4; i++) { + if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) { + sync_array_free_cell(sync_primary_wait_array, index); + return; /* Success */ + } + } +#else rw_lock_set_waiters(lock, 1); mutex_exit(rw_lock_get_mutex(lock)); +#endif if (srv_print_latch_waits) { fprintf(stderr, @@ -318,13 +380,17 @@ { ut_ad(rw_lock_is_locked(lock, RW_LOCK_EX)); +#ifndef HAVE_ATOMIC_BUILTINS mutex_enter(&(lock->mutex)); +#endif lock->writer_thread = os_thread_get_curr_id(); lock->pass = 0; +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(&(lock->mutex)); +#endif } /********************************************************************** @@ -342,6 +408,100 @@ const char* file_name,/* in: file name where lock requested */ ulint line) /* in: line where requested */ { +#ifdef HAVE_ATOMIC_BUILTINS + os_thread_id_t curr_thread = os_thread_get_curr_id(); + + if (rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED) { + /* try to lock writer */ + if(__sync_sub_and_fetch(&(lock->writer), + RW_LOCK_NOT_LOCKED) == 0) { + /* success */ + /* obtain RW_LOCK_WAIT_EX right */ + lock->writer_thread = curr_thread; + lock->pass = pass; + lock->writer_is_wait_ex = TRUE; + /* 0 -> RW_LOCK_WAIT_EX */ + __sync_fetch_and_add(&(lock->writer), + RW_LOCK_WAIT_EX); +#ifdef UNIV_SYNC_DEBUG + rw_lock_add_debug_info(lock, pass, RW_LOCK_WAIT_EX, + file_name, line); +#endif + } else { + /* fail */ + __sync_fetch_and_add(&(lock->writer), + RW_LOCK_NOT_LOCKED); + } + } + + if (!os_thread_eq(lock->writer_thread, curr_thread)) { + return(RW_LOCK_NOT_LOCKED); + } + + switch(rw_lock_get_writer(lock)) { + case RW_LOCK_WAIT_EX: + /* have right to try x-lock */ + if (lock->lock_word == RW_LOCK_BIAS) { + /* try x-lock */ + if(__sync_sub_and_fetch(&(lock->lock_word), + RW_LOCK_BIAS) == 0) { + /* success */ + __sync_fetch_and_sub(&(lock->writer), + RW_LOCK_WAIT_EX); + lock->pass = pass; + lock->writer_is_wait_ex = FALSE; + __sync_fetch_and_add(&(lock->writer), + RW_LOCK_EX); + __sync_fetch_and_add(&(lock->writer_count),1); + +#ifdef UNIV_SYNC_DEBUG + rw_lock_remove_debug_info(lock, pass, RW_LOCK_WAIT_EX); + rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, + file_name, line); +#endif + + lock->last_x_file_name = file_name; + lock->last_x_line = line; + + /* Locking succeeded, we may return */ + return(RW_LOCK_EX); + } else { + /* fail */ + __sync_fetch_and_add(&(lock->lock_word), + RW_LOCK_BIAS); + } + } + /* There are readers, we have to wait */ + return(RW_LOCK_WAIT_EX); + + break; + + case RW_LOCK_EX: + /* already have x-lock */ + if ((lock->pass == 0)&&(pass == 0)) { + __sync_fetch_and_add(&(lock->writer_count),1); + +#ifdef UNIV_SYNC_DEBUG + rw_lock_add_debug_info(lock, pass, RW_LOCK_EX, file_name, + line); +#endif + + lock->last_x_file_name = file_name; + lock->last_x_line = line; + + /* Locking succeeded, we may return */ + return(RW_LOCK_EX); + } + + return(RW_LOCK_NOT_LOCKED); + + break; + + default: /* ??? */ + return(RW_LOCK_NOT_LOCKED); + } +#else /* HAVE_ATOMIC_BUILTINS */ + #ifdef UNIV_SYNC_DEBUG ut_ad(mutex_own(rw_lock_get_mutex(lock))); #endif /* UNIV_SYNC_DEBUG */ @@ -423,6 +583,7 @@ /* Locking succeeded, we may return */ return(RW_LOCK_EX); } +#endif /* HAVE_ATOMIC_BUILTINS */ /* Locking did not succeed */ return(RW_LOCK_NOT_LOCKED); @@ -448,19 +609,33 @@ ulint line) /* in: line where requested */ { ulint index; /* index of the reserved wait cell */ - ulint state; /* lock state acquired */ + ulint state = RW_LOCK_NOT_LOCKED; /* lock state acquired */ +#ifdef HAVE_ATOMIC_BUILTINS + ulint prev_state = RW_LOCK_NOT_LOCKED; +#endif ulint i; /* spin round count */ ut_ad(rw_lock_validate(lock)); lock_loop: + i = 0; + +#ifdef HAVE_ATOMIC_BUILTINS + prev_state = state; +#else /* Acquire the mutex protecting the rw-lock fields */ mutex_enter_fast(&(lock->mutex)); +#endif state = rw_lock_x_lock_low(lock, pass, file_name, line); +#ifdef HAVE_ATOMIC_BUILTINS + if (state != prev_state) i=0; /* if progress, reset counter. */ +#else mutex_exit(&(lock->mutex)); +#endif +spin_loop: if (state == RW_LOCK_EX) { return; /* Locking succeeded */ @@ -468,10 +643,9 @@ } else if (state == RW_LOCK_NOT_LOCKED) { /* Spin waiting for the writer field to become free */ - i = 0; - while (rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED - && i < SYNC_SPIN_ROUNDS) { + while (i < SYNC_SPIN_ROUNDS + && rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); @@ -485,9 +659,12 @@ } else if (state == RW_LOCK_WAIT_EX) { /* Spin waiting for the reader count field to become zero */ - i = 0; +#ifdef HAVE_ATOMIC_BUILTINS + while (lock->lock_word != RW_LOCK_BIAS +#else while (rw_lock_get_reader_count(lock) != 0 +#endif && i < SYNC_SPIN_ROUNDS) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, @@ -500,7 +677,6 @@ os_thread_yield(); } } else { - i = 0; /* Eliminate a compiler warning */ ut_error; } @@ -516,16 +692,35 @@ /* We try once again to obtain the lock. Acquire the mutex protecting the rw-lock fields */ +#ifdef HAVE_ATOMIC_BUILTINS + prev_state = state; +#else mutex_enter(rw_lock_get_mutex(lock)); +#endif state = rw_lock_x_lock_low(lock, pass, file_name, line); +#ifdef HAVE_ATOMIC_BUILTINS + if (state != prev_state) i=0; /* if progress, reset counter. */ +#endif + if (state == RW_LOCK_EX) { +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(rw_lock_get_mutex(lock)); +#endif return; /* Locking succeeded */ } +#ifdef HAVE_ATOMIC_BUILTINS + /* like sync0sync.c doing */ + i++; + + if (i < SYNC_SPIN_ROUNDS) { + goto spin_loop; + } +#endif + rw_x_system_call_count++; sync_array_reserve_cell(sync_primary_wait_array, @@ -541,9 +736,28 @@ file_name, line, &index); +#ifdef HAVE_ATOMIC_BUILTINS + rw_lock_set_x_waiters(lock, 1); + + /* like sync0sync.c doing */ + for (i = 0; i < 4; i++) { + prev_state = state; + state = rw_lock_x_lock_low(lock, pass, file_name, line); + if (state == RW_LOCK_EX) { + sync_array_free_cell(sync_primary_wait_array, index); + return; /* Locking succeeded */ + } + if (state != prev_state) { + /* retry! */ + sync_array_free_cell(sync_primary_wait_array, index); + goto lock_loop; + } + } +#else rw_lock_set_waiters(lock, 1); mutex_exit(rw_lock_get_mutex(lock)); +#endif if (srv_print_latch_waits) { fprintf(stderr, @@ -718,7 +932,9 @@ ut_ad(lock); ut_ad(rw_lock_validate(lock)); +#ifndef HAVE_ATOMIC_BUILTINS mutex_enter(&(lock->mutex)); +#endif info = UT_LIST_GET_FIRST(lock->debug_list); @@ -728,7 +944,9 @@ && (info->pass == 0) && (info->lock_type == lock_type)) { +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(&(lock->mutex)); +#endif /* Found! */ return(TRUE); @@ -736,7 +954,9 @@ info = UT_LIST_GET_NEXT(list, info); } +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(&(lock->mutex)); +#endif return(FALSE); } @@ -758,21 +978,25 @@ ut_ad(lock); ut_ad(rw_lock_validate(lock)); +#ifndef HAVE_ATOMIC_BUILTINS mutex_enter(&(lock->mutex)); +#endif if (lock_type == RW_LOCK_SHARED) { if (lock->reader_count > 0) { ret = TRUE; } } else if (lock_type == RW_LOCK_EX) { - if (lock->writer == RW_LOCK_EX) { + if (rw_lock_get_writer(lock) == RW_LOCK_EX) { ret = TRUE; } } else { ut_error; } +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(&(lock->mutex)); +#endif return(ret); } @@ -801,16 +1025,31 @@ count++; +#ifndef HAVE_ATOMIC_BUILTINS mutex_enter(&(lock->mutex)); +#endif if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) || (rw_lock_get_reader_count(lock) != 0) +#ifdef HAVE_ATOMIC_BUILTINS + || (rw_lock_get_s_waiters(lock) != 0) + || (rw_lock_get_x_waiters(lock) != 0)) { +#else || (rw_lock_get_waiters(lock) != 0)) { +#endif fprintf(stderr, "RW-LOCK: %p ", lock); +#ifdef HAVE_ATOMIC_BUILTINS + if (rw_lock_get_s_waiters(lock)) { + fputs(" s_waiters for the lock exist,", stderr); + } + if (rw_lock_get_x_waiters(lock)) { + fputs(" x_waiters for the lock exist\n", stderr); +#else if (rw_lock_get_waiters(lock)) { fputs(" Waiters for the lock exist\n", stderr); +#endif } else { putc('\n', stderr); } @@ -822,7 +1061,9 @@ } } +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(&(lock->mutex)); +#endif lock = UT_LIST_GET_NEXT(list, lock); } @@ -847,10 +1088,23 @@ if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) || (rw_lock_get_reader_count(lock) != 0) +#ifdef HAVE_ATOMIC_BUILTINS + || (rw_lock_get_s_waiters(lock) != 0) + || (rw_lock_get_x_waiters(lock) != 0)) { +#else || (rw_lock_get_waiters(lock) != 0)) { +#endif +#ifdef HAVE_ATOMIC_BUILTINS + if (rw_lock_get_s_waiters(lock)) { + fputs(" s_waiters for the lock exist,", stderr); + } + if (rw_lock_get_x_waiters(lock)) { + fputs(" x_waiters for the lock exist\n", stderr); +#else if (rw_lock_get_waiters(lock)) { fputs(" Waiters for the lock exist\n", stderr); +#endif } else { putc('\n', stderr); } @@ -909,14 +1163,18 @@ lock = UT_LIST_GET_FIRST(rw_lock_list); while (lock != NULL) { +#ifndef HAVE_ATOMIC_BUILTINS mutex_enter(rw_lock_get_mutex(lock)); +#endif if ((rw_lock_get_writer(lock) != RW_LOCK_NOT_LOCKED) || (rw_lock_get_reader_count(lock) != 0)) { count++; } +#ifndef HAVE_ATOMIC_BUILTINS mutex_exit(rw_lock_get_mutex(lock)); +#endif lock = UT_LIST_GET_NEXT(list, lock); }