=== modified file 'storage/innobase/CMakeLists.txt' --- storage/innobase/CMakeLists.txt revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ storage/innobase/CMakeLists.txt 2014-05-12 02:13:32 +0000 @@ -117,6 +117,25 @@ IF(NOT CMAKE_CROSSCOMPILING) }" HAVE_IB_GCC_ATOMIC_BUILTINS_64 ) + CHECK_C_SOURCE_RUNS( + "#include + int main() + { + __sync_synchronize(); + return(0); + }" + HAVE_IB_GCC_SYNC_SYNCHRONISE + ) + CHECK_C_SOURCE_RUNS( + "#include + int main() + { + __atomic_thread_fence(__ATOMIC_ACQUIRE); + __atomic_thread_fence(__ATOMIC_RELEASE); + return(0); + }" + HAVE_IB_GCC_ATOMIC_THREAD_FENCE + ) ENDIF() IF(HAVE_IB_GCC_ATOMIC_BUILTINS) @@ -127,6 +146,14 @@ IF(HAVE_IB_GCC_ATOMIC_BUILTINS_64) ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_BUILTINS_64=1) ENDIF() +IF(HAVE_IB_GCC_SYNC_SYNCHRONISE) + ADD_DEFINITIONS(-DHAVE_IB_GCC_SYNC_SYNCHRONISE=1) +ENDIF() + +IF(HAVE_IB_GCC_ATOMIC_THREAD_FENCE) + ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_THREAD_FENCE=1) +ENDIF() + # either define HAVE_IB_ATOMIC_PTHREAD_T_GCC or not IF(NOT CMAKE_CROSSCOMPILING) CHECK_C_SOURCE_RUNS( @@ -208,10 +235,21 @@ IF(CMAKE_SYSTEM_NAME STREQUAL "SunOS") return(0); } " HAVE_IB_ATOMIC_PTHREAD_T_SOLARIS) + CHECK_C_SOURCE_COMPILES( + "#include + int main() { + __machine_r_barrier(); + __machine_w_barrier(); + return(0); + }" + HAVE_IB_MACHINE_BARRIER_SOLARIS) ENDIF() IF(HAVE_IB_ATOMIC_PTHREAD_T_SOLARIS) ADD_DEFINITIONS(-DHAVE_IB_ATOMIC_PTHREAD_T_SOLARIS=1) ENDIF() + IF(HAVE_IB_MACHINE_BARRIER_SOLARIS) + ADD_DEFINITIONS(-DHAVE_IB_MACHINE_BARRIER_SOLARIS=1) + ENDIF() ENDIF() === modified file 'storage/innobase/include/os0sync.h' --- storage/innobase/include/os0sync.h revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ storage/innobase/include/os0sync.h 2014-05-12 02:18:21 +0000 @@ -685,6 +685,44 @@ for synchronization */ os_decrement_counter_by_amount(mutex, counter, 1);\ } while (0); +/** barrier definitions for memory ordering */ +#ifdef HAVE_IB_GCC_ATOMIC_THREAD_FENCE +# define HAVE_MEMORY_BARRIER +# define os_rmb __atomic_thread_fence(__ATOMIC_ACQUIRE) +# define os_wmb __atomic_thread_fence(__ATOMIC_RELEASE) +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "GCC builtin __atomic_thread_fence() is used for memory barrier" + +#elif defined(HAVE_IB_GCC_SYNC_SYNCHRONISE) +# define HAVE_MEMORY_BARRIER +# define os_rmb __sync_synchronize() +# define os_wmb __sync_synchronize() +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "GCC builtin __sync_synchronize() is used for memory barrier" + +#elif defined(HAVE_IB_MACHINE_BARRIER_SOLARIS) +# define HAVE_MEMORY_BARRIER +# include +# define os_rmb __machine_r_barrier() +# define os_wmb __machine_w_barrier() +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "Soralis memory ordering functions are used for memory barrier" + +#elif defined(HAVE_WINDOWS_MM_FENCE) +# define HAVE_MEMORY_BARRIER +# include +# define os_rmb _mm_lfence() +# define os_wmb _mm_sfence() +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "_mm_lfence() and _mm_sfence() are used for memory barrier" + +#else +# define os_rmb +# define os_wmb +# define IB_MEMORY_BARRIER_STARTUP_MSG \ + "Memory barrier is not used" +#endif + #ifndef UNIV_NONINL #include "os0sync.ic" #endif === modified file 'storage/innobase/include/sync0rw.ic' --- storage/innobase/include/sync0rw.ic revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ storage/innobase/include/sync0rw.ic 2014-05-12 03:48:49 +0000 @@ -93,6 +93,7 @@ rw_lock_set_waiter_flag( (void) os_compare_and_swap_ulint(&lock->waiters, 0, 1); #else /* INNODB_RW_LOCKS_USE_ATOMICS */ lock->waiters = 1; + os_wmb; #endif /* INNODB_RW_LOCKS_USE_ATOMICS */ } @@ -110,6 +111,7 @@ rw_lock_reset_waiter_flag( (void) os_compare_and_swap_ulint(&lock->waiters, 1, 0); #else /* INNODB_RW_LOCKS_USE_ATOMICS */ lock->waiters = 0; + os_wmb; #endif /* INNODB_RW_LOCKS_USE_ATOMICS */ } @@ -199,7 +201,10 @@ rw_lock_lock_word_decr( ulint amount) /*!< in: amount to decrement */ { #ifdef INNODB_RW_LOCKS_USE_ATOMICS - lint local_lock_word = lock->lock_word; + lint local_lock_word; + + os_rmb; + local_lock_word = lock->lock_word; while (local_lock_word > 0) { if (os_compare_and_swap_lint(&lock->lock_word, local_lock_word, === modified file 'storage/innobase/include/sync0sync.ic' --- storage/innobase/include/sync0sync.ic revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ storage/innobase/include/sync0sync.ic 2014-05-12 03:21:51 +0000 @@ -92,6 +92,7 @@ ib_mutex_test_and_set( ut_a(mutex->lock_word == 0); mutex->lock_word = 1; + os_wmb; } return((byte) ret); === modified file 'storage/innobase/srv/srv0start.cc' --- storage/innobase/srv/srv0start.cc revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ storage/innobase/srv/srv0start.cc 2014-05-12 02:22:14 +0000 @@ -1647,6 +1647,18 @@ innobase_start_or_create_for_mysql(void) "" IB_ATOMICS_STARTUP_MSG ""); ib_logf(IB_LOG_LEVEL_INFO, + "" IB_MEMORY_BARRIER_STARTUP_MSG ""); + +#ifndef HAVE_MEMORY_BARRIER +#if defined __i386__ || defined __x86_64__ || defined _M_IX86 || defined _M_X64 +#else + ib_logf(IB_LOG_LEVEL_WARN, + "Built without memory barrier might cause problem" + " for this architecture."); +#endif /* IA32 or AMD64 */ +#endif /* HAVE_MEMORY_BARRIER */ + + ib_logf(IB_LOG_LEVEL_INFO, "Compressed tables use zlib " ZLIB_VERSION #ifdef UNIV_ZIP_DEBUG " with validation" === modified file 'storage/innobase/sync/sync0arr.cc' --- storage/innobase/sync/sync0arr.cc revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ storage/innobase/sync/sync0arr.cc 2014-05-12 03:25:10 +0000 @@ -740,6 +740,7 @@ sync_arr_cell_can_wake_up( mutex = static_cast(cell->wait_object); + os_rmb; if (mutex_get_lock_word(mutex) == 0) { return(TRUE); @@ -749,6 +750,7 @@ sync_arr_cell_can_wake_up( lock = static_cast(cell->wait_object); + os_rmb; if (lock->lock_word > 0) { /* Either unlocked or only read locked. */ @@ -760,6 +762,7 @@ sync_arr_cell_can_wake_up( lock = static_cast(cell->wait_object); /* lock_word == 0 means all readers have left */ + os_rmb; if (lock->lock_word == 0) { return(TRUE); @@ -768,6 +771,7 @@ sync_arr_cell_can_wake_up( lock = static_cast(cell->wait_object); /* lock_word > 0 means no writer or reserved writer */ + os_rmb; if (lock->lock_word > 0) { return(TRUE); === modified file 'storage/innobase/sync/sync0rw.cc' --- storage/innobase/sync/sync0rw.cc revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ storage/innobase/sync/sync0rw.cc 2014-05-12 04:16:49 +0000 @@ -381,15 +381,21 @@ rw_lock_s_lock_spin( lock_loop: /* Spin waiting for the writer field to become free */ + os_rmb; while (i < SYNC_SPIN_ROUNDS && lock->lock_word <= 0) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); } +#ifdef HAVE_MEMORY_BARRIER + i += 2; + os_rmb; +#else i++; +#endif /* HAVE_MEMORY_BARRIER */ } - if (i == SYNC_SPIN_ROUNDS) { + if (i >= SYNC_SPIN_ROUNDS) { os_thread_yield(); } @@ -476,6 +482,7 @@ rw_lock_x_lock_wait( counter_index = (size_t) os_thread_get_curr_id(); + os_rmb; ut_ad(lock->lock_word <= 0); while (lock->lock_word < 0) { @@ -483,7 +490,12 @@ rw_lock_x_lock_wait( ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); } if(i < SYNC_SPIN_ROUNDS) { +#ifdef HAVE_MEMORY_BARRIER + i += 2; + os_rmb; +#else i++; +#endif /* HAVE_MEMORY_BARRIER */ continue; } @@ -560,6 +572,10 @@ rw_lock_x_lock_low( } else { os_thread_id_t thread_id = os_thread_get_curr_id(); + if (!pass) { + os_rmb; + } + /* Decrement failed: relock or failed lock */ if (!pass && lock->recursive && os_thread_eq(lock->writer_thread, thread_id)) { @@ -638,6 +654,7 @@ lock_loop: } /* Spin waiting for the lock_word to become free */ + os_rmb; while (i < SYNC_SPIN_ROUNDS && lock->lock_word <= 0) { if (srv_spin_wait_delay) { @@ -645,9 +662,14 @@ lock_loop: srv_spin_wait_delay)); } +#ifdef HAVE_MEMORY_BARRIER + i += 2; + os_rmb; +#else i++; +#endif /* HAVE_MEMORY_BARRIER */ } - if (i == SYNC_SPIN_ROUNDS) { + if (i >= SYNC_SPIN_ROUNDS) { os_thread_yield(); } else { goto lock_loop; === modified file 'storage/innobase/sync/sync0sync.cc' --- storage/innobase/sync/sync0sync.cc revid:venkatesh.duggirala@oracle.com-20140509042350-ni6xx9khsej94hl9 +++ storage/innobase/sync/sync0sync.cc 2014-05-12 03:39:01 +0000 @@ -458,6 +458,7 @@ mutex_set_waiters( *ptr = n; /* Here we assume that the write of a single word in memory is atomic */ + os_wmb; } /******************************************************************//** @@ -499,16 +500,21 @@ mutex_loop: a memory word. */ spin_loop: - + os_rmb; while (mutex_get_lock_word(mutex) != 0 && i < SYNC_SPIN_ROUNDS) { if (srv_spin_wait_delay) { ut_delay(ut_rnd_interval(0, srv_spin_wait_delay)); } +#ifdef HAVE_MEMORY_BARRIER + i += 2; + os_rmb; +#else i++; +#endif /* HAVE_MEMORY_BARRIER */ } - if (i == SYNC_SPIN_ROUNDS) { + if (i >= SYNC_SPIN_ROUNDS) { os_thread_yield(); }