Merge pull request #1012 from dklempner/nobarrier

Add a no_barrier_load atomic operation.

include/grpc/support/atm_gcc_atomic.h

@@ -43,6 +43,7 @@ typedef gpr_intptr gpr_atm;
 #define gpr_atm_full_barrier() (__atomic_thread_fence(__ATOMIC_SEQ_CST))
 
 #define gpr_atm_acq_load(p) (__atomic_load_n((p), __ATOMIC_ACQUIRE))
+#define gpr_atm_no_barrier_load(p) (__atomic_load_n((p), __ATOMIC_RELAXED))
 #define gpr_atm_rel_store(p, value) \
   (__atomic_store_n((p), (gpr_intptr)(value), __ATOMIC_RELEASE))
 

include/grpc/support/atm_gcc_sync.h

@@ -40,9 +40,11 @@
 
 typedef gpr_intptr gpr_atm;
 
+#define GPR_ATM_COMPILE_BARRIER_() __asm__ __volatile__("" : : : "memory")
+
 #if defined(__i386) || defined(__x86_64__)
 /* All loads are acquire loads and all stores are release stores.  */
-#define GPR_ATM_LS_BARRIER_() __asm__ __volatile__("" : : : "memory")
+#define GPR_ATM_LS_BARRIER_() GPR_ATM_COMPILE_BARRIER_()
 #else
 #define GPR_ATM_LS_BARRIER_() gpr_atm_full_barrier()
 #endif
@@ -55,12 +57,19 @@ static __inline gpr_atm gpr_atm_acq_load(const gpr_atm *p) {
   return value;
 }
 
+static __inline gpr_atm gpr_atm_no_barrier_load(const gpr_atm *p) {
+  gpr_atm value = *p;
+  GPR_ATM_COMPILE_BARRIER_();
+  return value;
+}
+
 static __inline void gpr_atm_rel_store(gpr_atm *p, gpr_atm value) {
   GPR_ATM_LS_BARRIER_();
   *p = value;
 }
 
 #undef GPR_ATM_LS_BARRIER_
+#undef GPR_ATM_COMPILE_BARRIER_
 
 #define gpr_atm_no_barrier_fetch_add(p, delta) \
   gpr_atm_full_fetch_add((p), (delta))

include/grpc/support/atm_win32.h

@@ -49,6 +49,11 @@ static __inline gpr_atm gpr_atm_acq_load(const gpr_atm *p) {
   return result;
 }
 
+static __inline gpr_atm gpr_atm_no_barrier_load(const gpr_atm *p) {
+  /* TODO(dklempner): Can we implement something better here? */
+  gpr_atm_acq_load(p);
+}
+
 static __inline void gpr_atm_rel_store(gpr_atm *p, gpr_atm value) {
   gpr_atm_full_barrier();
   *p = value;

src/core/iomgr/fd_posix.c

@@ -210,7 +210,7 @@ static void notify_on(grpc_fd *fd, gpr_atm *st, grpc_iomgr_closure *closure,
     /* swap was unsuccessful due to an intervening set_ready call.
        Fall through to the READY code below */
     case READY:
-      assert(gpr_atm_acq_load(st) == READY);
+      assert(gpr_atm_no_barrier_load(st) == READY);
       gpr_atm_rel_store(st, NOT_READY);
       make_callback(closure->cb, closure->cb_arg,
                     !gpr_atm_acq_load(&fd->shutdown),
@@ -245,8 +245,8 @@ static void set_ready_locked(gpr_atm *st, grpc_iomgr_closure *callbacks,
          Fall through to the WAITING code below */
       state = gpr_atm_acq_load(st);
     default: /* waiting */
-      assert(gpr_atm_acq_load(st) != READY &&
+      assert(gpr_atm_no_barrier_load(st) != READY &&
-             gpr_atm_acq_load(st) != NOT_READY);
+             gpr_atm_no_barrier_load(st) != NOT_READY);
       callbacks[(*ncallbacks)++] = *(grpc_iomgr_closure *)state;
       gpr_atm_rel_store(st, NOT_READY);
       return;
@@ -271,7 +271,7 @@ void grpc_fd_shutdown(grpc_fd *fd) {
   grpc_iomgr_closure cb[2];
   size_t ncb = 0;
   gpr_mu_lock(&fd->set_state_mu);
-  GPR_ASSERT(!gpr_atm_acq_load(&fd->shutdown));
+  GPR_ASSERT(!gpr_atm_no_barrier_load(&fd->shutdown));
   gpr_atm_rel_store(&fd->shutdown, 1);
   set_ready_locked(&fd->readst, cb, &ncb);
   set_ready_locked(&fd->writest, cb, &ncb);