glusterfs: Use gcc builtin ATOMIC operator to increase/decreate refcount.

Problem: In glusterfs code base we call mutex_lock/unlock to take
         reference/dereference for a object.Sometime it could be
         reason for lock contention also.

Solution: There is no need to use mutex to increase/decrease ref
          counter, instead of using mutex use gcc builtin ATOMIC
          operation.

Test:   I have not observed yet how much performance gain after apply
        this patch specific to glusterfs but i have tested same
        with below small program(mutex and atomic both) and
        get good difference.

static int numOuterLoops;
static void *
threadFunc(void *arg)
{
    int j;

    for (j = 0; j < numOuterLoops; j++) {
            __atomic_add_fetch (&glob, 1,__ATOMIC_ACQ_REL);
    }
    return NULL;
}

int
main(int argc, char *argv[])
{
    int opt, s, j;
    int numThreads;
    pthread_t *thread;
    int verbose;
    int64_t n = 0;

    if (argc < 2 ) {
     printf(" Please provide 2 args Num of threads && Outer Loop\n");
     exit (-1);
    }
    numThreads = atoi(argv[1]);
    numOuterLoops = atoi (argv[2]);

    if (1) {
        printf("\tthreads: %d; outer loops: %d;\n",
                numThreads, numOuterLoops);
    }

    thread = calloc(numThreads, sizeof(pthread_t));
    if (thread == NULL) {
        printf ("calloc error so exit\n");
        exit (-1);
    }

    __atomic_store (&glob, &n, __ATOMIC_RELEASE);
    for (j = 0; j < numThreads; j++) {
        s = pthread_create(&thread[j], NULL, threadFunc, NULL);
        if (s != 0) {
            printf ("pthread_create failed so exit\n");
            exit (-1);
        }
    }

    for (j = 0; j < numThreads; j++) {
        s = pthread_join(thread[j], NULL);
        if (s != 0) {
            printf ("pthread_join failed so exit\n");
            exit (-1);
        }
    }
    printf("glob value is %ld\n",__atomic_load_n (&glob,__ATOMIC_RELAXED));

    exit(0);
}

   time ./thr_count 800 800000
   threads: 800; outer loops: 800000;
   glob value is 640000000

real	1m10.288s
user	0m57.269s
sys	3m31.565s

time ./thr_count_atomic 800 800000
     threads: 800; outer loops: 800000;
glob value is 640000000

real	0m20.313s
user	1m20.558s
sys	0m0.028

Change-Id: Ie5030a52ea264875e002e108dd4b207b15ab7cc7
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

22 files changed, 86 insertions, 268 deletions

diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c
index a47c8529709..d0be267d2ee 100644
--- a/libglusterfs/src/dict.c
+++ b/libglusterfs/src/dict.c
@@ -579,8 +579,8 @@ dict_destroy (dict_t *this)
         data_pair_t *pair = this->members_list;
         data_pair_t *prev = this->members_list;
         glusterfs_ctx_t *ctx = NULL;
-        uint32_t total_pairs = 0;
         uint64_t current_max = 0;
+        uint32_t total_pairs = 0;
 
         LOCK_DESTROY (&this->lock);
 
@@ -629,7 +629,7 @@ dict_destroy (dict_t *this)
 void
 dict_unref (dict_t *this)
 {
-        int32_t ref;
+        uint64_t ref = 0;
 
         if (!this) {
                 gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL,
@@ -637,12 +637,7 @@ dict_unref (dict_t *this)
                 return;
         }
 
-        LOCK (&this->lock);
-
-        this->refcount--;
-        ref = this->refcount;
-
-        UNLOCK (&this->lock);
+        ref = GF_ATOMIC_DEC (this->refcount);
 
         if (!ref)
                 dict_destroy (this);
@@ -657,12 +652,7 @@ dict_ref (dict_t *this)
                 return NULL;
         }
 
-        LOCK (&this->lock);
-
-        this->refcount++;
-
-        UNLOCK (&this->lock);
-
+        GF_ATOMIC_INC (this->refcount);
         return this;
 }
 
@@ -678,12 +668,7 @@ data_unref (data_t *this)
                 return;
         }
 
-        LOCK (&this->lock);
-
-        this->refcount--;
-        ref = this->refcount;
-
-        UNLOCK (&this->lock);
+        ref = GF_ATOMIC_DEC (this->refcount);
 
         if (!ref)
                 data_destroy (this);
@@ -698,11 +683,7 @@ data_ref (data_t *this)
                 return NULL;
         }
 
-        LOCK (&this->lock);
-
-        this->refcount++;
-
-        UNLOCK (&this->lock);
+        GF_ATOMIC_INC (this->refcount);
 
         return this;
 }
diff --git a/libglusterfs/src/dict.h b/libglusterfs/src/dict.h
index 713da61d4ce..48c6f242723 100644
--- a/libglusterfs/src/dict.h
+++ b/libglusterfs/src/dict.h
@@ -67,7 +67,7 @@ struct _data {
         unsigned char  is_const:1;
         int32_t        len;
         char          *data;
-        int32_t        refcount;
+        gf_atomic_t    refcount;
         gf_lock_t      lock;
 };
 
@@ -84,7 +84,7 @@ struct _dict {
         unsigned char   is_static:1;
         int32_t         hash_size;
         int32_t         count;
-        int32_t         refcount;
+        gf_atomic_t     refcount;
         data_pair_t   **members;
         data_pair_t    *members_list;
         char           *extra_free;
@@ -93,7 +93,7 @@ struct _dict {
         data_pair_t    *members_internal;
         data_pair_t     free_pair;
         gf_boolean_t    free_pair_in_use;
-        uint32_t        max_count;
+        uint64_t        max_count;
 };
 
 typedef gf_boolean_t (*dict_match_t) (dict_t *d, char *k, data_t *v,
diff --git a/libglusterfs/src/event-epoll.c b/libglusterfs/src/event-epoll.c
index 7fc53ffeaf6..45f724ab055 100644
--- a/libglusterfs/src/event-epoll.c
+++ b/libglusterfs/src/event-epoll.c
@@ -32,7 +32,7 @@ struct event_slot_epoll {
 	int fd;
 	int events;
 	int gen;
-	int ref;
+	gf_atomic_t ref;
 	int do_close;
 	int in_handler;
         int handled_error;
@@ -197,12 +197,7 @@ event_slot_get (struct event_pool *event_pool, int idx)
 		return NULL;
 
 	slot = &table[offset];
-
-	LOCK (&slot->lock);
-	{
-		slot->ref++;
-	}
-	UNLOCK (&slot->lock);
+        GF_ATOMIC_INC (slot->ref);
 
 	return slot;
 }
@@ -216,9 +211,10 @@ event_slot_unref (struct event_pool *event_pool, struct event_slot_epoll *slot,
 	int fd = -1;
 	int do_close = 0;
 
+        ref = GF_ATOMIC_DEC (slot->ref);
+
 	LOCK (&slot->lock);
 	{
-		ref = --slot->ref;
 		fd = slot->fd;
 		do_close = slot->do_close;
 	}
diff --git a/libglusterfs/src/fd-lk.c b/libglusterfs/src/fd-lk.c
index 358cf3b616a..e516f64bff6 100644
--- a/libglusterfs/src/fd-lk.c
+++ b/libglusterfs/src/fd-lk.c
@@ -65,15 +65,11 @@ fd_lk_ctx_unref (fd_lk_ctx_t *lk_ctx)
 
         GF_VALIDATE_OR_GOTO ("fd-lk", lk_ctx, err);
 
-        LOCK (&lk_ctx->lock);
-        {
-                ref = --lk_ctx->ref;
-                if (ref < 0)
-                        GF_ASSERT (!ref);
-                if (ref == 0)
-                        _fd_lk_destroy_lock_list (lk_ctx);
-        }
-        UNLOCK (&lk_ctx->lock);
+        ref = GF_ATOMIC_DEC (lk_ctx->ref);
+        if (ref < 0)
+                GF_ASSERT (!ref);
+        if (ref == 0)
+                _fd_lk_destroy_lock_list (lk_ctx);
 
         if (ref == 0) {
                 LOCK_DESTROY (&lk_ctx->lock);
@@ -86,58 +82,17 @@ err:
 }
 
 fd_lk_ctx_t *
-_fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx)
-{
-        if (!lk_ctx) {
-                gf_msg_callingfn ("fd-lk", GF_LOG_WARNING, EINVAL,
-                                  LG_MSG_INVALID_ARG, "invalid argument");
-                return NULL;
-        }
-
-        ++lk_ctx->ref;
-
-        return lk_ctx;
-}
-
-fd_lk_ctx_t *
 fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx)
 {
-        fd_lk_ctx_t *new_lk_ctx = NULL;
-
         if (!lk_ctx) {
                 gf_msg_callingfn ("fd-lk", GF_LOG_WARNING, EINVAL,
                                   LG_MSG_INVALID_ARG, "invalid argument");
                 return NULL;
         }
 
-        LOCK (&lk_ctx->lock);
-        {
-                new_lk_ctx = _fd_lk_ctx_ref (lk_ctx);
-        }
-        UNLOCK (&lk_ctx->lock);
-
-        return new_lk_ctx;
-}
-
-fd_lk_ctx_t *
-fd_lk_ctx_try_ref (fd_lk_ctx_t *lk_ctx)
-{
-        int         ret         = -1;
-        fd_lk_ctx_t *new_lk_ctx = NULL;
-
-        if (!lk_ctx) {
-                goto out;
-        }
-
-        ret = TRY_LOCK (&lk_ctx->lock);
-        if (ret)
-                goto out;
+        GF_ATOMIC_INC (lk_ctx->ref);
 
-        new_lk_ctx = _fd_lk_ctx_ref (lk_ctx);
-        UNLOCK (&lk_ctx->lock);
-
-out:
-        return new_lk_ctx;
+        return lk_ctx;
 }
 
 fd_lk_ctx_t *
diff --git a/libglusterfs/src/fd-lk.h b/libglusterfs/src/fd-lk.h
index 51f62991681..11d864dc207 100644
--- a/libglusterfs/src/fd-lk.h
+++ b/libglusterfs/src/fd-lk.h
@@ -30,7 +30,7 @@ struct _fd;
 
 struct fd_lk_ctx {
         struct list_head lk_list;
-        int   ref;
+        gf_atomic_t ref;
         gf_lock_t lock;
 };
 typedef struct fd_lk_ctx fd_lk_ctx_t;
@@ -45,16 +45,11 @@ struct fd_lk_ctx_node {
 };
 typedef struct fd_lk_ctx_node fd_lk_ctx_node_t;
 
-fd_lk_ctx_t *
-_fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx);
 
 fd_lk_ctx_t *
 fd_lk_ctx_ref (fd_lk_ctx_t *lk_ctx);
 
 fd_lk_ctx_t *
-fd_lk_ctx_try_ref (fd_lk_ctx_t *lk_ctx);
-
-fd_lk_ctx_t *
 fd_lk_ctx_create (void);
 
 int
diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index a8a88b0f96f..c6b1a4de611 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -435,7 +435,7 @@ gf_fd_fdptr_get (fdtable_t *fdtable, int64_t fd)
 fd_t *
 __fd_ref (fd_t *fd)
 {
-        ++fd->refcount;
+        GF_ATOMIC_INC (fd->refcount);
 
         return fd;
 }
@@ -444,28 +444,13 @@ __fd_ref (fd_t *fd)
 fd_t *
 fd_ref (fd_t *fd)
 {
-        fd_t *refed_fd = NULL;
-
         if (!fd) {
                 gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
                                   LG_MSG_INVALID_ARG, "null fd");
                 return NULL;
         }
 
-        LOCK (&fd->inode->lock);
-        refed_fd = __fd_ref (fd);
-        UNLOCK (&fd->inode->lock);
-
-        return refed_fd;
-}
-
-
-fd_t *
-__fd_unref (fd_t *fd)
-{
-        GF_ASSERT (fd->refcount);
-
-        --fd->refcount;
+        GF_ATOMIC_INC (fd->refcount);
 
         return fd;
 }
@@ -551,8 +536,7 @@ fd_unref (fd_t *fd)
 
         LOCK (&fd->inode->lock);
         {
-                __fd_unref (fd);
-                refcount = fd->refcount;
+                refcount = GF_ATOMIC_DEC (fd->refcount);
                 if (refcount == 0) {
                         if (!list_empty (&fd->inode_list)) {
                                 list_del_init (&fd->inode_list);
@@ -1186,7 +1170,7 @@ fdentry_dump_to_dict (fdentry_t *fdentry, char *prefix, dict_t *dict,
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "%s.refcount", prefix);
-                ret = dict_set_int32 (dict, key, fdentry->fd->refcount);
+                ret = dict_set_int32 (dict, key, GF_ATOMIC_GET (fdentry->fd->refcount));
                 if (ret)
                         return;
 
diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
index 3b4acf762ae..3f68b1c4997 100644
--- a/libglusterfs/src/fd.h
+++ b/libglusterfs/src/fd.h
@@ -40,7 +40,7 @@ struct _fd_ctx {
 struct _fd {
         uint64_t          pid;
 	int32_t           flags;
-        int32_t           refcount;
+        gf_atomic_t       refcount;
         struct list_head  inode_list;
         struct _inode    *inode;
         gf_lock_t         lock; /* used ONLY for manipulating
diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index 76584fc9cde..d4f0d89c338 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -126,7 +126,7 @@ __iobuf_arena_destroy_iobufs (struct iobuf_arena *iobuf_arena)
 
         iobuf = iobuf_arena->iobufs;
         for (i = 0; i < iobuf_cnt; i++) {
-                GF_ASSERT (iobuf->ref == 0);
+                GF_ASSERT (GF_ATOMIC_GET(iobuf->ref) == 0);
 
                 LOCK_DESTROY (&iobuf->lock);
                 list_del_init (&iobuf->list);
@@ -529,23 +529,6 @@ out:
 
 
 struct iobuf *
-__iobuf_ref (struct iobuf *iobuf)
-{
-        iobuf->ref++;
-
-        return iobuf;
-}
-
-
-struct iobuf *
-__iobuf_unref (struct iobuf *iobuf)
-{
-        iobuf->ref--;
-
-        return iobuf;
-}
-
-struct iobuf *
 __iobuf_get (struct iobuf_arena *iobuf_arena, size_t page_size)
 {
         struct iobuf      *iobuf        = NULL;
@@ -619,7 +602,7 @@ iobuf_get_from_stdalloc (struct iobuf_pool *iobuf_pool, size_t page_size)
         LOCK_INIT (&iobuf->lock);
 
         /* Hold a ref because you are allocating and using it */
-        iobuf->ref = 1;
+        GF_ATOMIC_INIT (iobuf->ref, 1);
 
         ret = 0;
 out:
@@ -835,12 +818,7 @@ iobuf_unref (struct iobuf *iobuf)
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf, out);
 
-        LOCK (&iobuf->lock);
-        {
-                __iobuf_unref (iobuf);
-                ref = iobuf->ref;
-        }
-        UNLOCK (&iobuf->lock);
+        ref = GF_ATOMIC_DEC (iobuf->ref);
 
         if (!ref)
                 iobuf_put (iobuf);
@@ -854,12 +832,7 @@ struct iobuf *
 iobuf_ref (struct iobuf *iobuf)
 {
         GF_VALIDATE_OR_GOTO ("iobuf", iobuf, out);
-
-        LOCK (&iobuf->lock);
-        {
-                __iobuf_ref (iobuf);
-        }
-        UNLOCK (&iobuf->lock);
+        GF_ATOMIC_INC (iobuf->ref);
 
 out:
         return iobuf;
@@ -888,8 +861,7 @@ iobref_new ()
 
         LOCK_INIT (&iobref->lock);
 
-        iobref->ref++;
-
+        GF_ATOMIC_INIT (iobref->ref, 1);
         return iobref;
 }
 
@@ -898,12 +870,7 @@ struct iobref *
 iobref_ref (struct iobref *iobref)
 {
         GF_VALIDATE_OR_GOTO ("iobuf", iobref, out);
-
-        LOCK (&iobref->lock);
-        {
-                iobref->ref++;
-        }
-        UNLOCK (&iobref->lock);
+        GF_ATOMIC_INC (iobref->ref);
 
 out:
         return iobref;
@@ -940,12 +907,7 @@ iobref_unref (struct iobref *iobref)
         int ref = 0;
 
         GF_VALIDATE_OR_GOTO ("iobuf", iobref, out);
-
-        LOCK (&iobref->lock);
-        {
-                ref = (--iobref->ref);
-        }
-        UNLOCK (&iobref->lock);
+        ref = GF_ATOMIC_DEC (iobref->ref);
 
         if (!ref)
                 iobref_destroy (iobref);
diff --git a/libglusterfs/src/iobuf.h b/libglusterfs/src/iobuf.h
index 03aa954bbdd..830904c2fe9 100644
--- a/libglusterfs/src/iobuf.h
+++ b/libglusterfs/src/iobuf.h
@@ -65,7 +65,7 @@ struct iobuf {
         struct iobuf_arena  *iobuf_arena;
 
         gf_lock_t            lock; /* for ->ptr and ->ref */
-        int                  ref;  /* 0 == passive, >0 == active */
+        gf_atomic_t          ref;  /* 0 == passive, >0 == active */
 
         void                *ptr;  /* usable memory region by the consumer */
 
@@ -150,7 +150,7 @@ void iobuf_to_iovec(struct iobuf *iob, struct iovec *iov);
 
 struct iobref {
         gf_lock_t          lock;
-        int                ref;
+        gf_atomic_t        ref;
         struct iobuf     **iobrefs;
 	int                alloced;
 	int                used;
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index cb59408dcf6..06aed0a80ae 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1740,11 +1740,8 @@ rpc_clnt_ref (struct rpc_clnt *rpc)
 {
         if (!rpc)
                 return NULL;
-        pthread_mutex_lock (&rpc->lock);
-        {
-                rpc->refcount++;
-        }
-        pthread_mutex_unlock (&rpc->lock);
+
+        GF_ATOMIC_INC (rpc->refcount);
         return rpc;
 }
 
@@ -1806,11 +1803,9 @@ rpc_clnt_unref (struct rpc_clnt *rpc)
 
         if (!rpc)
                 return NULL;
-        pthread_mutex_lock (&rpc->lock);
-        {
-                count = --rpc->refcount;
-        }
-        pthread_mutex_unlock (&rpc->lock);
+
+        count = GF_ATOMIC_DEC (rpc->refcount);
+
         if (!count) {
                 rpc_clnt_trigger_destroy (rpc);
                 return NULL;
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 428ffb80650..6503ca5a573 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -187,7 +187,7 @@ typedef struct rpc_clnt {
         struct mem_pool       *saved_frames_pool;
 
         glusterfs_ctx_t       *ctx;
-        int                   refcount;
+        gf_atomic_t           refcount;
         int                   auth_null;
         char                  disabled;
         xlator_t             *owner;
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index 40b5917f9b8..40c41ab7016 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -493,11 +493,7 @@ rpc_transport_ref (rpc_transport_t *this)
 
 	GF_VALIDATE_OR_GOTO("rpc_transport", this, fail);
 
-	pthread_mutex_lock (&this->lock);
-	{
-		this->refcount ++;
-	}
-	pthread_mutex_unlock (&this->lock);
+        GF_ATOMIC_INC (this->refcount);
 
 	return_this = this;
 fail:
@@ -513,11 +509,7 @@ rpc_transport_unref (rpc_transport_t *this)
 
 	GF_VALIDATE_OR_GOTO("rpc_transport", this, fail);
 
-	pthread_mutex_lock (&this->lock);
-	{
-                refcount = --this->refcount;
-	}
-	pthread_mutex_unlock (&this->lock);
+        refcount = GF_ATOMIC_DEC (this->refcount);
 
 	if (refcount == 0) {
                 if (this->mydata)
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 33f474e42c6..332efb45c25 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -184,7 +184,7 @@ struct rpc_transport {
         void                      *xl;       /* Used for THIS */
         void                      *mydata;
         pthread_mutex_t            lock;
-        int32_t                    refcount;
+        gf_atomic_t                refcount;
 
         int32_t                    outstanding_rpc_count;
 
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 3d6fbb1fd79..cc7d41b3bfa 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -87,8 +87,8 @@ struct dht_layout {
          */
         int                gen;
         int                type;
-        int                ref; /* use with dht_conf_t->layout_lock */
-        uint32_t           search_unhashed;
+        gf_atomic_t        ref; /* use with dht_conf_t->layout_lock */
+        gf_boolean_t       search_unhashed;
         struct {
                 int        err;   /* 0 = normal
                                      -1 = dir exists and no xattr
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 419ce329a1d..97b98e01451 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -48,12 +48,12 @@ dht_layout_new (xlator_t *this, int cnt)
                 layout->gen = conf->gen;
         }
 
-        layout->ref = 1;
+        GF_ATOMIC_INIT (layout->ref, 1);
 
         ENSURE(NULL != layout);
         ENSURE(layout->type == DHT_HASH_TYPE_DM);
         ENSURE(layout->cnt == cnt);
-        ENSURE(layout->ref == 1);
+        ENSURE(GF_ATOMIC_GET (layout->ref) == 1);
 out:
         return layout;
 }
@@ -62,24 +62,13 @@ out:
 dht_layout_t *
 dht_layout_get (xlator_t *this, inode_t *inode)
 {
-        dht_conf_t   *conf = NULL;
         dht_layout_t *layout = NULL;
         int           ret = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        LOCK (&conf->layout_lock);
-        {
-                ret = dht_inode_ctx_layout_get (inode, this, &layout);
-                if ((!ret) && layout) {
-                        layout->ref++;
-                }
+        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+        if ((!ret) && layout) {
+                GF_ATOMIC_INC (layout->ref);
         }
-        UNLOCK (&conf->layout_lock);
-
-out:
         return layout;
 }
 
@@ -100,7 +89,7 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
         {
                 oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
                 if (layout)
-                        layout->ref++;
+                        GF_ATOMIC_INC (layout->ref);
                 ret = dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
@@ -108,6 +97,8 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
         if (!oldret) {
                 dht_layout_unref (this, old_layout);
         }
+        if (ret)
+                GF_ATOMIC_DEC (layout->ref);
 
 out:
         return ret;
@@ -117,19 +108,12 @@ out:
 void
 dht_layout_unref (xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
         int          ref = 0;
 
         if (!layout || layout->preset || !this->private)
                 return;
 
-        conf = this->private;
-
-        LOCK (&conf->layout_lock);
-        {
-                ref = --layout->ref;
-        }
-        UNLOCK (&conf->layout_lock);
+        ref = GF_ATOMIC_DEC (layout->ref);
 
         if (!ref)
                 GF_FREE (layout);
@@ -139,17 +123,10 @@ dht_layout_unref (xlator_t *this, dht_layout_t *layout)
 dht_layout_t *
 dht_layout_ref (xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-
         if (layout->preset || !this->private)
                 return layout;
 
-        conf = this->private;
-        LOCK (&conf->layout_lock);
-        {
-                layout->ref++;
-        }
-        UNLOCK (&conf->layout_lock);
+        GF_ATOMIC_INC (layout->ref);
 
         return layout;
 }
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
index 84a89160e38..fb6445ce1f3 100644
--- a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
+++ b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
@@ -90,7 +90,7 @@ test_dht_layout_new(void **state)
     assert_non_null(layout);
     assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
     assert_int_equal(layout->cnt, cnt);
-    assert_int_equal(layout->ref, 1);
+    assert_int_equal(GF_ATOMIC_GET (layout->ref), 1);
     assert_int_equal(layout->gen, 0);
     assert_int_equal(layout->spread_cnt, 0);
     free(layout);
@@ -107,7 +107,7 @@ test_dht_layout_new(void **state)
     assert_non_null(layout);
     assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
     assert_int_equal(layout->cnt, cnt);
-    assert_int_equal(layout->ref, 1);
+    assert_int_equal(GF_ATOMIC_GET (layout->ref), 1);
     assert_int_equal(layout->gen, conf->gen);
     assert_int_equal(layout->spread_cnt, conf->dir_spread_cnt);
     free(layout);
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index c0cd0e2477b..4864c67a101 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -74,7 +74,7 @@ struct ios_stat {
         char           *filename;
         gf_atomic_t     counters[IOS_STATS_TYPE_MAX];
         struct ios_stat_lat thru_counters [IOS_STATS_THRU_MAX];
-        int             refcnt;
+        gf_atomic_t     refcnt;
 };
 
 struct ios_stat_list {
@@ -333,13 +333,10 @@ ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
 static int
 ios_stat_ref (struct ios_stat *iosstat)
 {
-        LOCK (&iosstat->lock);
-        {
-                iosstat->refcnt++;
-        }
-        UNLOCK (&iosstat->lock);
+        uint64_t refcnt = 0;
+        refcnt = GF_ATOMIC_INC (iosstat->refcnt);
 
-        return iosstat->refcnt;
+        return refcnt;
 }
 
 
@@ -347,18 +344,16 @@ static int
 ios_stat_unref (struct ios_stat *iosstat)
 {
         int cleanup = 0;
-        LOCK (&iosstat->lock);
-        {
-                iosstat->refcnt--;
-                if (iosstat->refcnt == 0) {
-                        if (iosstat->filename) {
-                                GF_FREE (iosstat->filename);
-                                iosstat->filename = NULL;
-                        }
-                        cleanup = 1;
+        uint64_t refcnt = 0;
+
+        refcnt = GF_ATOMIC_DEC (iosstat->refcnt);
+        if (refcnt == 0) {
+                if (iosstat->filename) {
+                        GF_FREE (iosstat->filename);
+                        iosstat->filename = NULL;
                 }
+                cleanup = 1;
         }
-        UNLOCK (&iosstat->lock);
 
         if (cleanup) {
                 LOCK_DESTROY (&iosstat->lock);
diff --git a/xlators/features/changelog/src/changelog-ev-handle.h b/xlators/features/changelog/src/changelog-ev-handle.h
index eef0492a9ee..e89af8793a8 100644
--- a/xlators/features/changelog/src/changelog-ev-handle.h
+++ b/xlators/features/changelog/src/changelog-ev-handle.h
@@ -24,7 +24,7 @@ typedef struct changelog_rpc_clnt {
 
         gf_lock_t lock;
 
-        unsigned long ref;
+        gf_atomic_t   ref;
         gf_boolean_t  disconnected;
 
         unsigned int filter;
@@ -43,11 +43,7 @@ typedef struct changelog_rpc_clnt {
 static inline void
 changelog_rpc_clnt_ref (changelog_rpc_clnt_t *crpc)
 {
-        LOCK (&crpc->lock);
-        {
-                ++crpc->ref;
-        }
-        UNLOCK (&crpc->lock);
+        GF_ATOMIC_INC (crpc->ref);
 }
 
 static inline void
@@ -66,16 +62,14 @@ static inline void
 changelog_rpc_clnt_unref (changelog_rpc_clnt_t *crpc)
 {
         gf_boolean_t gone = _gf_false;
+        uint64_t     ref  = 0;
+
+        ref = GF_ATOMIC_DEC (crpc->ref);
 
-        LOCK (&crpc->lock);
-        {
-                if (!(--crpc->ref)
-                    && changelog_rpc_clnt_is_disconnected (crpc)) {
-                        list_del (&crpc->list);
-                        gone = _gf_true;
-                }
+        if (!ref && changelog_rpc_clnt_is_disconnected (crpc)) {
+                list_del (&crpc->list);
+                gone = _gf_true;
         }
-        UNLOCK (&crpc->lock);
 
         if (gone)
                 crpc->cleanup (crpc);
diff --git a/xlators/features/changelog/src/changelog-rpc.c b/xlators/features/changelog/src/changelog-rpc.c
index 5524e433cbb..72b5cdbc7cb 100644
--- a/xlators/features/changelog/src/changelog-rpc.c
+++ b/xlators/features/changelog/src/changelog-rpc.c
@@ -211,7 +211,7 @@ changelog_rpc_clnt_init (xlator_t *this,
         /* Take a ref, the last unref will be on RPC_CLNT_DESTROY
          * which comes as a result of last rpc_clnt_unref.
          */
-        crpc->ref = 1;
+        GF_ATOMIC_INIT (crpc->ref, 1);
         changelog_set_disconnect_flag (crpc, _gf_false);
 
         crpc->filter = rpc_req->filter;
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index b053eb3250e..6428a77442e 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -539,8 +539,8 @@ static void
 __nfs3_call_state_wipe (nfs3_call_state_t *cs)
 {
         if (cs->fd) {
-                gf_msg_trace (GF_NFS3, 0, "fd 0x%lx ref: %d",
-                        (long)cs->fd, cs->fd->refcount);
+                gf_msg_trace (GF_NFS3, 0, "fd 0x%lx ref: %"PRId64,
+                        (long)cs->fd, GF_ATOMIC_GET (cs->fd->refcount));
                 fd_unref (cs->fd);
         }
 
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index ca3d140d08e..6429dc2e8ae 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -32,7 +32,7 @@ int client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
 int client_set_lk_version (xlator_t *this);
 
 typedef struct client_fd_lk_local {
-        int             ref;
+        gf_atomic_t     ref;
         gf_boolean_t    error;
         gf_lock_t       lock;
         clnt_fd_ctx_t *fdctx;
@@ -285,11 +285,7 @@ clnt_fd_lk_local_ref (xlator_t *this, clnt_fd_lk_local_t *local)
 {
         GF_VALIDATE_OR_GOTO (this->name, local, out);
 
-        LOCK (&local->lock);
-        {
-                local->ref++;
-        }
-        UNLOCK (&local->lock);
+        GF_ATOMIC_INC (local->ref);
 out:
         return local;
 }
@@ -301,11 +297,7 @@ clnt_fd_lk_local_unref (xlator_t *this, clnt_fd_lk_local_t *local)
 
         GF_VALIDATE_OR_GOTO (this->name, local, out);
 
-        LOCK (&local->lock);
-        {
-                ref = --local->ref;
-        }
-        UNLOCK (&local->lock);
+        ref = GF_ATOMIC_DEC (local->ref);
 
         if (ref == 0) {
                 LOCK_DESTROY (&local->lock);
@@ -325,7 +317,7 @@ clnt_fd_lk_local_create (clnt_fd_ctx_t *fdctx)
         if (!local)
                 goto out;
 
-        local->ref    = 1;
+        GF_ATOMIC_INIT (local->ref, 1);
         local->error  = _gf_false;
         local->fdctx = fdctx;
 
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 8df697ed883..8ec9eaeb8a9 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2844,7 +2844,7 @@ client_fd_lk_ctx_dump (xlator_t *this, fd_lk_ctx_t *lk_ctx, int nth_fd)
         fd_lk_ctx_node_t        *plock                   = NULL;
         char                    key[GF_DUMP_MAX_BUF_LEN] = {0,};
 
-        lk_ctx_ref = fd_lk_ctx_try_ref (lk_ctx);
+        lk_ctx_ref = fd_lk_ctx_ref (lk_ctx);
         if (!lk_ctx_ref)
                 return;