glusterfs: Use gcc builtin ATOMIC operator to increase/decreate refcount.

Problem: In glusterfs code base we call mutex_lock/unlock to take
         reference/dereference for a object.Sometime it could be
         reason for lock contention also.

Solution: There is no need to use mutex to increase/decrease ref
          counter, instead of using mutex use gcc builtin ATOMIC
          operation.

Test:   I have not observed yet how much performance gain after apply
        this patch specific to glusterfs but i have tested same
        with below small program(mutex and atomic both) and
        get good difference.

static int numOuterLoops;
static void *
threadFunc(void *arg)
{
    int j;

    for (j = 0; j < numOuterLoops; j++) {
            __atomic_add_fetch (&glob, 1,__ATOMIC_ACQ_REL);
    }
    return NULL;
}

int
main(int argc, char *argv[])
{
    int opt, s, j;
    int numThreads;
    pthread_t *thread;
    int verbose;
    int64_t n = 0;

    if (argc < 2 ) {
     printf(" Please provide 2 args Num of threads && Outer Loop\n");
     exit (-1);
    }
    numThreads = atoi(argv[1]);
    numOuterLoops = atoi (argv[2]);

    if (1) {
        printf("\tthreads: %d; outer loops: %d;\n",
                numThreads, numOuterLoops);
    }

    thread = calloc(numThreads, sizeof(pthread_t));
    if (thread == NULL) {
        printf ("calloc error so exit\n");
        exit (-1);
    }

    __atomic_store (&glob, &n, __ATOMIC_RELEASE);
    for (j = 0; j < numThreads; j++) {
        s = pthread_create(&thread[j], NULL, threadFunc, NULL);
        if (s != 0) {
            printf ("pthread_create failed so exit\n");
            exit (-1);
        }
    }

    for (j = 0; j < numThreads; j++) {
        s = pthread_join(thread[j], NULL);
        if (s != 0) {
            printf ("pthread_join failed so exit\n");
            exit (-1);
        }
    }
    printf("glob value is %ld\n",__atomic_load_n (&glob,__ATOMIC_RELAXED));

    exit(0);
}

   time ./thr_count 800 800000
   threads: 800; outer loops: 800000;
   glob value is 640000000

real	1m10.288s
user	0m57.269s
sys	3m31.565s

time ./thr_count_atomic 800 800000
     threads: 800; outer loops: 800000;
glob value is 640000000

real	0m20.313s
user	1m20.558s
sys	0m0.028

Change-Id: Ie5030a52ea264875e002e108dd4b207b15ab7cc7
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

1 files changed, 4 insertions, 20 deletions

diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index a8a88b0f96f..c6b1a4de611 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -435,7 +435,7 @@ gf_fd_fdptr_get (fdtable_t *fdtable, int64_t fd)
 fd_t *
 __fd_ref (fd_t *fd)
 {
-        ++fd->refcount;
+        GF_ATOMIC_INC (fd->refcount);
 
         return fd;
 }
@@ -444,28 +444,13 @@ __fd_ref (fd_t *fd)
 fd_t *
 fd_ref (fd_t *fd)
 {
-        fd_t *refed_fd = NULL;
-
         if (!fd) {
                 gf_msg_callingfn ("fd", GF_LOG_ERROR, EINVAL,
                                   LG_MSG_INVALID_ARG, "null fd");
                 return NULL;
         }
 
-        LOCK (&fd->inode->lock);
-        refed_fd = __fd_ref (fd);
-        UNLOCK (&fd->inode->lock);
-
-        return refed_fd;
-}
-
-
-fd_t *
-__fd_unref (fd_t *fd)
-{
-        GF_ASSERT (fd->refcount);
-
-        --fd->refcount;
+        GF_ATOMIC_INC (fd->refcount);
 
         return fd;
 }
@@ -551,8 +536,7 @@ fd_unref (fd_t *fd)
 
         LOCK (&fd->inode->lock);
         {
-                __fd_unref (fd);
-                refcount = fd->refcount;
+                refcount = GF_ATOMIC_DEC (fd->refcount);
                 if (refcount == 0) {
                         if (!list_empty (&fd->inode_list)) {
                                 list_del_init (&fd->inode_list);
@@ -1186,7 +1170,7 @@ fdentry_dump_to_dict (fdentry_t *fdentry, char *prefix, dict_t *dict,
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "%s.refcount", prefix);
-                ret = dict_set_int32 (dict, key, fdentry->fd->refcount);
+                ret = dict_set_int32 (dict, key, GF_ATOMIC_GET (fdentry->fd->refcount));
                 if (ret)
                         return;