glusterd: acquire lock to update volinfo structure

Problem: With commit cb0339f92, we are using a separate syntask
for restart_bricks. There can be a situation where two threads
are accessing the same volinfo structure at the same time and
updating volinfo structure. This can lead volinfo to have
inconsistent values and assertion failures because of unexpected
values.

Solution: While updating the volinfo structure, acquire a
store_volinfo_lock, and release the lock only when the thread
completed its critical section part.

> BUG: bz#1627610
> Signed-off-by: Sanju Rakonde <srakonde@redhat.com>

> Change-Id: I545e4e2368e3285d8f7aa28081ff4448abb72f5d
(cherry picked from commit 484f417da945cf83afdbf136bb4817311862a8d2)

fixes: bz#1633552

Change-Id: I545e4e2368e3285d8f7aa28081ff4448abb72f5d
Signed-off-by: Sanju Rakonde <srakonde@redhat.com>

3 files changed, 42 insertions, 34 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 2300a548e5b..d3e4415cb45 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1785,44 +1785,47 @@ glusterd_store_volinfo(glusterd_volinfo_t *volinfo,
 
     GF_ASSERT(volinfo);
 
-    glusterd_perform_volinfo_version_action(volinfo, ac);
-    ret = glusterd_store_create_volume_dir(volinfo);
-    if (ret)
-        goto out;
-
-    ret = glusterd_store_create_volume_run_dir(volinfo);
-    if (ret)
-        goto out;
+    pthread_mutex_lock(&volinfo->store_volinfo_lock);
+    {
+        glusterd_perform_volinfo_version_action(volinfo, ac);
+        ret = glusterd_store_create_volume_dir(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_create_vol_shandle_on_absence(volinfo);
-    if (ret)
-        goto out;
+        ret = glusterd_store_create_volume_run_dir(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_create_nodestate_sh_on_absence(volinfo);
-    if (ret)
-        goto out;
+        ret = glusterd_store_create_vol_shandle_on_absence(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_perform_volume_store(volinfo);
-    if (ret)
-        goto out;
+        ret = glusterd_store_create_nodestate_sh_on_absence(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_volume_atomic_update(volinfo);
-    if (ret) {
-        glusterd_perform_volinfo_version_action(
-            volinfo, GLUSTERD_VOLINFO_VER_AC_DECREMENT);
-        goto out;
-    }
+        ret = glusterd_store_perform_volume_store(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_perform_node_state_store(volinfo);
-    if (ret)
-        goto out;
+        ret = glusterd_store_volume_atomic_update(volinfo);
+        if (ret) {
+            glusterd_perform_volinfo_version_action(
+                volinfo, GLUSTERD_VOLINFO_VER_AC_DECREMENT);
+            goto unlock;
+        }
 
-    /* checksum should be computed at the end */
-    ret = glusterd_compute_cksum(volinfo, _gf_false);
-    if (ret)
-        goto out;
+        ret = glusterd_store_perform_node_state_store(volinfo);
+        if (ret)
+            goto unlock;
 
-out:
+        /* checksum should be computed at the end */
+        ret = glusterd_compute_cksum(volinfo, _gf_false);
+        if (ret)
+            goto unlock;
+    }
+unlock:
+    pthread_mutex_unlock(&volinfo->store_volinfo_lock);
     if (ret)
         glusterd_store_volume_cleanup_tmp(volinfo);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index d4dd1dc11a8..a6561e101d5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2283,6 +2283,8 @@ glusterd_op_create_volume(dict_t *dict, char **op_errstr)
         goto out;
     }
 
+    pthread_mutex_init(&volinfo->store_volinfo_lock, NULL);
+
     ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
     if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 19cdee392f6..412ba7415f0 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -494,9 +494,12 @@ struct glusterd_volinfo_ {
     glusterd_tierdsvc_t tierd;
     glusterd_gfproxydsvc_t gfproxyd;
     int32_t quota_xattr_version;
-    gf_boolean_t stage_deleted; /* volume has passed staging
-                                 * for delete operation
-                                 */
+    gf_boolean_t stage_deleted;         /* volume has passed staging
+                                         * for delete operation
+                                         */
+    pthread_mutex_t store_volinfo_lock; /* acquire lock for
+                                         * updating the volinfo
+                                         */
 };
 
 typedef enum gd_snap_status_ {