glusterd: acquire lock to update volinfo structure

Problem: With commit cb0339f92, we are using a separate syntask
for restart_bricks. There can be a situation where two threads
are accessing the same volinfo structure at the same time and
updating volinfo structure. This can lead volinfo to have
inconsistent values and assertion failures because of unexpected
values.

Solution: While updating the volinfo structure, acquire a
store_volinfo_lock, and release the lock only when the thread
completed its critical section part.

Fixes: bz#1627610
Signed-off-by: Sanju Rakonde <srakonde@redhat.com>

Change-Id: I545e4e2368e3285d8f7aa28081ff4448abb72f5d

3 files changed, 42 insertions, 34 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index e04a3d493d9..26984b6ffba 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1785,44 +1785,47 @@ glusterd_store_volinfo(glusterd_volinfo_t *volinfo,
 
     GF_ASSERT(volinfo);
 
-    glusterd_perform_volinfo_version_action(volinfo, ac);
-    ret = glusterd_store_create_volume_dir(volinfo);
-    if (ret)
-        goto out;
-
-    ret = glusterd_store_create_volume_run_dir(volinfo);
-    if (ret)
-        goto out;
+    pthread_mutex_lock(&volinfo->store_volinfo_lock);
+    {
+        glusterd_perform_volinfo_version_action(volinfo, ac);
+        ret = glusterd_store_create_volume_dir(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_create_vol_shandle_on_absence(volinfo);
-    if (ret)
-        goto out;
+        ret = glusterd_store_create_volume_run_dir(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_create_nodestate_sh_on_absence(volinfo);
-    if (ret)
-        goto out;
+        ret = glusterd_store_create_vol_shandle_on_absence(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_perform_volume_store(volinfo);
-    if (ret)
-        goto out;
+        ret = glusterd_store_create_nodestate_sh_on_absence(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_volume_atomic_update(volinfo);
-    if (ret) {
-        glusterd_perform_volinfo_version_action(
-            volinfo, GLUSTERD_VOLINFO_VER_AC_DECREMENT);
-        goto out;
-    }
+        ret = glusterd_store_perform_volume_store(volinfo);
+        if (ret)
+            goto unlock;
 
-    ret = glusterd_store_perform_node_state_store(volinfo);
-    if (ret)
-        goto out;
+        ret = glusterd_store_volume_atomic_update(volinfo);
+        if (ret) {
+            glusterd_perform_volinfo_version_action(
+                volinfo, GLUSTERD_VOLINFO_VER_AC_DECREMENT);
+            goto unlock;
+        }
 
-    /* checksum should be computed at the end */
-    ret = glusterd_compute_cksum(volinfo, _gf_false);
-    if (ret)
-        goto out;
+        ret = glusterd_store_perform_node_state_store(volinfo);
+        if (ret)
+            goto unlock;
 
-out:
+        /* checksum should be computed at the end */
+        ret = glusterd_compute_cksum(volinfo, _gf_false);
+        if (ret)
+            goto unlock;
+    }
+unlock:
+    pthread_mutex_unlock(&volinfo->store_volinfo_lock);
     if (ret)
         glusterd_store_volume_cleanup_tmp(volinfo);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index d4dd1dc11a8..a6561e101d5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2283,6 +2283,8 @@ glusterd_op_create_volume(dict_t *dict, char **op_errstr)
         goto out;
     }
 
+    pthread_mutex_init(&volinfo->store_volinfo_lock, NULL);
+
     ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
     if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 19cdee392f6..412ba7415f0 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -494,9 +494,12 @@ struct glusterd_volinfo_ {
     glusterd_tierdsvc_t tierd;
     glusterd_gfproxydsvc_t gfproxyd;
     int32_t quota_xattr_version;
-    gf_boolean_t stage_deleted; /* volume has passed staging
-                                 * for delete operation
-                                 */
+    gf_boolean_t stage_deleted;         /* volume has passed staging
+                                         * for delete operation
+                                         */
+    pthread_mutex_t store_volinfo_lock; /* acquire lock for
+                                         * updating the volinfo
+                                         */
 };
 
 typedef enum gd_snap_status_ {