glusterd/snapshot: Recreate the mount dirs and mount the lvm snapshots on node reboot.

The lvm snapshots of the bricks are mounted at /var/run/gluster/snaps/ or
/run/gluster/snaps. These paths being on a tempfs, on reboot are removed.
So when glusterd starts, we need to recreate these paths, activate the
respective logical volumes (lvm snapshots of the bricks), and mount
these logical volumes at their respective paths.

Change-Id: Ic5ef61e79a25d9830df717c592391965fe09db62
BUG: 1061685
Signed-off-by: Avra Sengupta <asengupt@redhat.com>
Reviewed-on: http://review.gluster.org/7452
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

4 files changed, 415 insertions, 70 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index c1967a29ca9..0e824a02256 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -1104,8 +1104,12 @@ glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
         brick_count = -1;
         list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
                 brick_count++;
-                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                if (uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s:%s belongs to a different node",
+                                brickinfo->hostname, brickinfo->path);
                         continue;
+                }
 
                 if (brickinfo->snap_status == -1) {
                         gf_log (this->name, GF_LOG_INFO,
@@ -1170,6 +1174,7 @@ glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
 
         ret = 0;
 out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
         return ret;
 }
 
@@ -2855,9 +2860,7 @@ glusterd_snap_brick_create (char *device, glusterd_volinfo_t *snap_volinfo,
         glusterd_conf_t *priv                            = NULL;
         char             snap_brick_mount_path[PATH_MAX] = "";
         char             snap_brick_path[PATH_MAX]       = "";
-        char             msg[1024]                       = "";
         struct stat      statbuf                         = {0, };
-        runner_t         runner                          = {0, };
 
         this = THIS;
         priv = this->private;
@@ -2888,25 +2891,12 @@ glusterd_snap_brick_create (char *device, glusterd_volinfo_t *snap_volinfo,
                         MS_MGC_VAL, "nouuid");
            But for now, mounting using runner apis.
         */
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "mounting snapshot of the brick %s:%s",
-                  original_brickinfo->hostname, original_brickinfo->path);
-        runner_add_args (&runner, "mount", "-o", "nouuid", device,
-                         snap_brick_mount_path, NULL);
-        runner_log (&runner, "", GF_LOG_DEBUG, msg);
-
-        /* let glusterd get blocked till snapshot is over */
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
+        ret = glusterd_mount_lvm_snapshot (device, snap_brick_mount_path);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "mounting the snapshot "
-                        "logical device %s failed (error: %s)", device,
-                        strerror (errno));
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to mount lvm snapshot.");
                 goto out;
-        } else
-                gf_log (this->name, GF_LOG_DEBUG, "mounting the snapshot "
-                        "logical device %s successful", device);
+        }
 
         ret = stat (snap_brick_path, &statbuf);
         if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 1e39fb27e5f..d9d2a3e50ac 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -44,6 +44,7 @@
 #include <sys/resource.h>
 #include <inttypes.h>
 #include <dirent.h>
+#include <mntent.h>
 
 void
 glusterd_replace_slash_with_hyphen (char *str)
@@ -2424,7 +2425,7 @@ glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
                               strlen (GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP))) {
                         ret = uuid_parse (value, volinfo->restored_from_snap);
                         if (ret)
-                                gf_log ("", GF_LOG_WARNING,
+                                gf_log (this->name, GF_LOG_WARNING,
                                         "failed to parse restored snap's uuid");
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
                                 strlen (GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
@@ -2792,6 +2793,209 @@ out:
         return ret;
 }
 
+/* Figure out the brick mount path, from the brick path */
+int32_t
+glusterd_find_brick_mount_path (char *brick_path, int32_t brick_count,
+                                char **brick_mount_path)
+{
+        char                     brick_num[PATH_MAX] = "";
+        char                    *ptr                 = NULL;
+        int32_t                  ret                 = -1;
+        xlator_t                *this                = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (brick_path);
+        GF_ASSERT (brick_mount_path);
+
+        *brick_mount_path = gf_strdup (brick_path);
+        if (!*brick_mount_path) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (brick_num, sizeof(brick_num), "brick%d", brick_count);
+
+        /* Finding the pointer to the end of
+         * /var/run/gluster/snaps/<snap-uuid>
+         */
+        ptr = strstr (*brick_mount_path, brick_num);
+        if (!ptr) {
+                /* Snapshot bricks must have brick num as part
+                 * of the brickpath
+                 */
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid brick path(%s)", brick_path);
+                ret = -1;
+                goto out;
+        }
+
+        /* Moving the pointer to the end of
+         * /var/run/gluster/snaps/<snap-uuid>/<brick_num>
+         * and assigning '\0' to it.
+         */
+        ptr += strlen(brick_num);
+        *ptr = '\0';
+
+        ret = 0;
+out:
+        if (ret && *brick_mount_path) {
+                GF_FREE (*brick_mount_path);
+                *brick_mount_path = NULL;
+        }
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+/* Check if brick_mount_path is already mounted. If not, mount the device_path
+ * at the brick_mount_path
+ */
+int32_t
+glusterd_mount_brick_paths (char *brick_mount_path, char *device_path)
+{
+        FILE                    *mtab                = NULL;
+        int32_t                  ret                 = -1;
+        runner_t                 runner              = {0, };
+        struct mntent           *entry               = NULL;
+        xlator_t                *this                = NULL;
+        glusterd_conf_t         *priv                = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (brick_mount_path);
+        GF_ASSERT (device_path);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Check if the brick_mount_path is already mounted */
+        entry = glusterd_get_mnt_entry_info (brick_mount_path, mtab);
+        if (entry) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "brick_mount_path (%s) already mounted.",
+                        brick_mount_path);
+                ret = 0;
+                goto out;
+        }
+
+        /* TODO RHEL 6.5 has the logical volumes inactive by default
+         * on reboot. Hence activating the logical vol. Check behaviour
+         * on other systems
+         */
+        /* Activate the snapshot */
+        runinit (&runner);
+        runner_add_args (&runner, "lvchange", "-ay", device_path,
+                         NULL);
+        ret = runner_run (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to activate %s. Error: %s",
+                        device_path, strerror(errno));
+                goto out;
+        } else
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Activating %s successful", device_path);
+
+        /* Mount the snapshot */
+        ret = glusterd_mount_lvm_snapshot (device_path, brick_mount_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to mount lvm snapshot.");
+                goto out;
+        }
+
+out:
+        if (mtab)
+                endmntent (mtab);
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_store_recreate_brick_mounts (glusterd_volinfo_t *volinfo)
+{
+        char                    *brick_mount_path    = NULL;
+        glusterd_brickinfo_t    *brickinfo           = NULL;
+        int32_t                  ret                 = -1;
+        int32_t                  brick_count         = -1;
+        struct stat              st_buf              = {0, };
+        xlator_t                *this                = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (volinfo);
+
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                brick_count++;
+                /* If the brick is not of this node, or its
+                 * snapshot is pending, or the brick is not
+                 * a snapshotted brick, we continue
+                */
+                if ((uuid_compare (brickinfo->uuid, MY_UUID)) ||
+                    (brickinfo->snap_status == -1) ||
+                    (strlen(brickinfo->device_path) == 0))
+                        continue;
+
+                /* Fetch the brick mount path from the brickinfo->path */
+                ret = glusterd_find_brick_mount_path (brickinfo->path,
+                                                      brick_count,
+                                                      &brick_mount_path);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to find brick_mount_path for %s",
+                                brickinfo->path);
+                        goto out;
+                }
+
+                /* Check if the brickinfo path is present.
+                 * If not create the brick_mount_path */
+                ret = lstat (brickinfo->path, &st_buf);
+                if (ret) {
+                        if (errno == ENOENT) {
+                                ret = mkdir_p (brick_mount_path, 0777,
+                                               _gf_true);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to create %s. "
+                                                "Error: %s", brick_mount_path,
+                                                strerror (errno));
+                                        goto out;
+                                }
+                        } else {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Brick Path(%s) not valid. "
+                                        "Error: %s", brickinfo->path,
+                                        strerror(errno));
+                                goto out;
+                        }
+                }
+
+                /* Check if brick_mount_path is already mounted.
+                 * If not, mount the device_path at the brick_mount_path */
+                ret = glusterd_mount_brick_paths (brick_mount_path,
+                                                  brickinfo->device_path);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to mount brick_mount_path");
+                        goto out;
+                }
+
+                if (brick_mount_path) {
+                        GF_FREE (brick_mount_path);
+                        brick_mount_path = NULL;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret && brick_mount_path)
+                GF_FREE (brick_mount_path);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_resolve_snap_bricks (xlator_t  *this, glusterd_snap_t *snap)
 {
@@ -2911,25 +3115,16 @@ out:
 int32_t
 glusterd_store_retrieve_snap (char *snapname)
 {
-        int32_t                ret    = -1;
-        dict_t                *dict   = NULL;
-        glusterd_snap_t       *snap   = NULL;
-        glusterd_conf_t       *priv   = NULL;
-        xlator_t              *this   = NULL;
+        int32_t                ret      = -1;
+        glusterd_snap_t       *snap     = NULL;
+        glusterd_conf_t       *priv     = NULL;
+        xlator_t              *this     = NULL;
 
         this = THIS;
         priv = this->private;
         GF_ASSERT (priv);
         GF_ASSERT (snapname);
 
-        dict = dict_new();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create dict");
-                ret = -1;
-                goto out;
-        }
-
         snap = glusterd_new_snap_object ();
         if (!snap) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to create "
@@ -2952,34 +3147,6 @@ glusterd_store_retrieve_snap (char *snapname)
                 goto out;
         }
 
-        /* Unlike bricks of normal volumes which are resolved at the end of
-           the glusterd restore, the bricks belonging to the snap volumes of
-           each snap should be resolved as part of snapshot restore itself.
-           Because if the snapshot has to be removed, then resolving bricks
-           helps glusterd in understanding what all bricks have its own uuid
-           and killing those bricks.
-        */
-        ret = glusterd_resolve_snap_bricks (this, snap);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "resolving the snap bricks"
-                        " failed (snap: %s)", snap?snap->snapname:"");
-
-        /* When the snapshot command from cli is received, the on disk and
-           in memory structures for the snapshot are created (with the status)
-           being marked as GD_SNAP_STATUS_INIT. Once the backend snapshot is
-           taken, the status is changed to GD_SNAP_STATUS_IN_USE. If glusterd
-           dies after taking the backend snapshot, but before updating the
-           status, then when glusterd comes up, it should treat that snapshot
-           as a failed snapshot and clean it up.
-        */
-        if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
-                ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_true);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING, "failed to remove"
-                                " the snapshot %s", snap->snapname);
-                goto out;
-        }
-
         /* TODO: list_add_order can do 'N-square' comparisions and
            is not efficient. Find a better solution to store the snap
            in order */
@@ -2987,9 +3154,6 @@ glusterd_store_retrieve_snap (char *snapname)
                         glusterd_compare_snap_time);
 
 out:
-        if (dict)
-                dict_unref (dict);
-
         gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
         return ret;
 }
@@ -3616,19 +3780,147 @@ out:
         return ret;
 }
 
+static int32_t
+glusterd_recreate_vol_brick_mounts (xlator_t  *this,
+                                    glusterd_volinfo_t *volinfo)
+{
+        int32_t                  ret       = 0;
+        glusterd_conf_t         *priv      = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_store_recreate_brick_mounts (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to recreate brick mounts "
+                                "for %s", volinfo->volname);
+                        goto out;
+                }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+/* Bricks for snap volumes are hosted at /var/run/gluster/snaps
+ * When a volume is restored, it points to the bricks of the snap
+ * volume it was restored from. Hence on a node restart these
+ * paths need to be recreated and re-mounted
+ */
+int32_t
+glusterd_recreate_all_snap_brick_mounts (xlator_t  *this)
+{
+        int32_t                  ret       = 0;
+        glusterd_conf_t         *priv      = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
+        glusterd_snap_t         *snap      = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Recreate bricks of volumes restored from snaps */
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                /* If the volume is not a restored volume then continue */
+                if (uuid_is_null (volinfo->restored_from_snap))
+                        continue;
+
+                ret = glusterd_recreate_vol_brick_mounts (this, volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to recreate brick mounts "
+                                "for %s", volinfo->volname);
+                        goto out;
+                }
+        }
+
+        /* Recreate bricks of snapshot volumes */
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                list_for_each_entry (volinfo, &snap->volumes, vol_list) {
+                        ret = glusterd_recreate_vol_brick_mounts (this,
+                                                                  volinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to recreate brick mounts "
+                                        "for %s", snap->snapname);
+                                goto out;
+                        }
+                }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+/* When the snapshot command from cli is received, the on disk and
+ * in memory structures for the snapshot are created (with the status)
+ * being marked as GD_SNAP_STATUS_INIT. Once the backend snapshot is
+ * taken, the status is changed to GD_SNAP_STATUS_IN_USE. If glusterd
+ * dies after taking the backend snapshot, but before updating the
+ * status, then when glusterd comes up, it should treat that snapshot
+ * as a failed snapshot and clean it up.
+ */
+int32_t
+glusterd_snap_cleanup (xlator_t  *this)
+{
+        dict_t               *dict = NULL;
+        int32_t               ret  = 0;
+        glusterd_conf_t      *priv = NULL;
+        glusterd_snap_t      *snap = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        dict = dict_new();
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create dict");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
+                        ret = glusterd_snap_remove (dict, snap,
+                                                    _gf_true, _gf_true);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to remove the snapshot %s",
+                                        snap->snapname);
+                                goto out;
+                        }
+                }
+        }
+out:
+        if (dict)
+                dict_unref (dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_resolve_all_bricks (xlator_t  *this)
 {
-        int32_t                 ret = 0;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
+        int32_t                 ret        = 0;
+        glusterd_conf_t         *priv      = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
         glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_snap_t         *snap      = NULL;
 
         GF_ASSERT (this);
         priv = this->private;
 
         GF_ASSERT (priv);
 
+        /* Resolve bricks of volumes */
         list_for_each_entry (volinfo, &priv->volumes, vol_list) {
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         ret = glusterd_resolve_brick (brickinfo);
@@ -3640,9 +3932,20 @@ glusterd_resolve_all_bricks (xlator_t  *this)
                 }
         }
 
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        /* Resolve bricks of snapshot volumes */
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                ret = glusterd_resolve_snap_bricks (this, snap);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "resolving the snap bricks"
+                                " failed for snap: %s",
+                                snap->snapname);
+                        goto out;
+                }
+        }
 
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
         return ret;
 }
 
@@ -3677,6 +3980,20 @@ glusterd_restore ()
         if (ret)
                 goto out;
 
+        ret = glusterd_snap_cleanup (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to perform "
+                        "a cleanup of the snapshots");
+                goto out;
+        }
+
+        ret = glusterd_recreate_all_snap_brick_mounts (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to recreate "
+                        "all snap brick mounts");
+                goto out;
+        }
+
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 0dda3910363..7bd9348aee6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -10673,3 +10673,38 @@ glusterd_compare_volume_name(struct list_head *list1, struct list_head *list2)
         volinfo2 = list_entry(list2, glusterd_volinfo_t, vol_list);
         return strcmp(volinfo1->volname, volinfo2->volname);
 }
+
+int32_t
+glusterd_mount_lvm_snapshot (char *device_path, char *brick_mount_path)
+{
+        char               msg[NAME_MAX] = "";
+        int32_t            ret           = -1;
+        runner_t           runner        = {0, };
+        xlator_t          *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (brick_mount_path);
+        GF_ASSERT (device_path);
+
+
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "mount -o nouuid %s %s",
+                  device_path, brick_mount_path);
+        runner_add_args (&runner, "mount", "-o", "nouuid", device_path,
+                         brick_mount_path, NULL);
+        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
+        ret = runner_run (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mounting the snapshot "
+                        "logical device %s failed (error: %s)", device_path,
+                        strerror (errno));
+                goto out;
+        } else
+                gf_log (this->name, GF_LOG_DEBUG, "mounting the snapshot "
+                        "logical device %s successful", device_path);
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 1a4490ab988..76198428c18 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -731,4 +731,7 @@ gd_restore_snap_volume (dict_t *rsp_dict,
                         glusterd_volinfo_t *orig_vol,
                         glusterd_volinfo_t *snap_vol);
 
+int32_t
+glusterd_mount_lvm_snapshot (char *device_path, char *brick_mount_path);
+
 #endif