glusterd/snapshot: Perform missed snap deletes and restores.

Replacing is_volume_restored(gf_boolean_t) with
restored_from_snap(uuid_t) in glusterd_volinfo_

Also removed gd_restore_snap_volume from glusterd-volgen.c
to glusterd-snapshot.c

Change-Id: Ic615a1658cfaffa98d4590506ac82f20bf709ad6
BUG: 1089906
Signed-off-by: Avra Sengupta <asengupt@redhat.com>
Reviewed-on: http://review.gluster.org/7455
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

8 files changed, 374 insertions, 171 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index b99f2ac89..c1967a29c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -124,7 +124,6 @@ int
 glusterd_snapshot_restore (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
         int                     ret             = -1;
-        char                    *volname        = NULL;
         char                    *snapname       = NULL;
         xlator_t                *this           = NULL;
         glusterd_volinfo_t      *snap_volinfo   = NULL;
@@ -152,7 +151,7 @@ glusterd_snapshot_restore (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         snap = glusterd_find_snap_by_name (snapname);
         if (NULL == snap) {
                 ret = gf_asprintf (op_errstr, "Snap (%s) not found",
-                                        snapname);
+                                   snapname);
                 if (ret < 0) {
                         goto out;
                 }
@@ -195,7 +194,7 @@ glusterd_snapshot_restore (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                  * failure.
                  */
                 gf_log (this->name, GF_LOG_ERROR, "Failed to restore "
-                        "snap for %s volume", volname);
+                        "snap for %s", snapname);
                 goto out;
         }
 
@@ -218,7 +217,7 @@ out:
  * @param rsp_dict      response dictionary
  * @return              Negative value on Failure and 0 in success
  */
-int
+int32_t
 glusterd_snapshot_restore_prevalidate (dict_t *dict, char **op_errstr,
                                        dict_t *rsp_dict)
 {
@@ -1092,11 +1091,13 @@ glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
 
         this = THIS;
         GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
         GF_ASSERT (snap_vol);
 
-        if (!snap_vol) {
-                gf_log (this->name, GF_LOG_ERROR, "snap volinfo is NULL");
+        if ((snap_vol->is_snap_volume == _gf_false) &&
+            (uuid_is_null (snap_vol->restored_from_snap))) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not a snap volume, or a restored snap volume.");
+                ret = 0;
                 goto out;
         }
 
@@ -1113,20 +1114,23 @@ glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
                                 brickinfo->hostname, brickinfo->path,
                                 snap_vol->snapshot->snapname);
 
-                        /* Adding missed delete to the dict */
-                        ret = glusterd_add_missed_snaps_to_dict
+                        if (rsp_dict &&
+                            (snap_vol->is_snap_volume == _gf_true)) {
+                                /* Adding missed delete to the dict */
+                                ret = glusterd_add_missed_snaps_to_dict
                                                    (rsp_dict,
                                                     snap_vol,
                                                     brickinfo,
                                                     brick_count + 1,
                                                     GF_SNAP_OPTION_TYPE_DELETE);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to add missed snapshot info "
-                                        "for %s:%s in the rsp_dict",
-                                        brickinfo->hostname,
-                                        brickinfo->path);
-                                goto out;
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to add missed snapshot "
+                                                "info for %s:%s in the "
+                                                "rsp_dict", brickinfo->hostname,
+                                                brickinfo->path);
+                                        goto out;
+                                }
                         }
 
                         continue;
@@ -5668,3 +5672,126 @@ out:
         gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
         return ret;
 }
+
+/* This function will restore origin volume to it's snap.
+ * The restore operation will simply replace the Gluster origin
+ * volume with the snap volume.
+ * TODO: Multi-volume delete to be done.
+ *       Cleanup in case of restore failure is pending.
+ *
+ * @param orig_vol      volinfo of origin volume
+ * @param snap_vol      volinfo of snapshot volume
+ *
+ * @return 0 on success and negative value on error
+ */
+int
+gd_restore_snap_volume (dict_t *rsp_dict,
+                        glusterd_volinfo_t *orig_vol,
+                        glusterd_volinfo_t *snap_vol)
+{
+        int                     ret             = -1;
+        glusterd_volinfo_t      *new_volinfo    = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_conf_t         *conf           = NULL;
+        glusterd_volinfo_t      *temp_volinfo   = NULL;
+        glusterd_volinfo_t      *voliter        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_VALIDATE_OR_GOTO (this->name, orig_vol, out);
+        GF_VALIDATE_OR_GOTO (this->name, snap_vol, out);
+        snap = snap_vol->snapshot;
+        GF_VALIDATE_OR_GOTO (this->name, snap, out);
+
+        /* Snap volume must be stoped before performing the
+         * restore operation.
+         */
+        ret = glusterd_stop_volume (snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
+                        "snap volume");
+                goto out;
+        }
+
+        /* Create a new volinfo for the restored volume */
+        ret = glusterd_volinfo_dup (snap_vol, &new_volinfo, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create volinfo");
+                goto out;
+        }
+
+        /* Following entries need to be derived from origin volume. */
+        strcpy (new_volinfo->volname, orig_vol->volname);
+        uuid_copy (new_volinfo->volume_id, orig_vol->volume_id);
+        new_volinfo->snap_count = orig_vol->snap_count;
+        new_volinfo->snap_max_hard_limit = orig_vol->snap_max_hard_limit;
+        uuid_copy (new_volinfo->restored_from_snap,
+                   snap_vol->snapshot->snap_id);
+
+        /* Bump the version of the restored volume, so that nodes *
+         * which are done can sync during handshake */
+        new_volinfo->version = orig_vol->version;
+
+        list_for_each_entry_safe (voliter, temp_volinfo,
+                         &orig_vol->snap_volumes, snapvol_list) {
+                list_add_tail (&voliter->snapvol_list,
+                               &new_volinfo->snap_volumes);
+        }
+        /* Copy the snap vol info to the new_volinfo.*/
+        ret = glusterd_snap_volinfo_restore (rsp_dict, new_volinfo, snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to restore snap");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        ret = glusterd_lvm_snapshot_remove (rsp_dict, orig_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to remove "
+                        "LVM backend");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        /* New volinfo always shows the status as created. Therefore
+         * set the status to the original volume's status. */
+        glusterd_set_volume_status (new_volinfo, orig_vol->status);
+
+        /* Once the new_volinfo is completely constructed then delete
+         * the orinal volinfo
+         */
+        ret = glusterd_volinfo_delete (orig_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to delete volinfo");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        list_add_tail (&new_volinfo->vol_list, &conf->volumes);
+
+        /* Now delete the snap entry. As a first step delete the snap
+         * volume information stored in store. */
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_false, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
+                        "snap %s", snap->snapname);
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (new_volinfo,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 9c04aab50..1e39fb27e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -525,10 +525,13 @@ int _storeopts (dict_t *this, char *key, data_t *value, void *data)
 int32_t
 glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
 {
-        char    *str            = NULL;
-        char    buf[PATH_MAX]   = {0,};
-        int32_t ret             = -1;
+        char        *str            = NULL;
+        char         buf[PATH_MAX]  = "";
+        int32_t      ret            = -1;
+        xlator_t    *this           = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (fd > 0);
         GF_ASSERT (volinfo);
 
@@ -576,7 +579,7 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
         snprintf (buf, sizeof (buf), "%s", volinfo->parent_volname);
         ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PARENT_VOLNAME, buf);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to store "
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store "
                         GLUSTERD_STORE_KEY_PARENT_VOLNAME);
                 goto out;
         }
@@ -620,11 +623,11 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
                         goto out;
         }
 
-        snprintf (buf, sizeof (buf), "%d", volinfo->is_volume_restored);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_IS_RESTORED, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
+                                   uuid_utoa (volinfo->restored_from_snap));
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Unable to write is_volume_restored");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to write restored_from_snap");
                 goto out;
         }
 
@@ -632,14 +635,14 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
         ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
                                    buf);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "Unable to write snap-max-hard-limit");
                 goto out;
         }
 
 out:
         if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to write volume "
+                gf_log (this->name, GF_LOG_ERROR, "Unable to write volume "
                         "values for %s", volinfo->volname);
         return ret;
 }
@@ -2417,9 +2420,12 @@ glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
                                 strlen (GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
                         volinfo->snap_max_hard_limit = (uint64_t) atoll (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_IS_RESTORED,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_IS_RESTORED))) {
-                        volinfo->is_volume_restored = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
+                              strlen (GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP))) {
+                        ret = uuid_parse (value, volinfo->restored_from_snap);
+                        if (ret)
+                                gf_log ("", GF_LOG_WARNING,
+                                        "failed to parse restored snap's uuid");
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
                                 strlen (GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
                         strncpy (volinfo->parent_volname, value, sizeof(volinfo->parent_volname) - 1);
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index 19123f005..63d510cbf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -48,7 +48,7 @@ typedef enum glusterd_store_ver_ac_{
 #define GLUSTERD_STORE_KEY_VOL_VERSION          "version"
 #define GLUSTERD_STORE_KEY_VOL_TRANSPORT        "transport-type"
 #define GLUSTERD_STORE_KEY_VOL_ID               "volume-id"
-#define GLUSTERD_STORE_KEY_VOL_IS_RESTORED      "is-volume-restored"
+#define GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP    "restored_from_snap"
 #define GLUSTERD_STORE_KEY_RB_STATUS            "rb_status"
 #define GLUSTERD_STORE_KEY_RB_SRC_BRICK         "rb_src"
 #define GLUSTERD_STORE_KEY_RB_DST_BRICK         "rb_dst"
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index b4644432d..0dda39103 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -2273,14 +2273,12 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
-        snprintf (key, sizeof (key), "volume%d.is_volume_restored", count);
-        ret = dict_set_int32 (dict, key, volinfo->is_volume_restored);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set "
-                        "is_volume_restored option for %s volume",
-                        volinfo->volname);
+        snprintf (key, sizeof (key), "volume%d.restored_from_snap", count);
+        ret = dict_set_dynstr_with_alloc
+                                  (dict, key,
+                                   uuid_utoa (volinfo->restored_from_snap));
+        if (ret)
                 goto out;
-        }
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.brick_count", count);
@@ -3548,6 +3546,7 @@ glusterd_import_volinfo (dict_t *vols, int count,
         char               *volname          = NULL;
         glusterd_volinfo_t *new_volinfo      = NULL;
         char               *volume_id_str    = NULL;
+        char               *restored_snap    = NULL;
         char               msg[2048]         = {0};
         char               *src_brick        = NULL;
         char               *dst_brick        = NULL;
@@ -3713,15 +3712,16 @@ glusterd_import_volinfo (dict_t *vols, int count,
 
         new_volinfo->is_snap_volume = is_snap_volume;
 
-        snprintf (key, sizeof (key), "volume%d.is_volume_restored", count);
-        ret = dict_get_uint32 (vols, key, &new_volinfo->is_volume_restored);
+        snprintf (key, sizeof (key), "volume%d.restored_from_snap", count);
+        ret = dict_get_str (vols, key, &restored_snap);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
-                        "is_volume_restored option for %s",
-                        volname);
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
                 goto out;
         }
 
+        uuid_parse (restored_snap, new_volinfo->restored_from_snap);
+
         snprintf (key, sizeof (key), "volume%d.snap-max-hard-limit", count);
         ret = dict_get_uint64 (vols, key, &new_volinfo->snap_max_hard_limit);
         if (ret) {
@@ -3965,9 +3965,34 @@ int32_t
 glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
                               glusterd_volinfo_t *valid_volinfo)
 {
+        int32_t                  ret            = -1;
+        glusterd_volinfo_t      *temp_volinfo   = NULL;
+        glusterd_volinfo_t      *voliter        = NULL;
+        xlator_t                *this           = NULL;
+
         GF_ASSERT (stale_volinfo);
         GF_ASSERT (valid_volinfo);
 
+        /* Copy snap_volumes list from stale_volinfo to valid_volinfo */
+        valid_volinfo->snap_count = 0;
+        list_for_each_entry_safe (voliter, temp_volinfo,
+                                  &stale_volinfo->snap_volumes, snapvol_list) {
+                list_add_tail (&voliter->snapvol_list,
+                               &valid_volinfo->snap_volumes);
+                valid_volinfo->snap_count++;
+        }
+
+        if ((!uuid_is_null (stale_volinfo->restored_from_snap)) &&
+            (uuid_compare (stale_volinfo->restored_from_snap,
+                           valid_volinfo->restored_from_snap))) {
+                ret = glusterd_lvm_snapshot_remove (NULL, stale_volinfo);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING,
+                               "Failed to remove lvm snapshot for "
+                               "restored volume %s", stale_volinfo->volname);
+                }
+        }
+
         /* If stale volume is in started state, copy the port numbers of the
          * local bricks if they exist in the valid volume information.
          * stop stale bricks. Stale volume information is going to be deleted.
@@ -4220,6 +4245,164 @@ out:
         return ret;
 }
 
+int32_t
+glusterd_perform_missed_op (glusterd_snap_t *snap, int32_t op)
+{
+        dict_t                  *dict         = NULL;
+        int32_t                  ret          = -1;
+        glusterd_conf_t         *priv         = NULL;
+        glusterd_volinfo_t      *snap_volinfo = NULL;
+        glusterd_volinfo_t      *volinfo      = NULL;
+        xlator_t                *this         = NULL;
+        uuid_t                   null_uuid    = {0};
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+
+        dict = dict_new();
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
+                ret = -1;
+                goto out;
+        }
+
+        switch (op) {
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_false);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to remove snap");
+                        goto out;
+                }
+
+                break;
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                /* TODO : As of now there is only volume in snapshot.
+                 * Change this when multiple volume snapshot is introduced
+                 */
+                snap_volinfo = list_entry (snap->volumes.next,
+                                           glusterd_volinfo_t, vol_list);
+
+                /* Find the parent volinfo */
+                ret = glusterd_volinfo_find (snap_volinfo->parent_volname,
+                                             &volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not get volinfo of %s",
+                                snap_volinfo->parent_volname);
+                        goto out;
+                }
+
+                /* Bump down the original volinfo's version, coz it would have
+                 * incremented already due to volume handshake
+                 */
+                volinfo->version--;
+                uuid_copy (volinfo->restored_from_snap, null_uuid);
+
+                /* Perform the restore */
+                ret = gd_restore_snap_volume (dict, volinfo, snap_volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to restore "
+                                "snap for %s", snap->snapname);
+                        volinfo->version++;
+                        goto out;
+                }
+
+                break;
+        default:
+                /* The entry must be a create, delete, or
+                 * restore entry
+                 */
+                gf_log (this->name, GF_LOG_ERROR, "Invalid missed snap entry");
+                ret = -1;
+                goto out;
+        }
+
+out:
+        dict_unref (dict);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Perform missed deletes and restores on this node */
+int32_t
+glusterd_perform_missed_snap_ops ()
+{
+        int32_t                      ret                 = -1;
+        int32_t                      op_status           = -1;
+        glusterd_conf_t             *priv                = NULL;
+        glusterd_missed_snap_info   *missed_snapinfo     = NULL;
+        glusterd_snap_op_t          *snap_opinfo         = NULL;
+        glusterd_snap_t             *snap                = NULL;
+        uuid_t                       snap_uuid           = {0,};
+        xlator_t                    *this                = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
+                             missed_snaps) {
+                /* If the pending snap_op is not for this node then continue */
+                if (strcmp (missed_snapinfo->node_uuid, uuid_utoa (MY_UUID)))
+                        continue;
+
+                /* Find the snap id */
+                uuid_parse (missed_snapinfo->snap_uuid, snap_uuid);
+                snap = NULL;
+                snap = glusterd_find_snap_by_id (snap_uuid);
+                if (!snap) {
+                        /* If the snap is not found, then a delete or a
+                         * restore can't be pending on that snap_uuid.
+                         */
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Not a pending delete or restore op");
+                        continue;
+                }
+
+                op_status = GD_MISSED_SNAP_PENDING;
+                list_for_each_entry (snap_opinfo, &missed_snapinfo->snap_ops,
+                                     snap_ops_list) {
+                        /* If the snap_op is create or its status is
+                         * GD_MISSED_SNAP_DONE then continue
+                         */
+                        if ((snap_opinfo->status == GD_MISSED_SNAP_DONE) ||
+                            (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE))
+                                continue;
+
+                        /* Perform the actual op for the first time for
+                         * this snap, and mark the snap_status as
+                         * GD_MISSED_SNAP_DONE. For other entries for the same
+                         * snap, just mark the entry as done.
+                         */
+                        if (op_status == GD_MISSED_SNAP_PENDING) {
+                                ret = glusterd_perform_missed_op
+                                                             (snap,
+                                                              snap_opinfo->op);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to perform missed snap op");
+                                        goto out;
+                                }
+                                op_status = GD_MISSED_SNAP_DONE;
+                        }
+
+                        snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
 /* Import friend volumes missed_snap_list and update *
  * missed_snap_list if need be */
 int32_t
@@ -4253,6 +4436,16 @@ glusterd_import_friend_missed_snap_list (dict_t *vols)
                 goto out;
         }
 
+        ret = glusterd_perform_missed_snap_ops ();
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to perform snap operations");
+                /* Not going to out at this point coz some *
+                 * missed ops might have been performed. We *
+                 * need to persist the current list *
+                 */
+        }
+
         ret = glusterd_store_update_missed_snaps ();
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index ec65357f9..1a4490ab9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -726,4 +726,9 @@ glusterd_add_missed_snaps_to_export_dict (dict_t *vols);
 int32_t
 glusterd_import_friend_missed_snap_list (dict_t *vols);
 
+int32_t
+gd_restore_snap_volume (dict_t *rsp_dict,
+                        glusterd_volinfo_t *orig_vol,
+                        glusterd_volinfo_t *snap_vol);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 797370a87..ae095bf7c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -4266,128 +4266,3 @@ gd_is_boolean_option (char *key)
 
         return _gf_false;
 }
-
-/* This function will restore origin volume to it's snap.
- * The restore operation will simply replace the Gluster origin
- * volume with the snap volume.
- * TODO: Multi-volume delete to be done.
- *       Cleanup in case of restore failure is pending.
- *
- * @param orig_vol      volinfo of origin volume
- * @param snap_vol      volinfo of snapshot volume
- *
- * @return 0 on success and negative value on error
- */
-int
-gd_restore_snap_volume (dict_t *rsp_dict,
-                        glusterd_volinfo_t *orig_vol,
-                        glusterd_volinfo_t *snap_vol)
-{
-        int                     ret             = -1;
-        glusterd_volinfo_t      *new_volinfo    = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        xlator_t                *this           = NULL;
-        glusterd_conf_t         *conf           = NULL;
-        glusterd_volinfo_t      *temp_volinfo   = NULL;
-        glusterd_volinfo_t      *voliter        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_VALIDATE_OR_GOTO (this->name, orig_vol, out);
-        GF_VALIDATE_OR_GOTO (this->name, snap_vol, out);
-        snap = snap_vol->snapshot;
-        GF_VALIDATE_OR_GOTO (this->name, snap, out);
-
-        /* Snap volume must be stoped before performing the
-         * restore operation.
-         */
-        ret = glusterd_stop_volume (snap_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
-                        "snap volume(%s)", snap_vol->volname);
-                goto out;
-        }
-
-        /* Create a new volinfo for the restored volume */
-        ret = glusterd_volinfo_dup (snap_vol, &new_volinfo, _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create volinfo");
-                goto out;
-        }
-
-        /* Following entries need to be derived from origin volume. */
-        strcpy (new_volinfo->volname, orig_vol->volname);
-        uuid_copy (new_volinfo->volume_id, orig_vol->volume_id);
-        new_volinfo->snap_count = orig_vol->snap_count;
-        new_volinfo->snap_max_hard_limit = orig_vol->snap_max_hard_limit;
-        new_volinfo->is_volume_restored = _gf_true;
-
-        /* Bump the version of the restored volume, so that nodes *
-         * which are done can sync during handshake */
-        new_volinfo->version = orig_vol->version;
-
-        list_for_each_entry_safe (voliter, temp_volinfo,
-                         &orig_vol->snap_volumes, snapvol_list) {
-                list_add_tail (&voliter->snapvol_list,
-                               &new_volinfo->snap_volumes);
-        }
-        /* Copy the snap vol info to the new_volinfo.*/
-        ret = glusterd_snap_volinfo_restore (rsp_dict, new_volinfo, snap_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to restore snap");
-                (void)glusterd_volinfo_delete (new_volinfo);
-                goto out;
-        }
-
-        /* If the orig_vol is already restored then we should delete
-         * the backend LVMs */
-        if (orig_vol->is_volume_restored) {
-                ret = glusterd_lvm_snapshot_remove (rsp_dict, orig_vol);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to remove "
-                                "LVM backend");
-                        (void)glusterd_volinfo_delete (new_volinfo);
-                        goto out;
-                }
-        }
-
-        /* Once the new_volinfo is completely constructed then delete
-         * the orinal volinfo
-         */
-        ret = glusterd_volinfo_delete (orig_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to delete volinfo");
-                (void)glusterd_volinfo_delete (new_volinfo);
-                goto out;
-        }
-        /* New volinfo always shows the status as created. Therefore
-         * set the status to stop. */
-        glusterd_set_volume_status (new_volinfo, GLUSTERD_STATUS_STOPPED);
-
-        list_add_tail (&new_volinfo->vol_list, &conf->volumes);
-
-        /* Now delete the snap entry. As a first step delete the snap
-         * volume information stored in store. */
-        ret = glusterd_snap_remove (rsp_dict, snap, _gf_false, _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
-                        "snap %s", snap->snapname);
-                goto out;
-        }
-
-        ret = glusterd_store_volinfo (new_volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo");
-                goto out;
-        }
-
-        ret = 0;
-out:
-
-        return ret;
-}
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 1f9416106..ef92087fc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -172,7 +172,4 @@ gd_is_xlator_option (char *key);
 gf_boolean_t
 gd_is_boolean_option (char *key);
 
-int gd_restore_snap_volume (dict_t *rsp_dict,
-                            glusterd_volinfo_t *orig_vol,
-                            glusterd_volinfo_t *snap_vol);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index d50fa03d2..b53d8e412 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -298,7 +298,7 @@ struct glusterd_volinfo_ {
         char                      volname[GLUSTERD_MAX_VOLUME_NAME];
         gf_boolean_t              is_snap_volume;
         glusterd_snap_t          *snapshot;
-        gf_boolean_t              is_volume_restored;
+        uuid_t                    restored_from_snap;
         char                      parent_volname[GLUSTERD_MAX_VOLUME_NAME];
                                          /* In case of a snap volume
                                             i.e (is_snap_volume == TRUE) this