glusterd/snapshot: Recording the snapshots missed in each brick.

Persisting missing snapshot info on disk as well as in memory in
the following format:
-------------NODE-UUID--------------:---------SNAP-UUID--------------=BRICKNUM:-------BRICKPATH--------:OPERATION:STATUS
927cb5fe-63da-48f5-82f6-e6a09ddc81c4:a17b4fe42c5a45f7a916438643edaa13=   3    :/brick/brick-dirs/brick3:    1    :   1
927cb5fe-63da-48f5-82f6-e6a09ddc81c4:a17b4fe42c5a45f7a916438643edaa13=   3    :/brick/brick-dirs/brick3:    3    :   1
927cb5fe-63da-48f5-82f6-e6a09ddc81c4:83a3cc05453b46b2a7eda4c9a9208638=   3    :/brick/brick-dirs/brick3:    1    :   1

This data will be stored on disk at /var/lib/glusterd/snaps/missed_snaps_list

In memory we maintain the data as a list of glusterd_missed_snap_info
in conf, the key for this list are the first two fields,
i.e NODE-UUID:SNAP-UUID.

For every NODE-UUID:SNAP-UUID, there can be multiple operations missed
on multiple bricks. So we maintain a list of glusterd_snap_op_t
for evert node of glusterd_missed_snap_info

This list is maintained or updated during snapshot create, delete, and restore
operations which are the only operations that if missed, are recorded in this
list.

During snapshot create, if a node is down, or a brick is down, we don't
receive their mount point infos. snap_status of such bricks is marked as
-1, and their brick details are added to this list.

During snapshot delete, we check from originator node, if any other
nodes, holding bricks of the said snap are down. Those are also added to the list.
Also if the node is up, but the snapshot was pending for a snap
brick, and its snap_status is -1, we add that to the list too.
When a subsequent delete entry is processed for an already existing
create entry, we just mark the create entries status as done (2), and don't
add the delete entry to the list.

During snapshot restore, we check from originator node, if any other
nodes, holding bricks of the said snap are down. Those are also added to the list.
Also if the node is up, but the snapshot was pending for a snap
brick, and its snap_status is -1, we add that to the list too.

Change-Id: I22578d14f81a54e13f6832966b70cd4cfdfd5b44
Signed-off-by: Avra Sengupta <asengupt@redhat.com>
Reviewed-on: http://review.gluster.org/7208
Reviewed-by: Vijaikumar Mallikarjuna <vmallika@redhat.com>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Tested-by: Rajesh Joseph <rjoseph@redhat.com>

1 files changed, 281 insertions, 7 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 256519598..2340c4030 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -802,6 +802,21 @@ glusterd_store_node_state_path_set (glusterd_volinfo_t *volinfo,
 }
 
 static void
+glusterd_store_missed_snaps_list_path_set (char *missed_snaps_list,
+                                           size_t len)
+{
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (missed_snaps_list);
+        GF_ASSERT (len <= PATH_MAX);
+
+        snprintf (missed_snaps_list, len, "%s/snaps/"
+                  GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
+}
+
+static void
 glusterd_store_snapfpath_set (glusterd_snap_t *snap, char *snap_fpath,
                               size_t len)
 {
@@ -861,6 +876,29 @@ glusterd_store_create_nodestate_sh_on_absence (glusterd_volinfo_t *volinfo)
         return ret;
 }
 
+static int32_t
+glusterd_store_create_missed_snaps_list_shandle_on_absence ()
+{
+        char               missed_snaps_list[PATH_MAX] = "";
+        int32_t            ret                         = -1;
+        glusterd_conf_t   *priv                        = NULL;
+        xlator_t          *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        glusterd_store_missed_snaps_list_path_set (missed_snaps_list,
+                                                   sizeof(missed_snaps_list));
+
+        ret = gf_store_handle_create_on_absence
+                                            (&priv->missed_snaps_list_shandle,
+                                             missed_snaps_list);
+        return ret;
+}
+
 int32_t
 glusterd_store_create_snap_shandle_on_absence (glusterd_snap_t *snap)
 {
@@ -2748,6 +2786,7 @@ int32_t
 glusterd_store_retrieve_snap (char *snapname)
 {
         int32_t                ret    = -1;
+        dict_t                *dict   = NULL;
         glusterd_snap_t       *snap   = NULL;
         glusterd_conf_t       *priv   = NULL;
         xlator_t              *this   = NULL;
@@ -2757,6 +2796,14 @@ glusterd_store_retrieve_snap (char *snapname)
         GF_ASSERT (priv);
         GF_ASSERT (snapname);
 
+        dict = dict_new();
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create dict");
+                ret = -1;
+                goto out;
+        }
+
         snap = glusterd_new_snap_object ();
         if (!snap) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to create "
@@ -2800,7 +2847,7 @@ glusterd_store_retrieve_snap (char *snapname)
            as a failed snapshot and clean it up.
         */
         if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
-                ret = glusterd_snap_remove (snap, _gf_true, _gf_true);
+                ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_true);
                 if (ret)
                         gf_log (this->name, GF_LOG_WARNING, "failed to remove"
                                 " the snapshot %s", snap->snapname);
@@ -2814,8 +2861,104 @@ glusterd_store_retrieve_snap (char *snapname)
                         glusterd_compare_snap_time);
 
 out:
-        gf_log ("", GF_LOG_TRACE, "Returning with %d", ret);
+        if (dict)
+                dict_unref (dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+/* Read the missed_snap_list and update the in-memory structs */
+int32_t
+glusterd_store_retrieve_missed_snaps_list (xlator_t  *this)
+{
+        char                   buf[PATH_MAX]    = "";
+        char                   path[PATH_MAX]   = "";
+        char                  *missed_node_info = NULL;
+        char                  *brick_path       = NULL;
+        char                  *value            = NULL;
+        char                  *save_ptr         = NULL;
+        FILE                  *fp               = NULL;
+        int32_t                brick_num        = -1;
+        int32_t                snap_op          = -1;
+        int32_t                snap_status      = -1;
+        int32_t                ret              = -1;
+        glusterd_conf_t       *priv             = NULL;
+        gf_store_op_errno_t    store_errno      = GD_STORE_SUCCESS;
+
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Get the path of the missed_snap_list */
+        glusterd_store_missed_snaps_list_path_set (path, sizeof(path));
+
+        fp = fopen (path, "r");
+        if (!fp) {
+                /* If errno is ENOENT then there are no missed snaps yet */
+                if (errno != ENOENT) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to open %s. "
+                                "Error: %s", path, strerror(errno));
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "No missed snaps list.");
+                        ret = 0;
+                }
+                goto out;
+        }
+
+        do {
+                ret = gf_store_read_and_tokenize (fp, buf,
+                                                  &missed_node_info, &value,
+                                                  &store_errno);
+                if (ret) {
+                        if (store_errno == GD_STORE_EOF) {
+                                gf_log (this->name,
+                                        GF_LOG_DEBUG,
+                                        "EOF for missed_snap_list");
+                                ret = 0;
+                                break;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to fetch data from "
+                                "missed_snaps_list. Error: %s",
+                                gf_store_strerror (store_errno));
+                        goto out;
+                }
+
+                /* Fetch the brick_num, brick_path, snap_op and snap status */
+                brick_num = atoi(strtok_r (value, ":", &save_ptr));
+                brick_path = strtok_r (NULL, ":", &save_ptr);
+                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
+                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
 
+                if (!missed_node_info || !brick_path ||
+                    brick_num < 1 || snap_op < 1 ||
+                    snap_status < 1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Invalid missed_snap_entry");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_store_missed_snaps_list (missed_node_info,
+                                                        brick_num,
+                                                        brick_path,
+                                                        snap_op,
+                                                        snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store missed snaps_list");
+                        goto out;
+                }
+
+        } while (store_errno == GD_STORE_SUCCESS);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
         return ret;
 }
 
@@ -2850,16 +2993,27 @@ glusterd_store_retrieve_snaps (xlator_t  *this)
         glusterd_for_each_entry (entry, dir);
 
         while (entry) {
-                ret = glusterd_store_retrieve_snap (entry->d_name);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to restore "
-                                "snapshot: %s", entry->d_name);
-                        goto out;
+                if (entry->d_type == DT_DIR) {
+                        ret = glusterd_store_retrieve_snap (entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to restore snapshot: %s",
+                                        entry->d_name);
+                                goto out;
+                        }
                 }
 
                 glusterd_for_each_entry (entry, dir);
         }
 
+        /* Retrieve missed_snaps_list */
+        ret = glusterd_store_retrieve_missed_snaps_list (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Failed to retrieve missed_snaps_list");
+                goto out;
+        }
+
 out:
         if (dir)
                 closedir (dir);
@@ -2868,6 +3022,126 @@ out:
         return ret;
 }
 
+/* Writes all the contents of conf->missed_snap_list */
+int32_t
+glusterd_store_write_missed_snapinfo (int32_t fd)
+{
+        char                           value[PATH_MAX]             = "";
+        int32_t                        ret                         = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
+        glusterd_snap_op_t            *snap_opinfo                 = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Write the missed_snap_entry */
+        list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
+                             missed_snaps) {
+                list_for_each_entry (snap_opinfo,
+                                     &missed_snapinfo->snap_ops,
+                                     snap_ops_list) {
+                        snprintf (value, sizeof(value), "%d:%s:%d:%d",
+                                  snap_opinfo->brick_num,
+                                  snap_opinfo->brick_path,
+                                  snap_opinfo->op, snap_opinfo->status);
+                        ret = gf_store_save_value
+                                           (fd,
+                                            missed_snapinfo->node_snap_info,
+                                            value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to write missed snapinfo");
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Adds the missed snap entries to the in-memory conf->missed_snap_list *
+ * and writes them to disk */
+int32_t
+glusterd_store_update_missed_snaps (dict_t *dict, int32_t missed_snap_count)
+{
+        int32_t                        fd                          = -1;
+        int32_t                        ret                         = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (missed_snap_count < 1) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_store_create_missed_snaps_list_shandle_on_absence ();
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to obtain "
+                        "missed_snaps_list store handle.");
+                goto out;
+        }
+
+        fd = gf_store_mkstemp (priv->missed_snaps_list_shandle);
+        if (fd <= 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create tmp file");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_add_missed_snaps_to_list (dict, missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to add missed snaps to list");
+                goto out;
+        }
+
+        ret = glusterd_store_write_missed_snapinfo (fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to write missed snaps to disk");
+                goto out;
+        }
+
+        ret = gf_store_rename_tmppath (priv->missed_snaps_list_shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to rename the tmp file");
+                goto out;
+        }
+out:
+        if (ret && (fd > 0)) {
+                ret = gf_store_unlink_tmppath (priv->missed_snaps_list_shandle);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to unlink the tmp file");
+                }
+                ret = -1;
+        }
+
+        if (fd > 0)
+                close (fd);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
 {