glusterd: restore peer information upon restart

Signed-off-by: Vijay Bellur <vijay@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 1310 ()
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1310

7 files changed, 263 insertions, 26 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index ce4c66182..df3fd6f07 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -41,6 +41,7 @@
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-store.h"
 
 #include "glusterd1.h"
 #include "cli1.h"
@@ -50,18 +51,10 @@
 #include <sys/resource.h>
 #include <inttypes.h>
 
-/* for default_*_cbk functions */
 #include "defaults.c"
 #include "common-utils.h"
 
 
-/*typedef int32_t (*glusterd_mop_t) (call_frame_t *frame,
-                            gf_hdr_common_t *hdr, size_t hdrlen);*/
-
-//static glusterd_mop_t glusterd_ops[GF_MOP_MAXVALUE];
-
-
-
 static int
 glusterd_friend_find_by_hostname (const char *hoststr,
                                   glusterd_peerinfo_t  **peerinfo)
@@ -1417,7 +1410,7 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
 
                 ret = glusterd_friend_add (hostname, friend_req.port,
                                            GD_FRIEND_STATE_BEFRIENDED,
-                                           &uuid, NULL, &peerinfo);
+                                           &uuid, NULL, &peerinfo, 0);
 
                 i++;
         }
@@ -1487,7 +1480,8 @@ glusterd_friend_add (const char *hoststr, int port,
                      glusterd_friend_sm_state_t state,
                      uuid_t *uuid,
                      struct rpc_clnt    *rpc,
-                     glusterd_peerinfo_t **friend)
+                     glusterd_peerinfo_t **friend,
+                     gf_boolean_t restore)
 {
         int                     ret = 0;
         glusterd_conf_t         *priv = NULL;
@@ -1564,9 +1558,12 @@ glusterd_friend_add (const char *hoststr, int port,
 
         }
 
-        gf_log ("glusterd", GF_LOG_NORMAL, "connect returned %d", ret);
+        if (!restore)
+                ret = glusterd_store_update_peerinfo (peerinfo);
+
 
 out:
+        gf_log ("glusterd", GF_LOG_NORMAL, "connect returned %d", ret);
         return ret;
 
 }
@@ -1591,7 +1588,7 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
                         " for host: %s (%d)", hoststr, port);
                 ret = glusterd_friend_add ((char *)hoststr, port,
                                            GD_FRIEND_STATE_DEFAULT,
-                                           NULL, NULL, &peerinfo);
+                                           NULL, NULL, &peerinfo, 0);
         }
 
         ret = glusterd_friend_sm_new_event
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index b70705508..57ec659f9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -914,7 +914,7 @@ replace_brick_start_dst_brick (glusterd_volinfo_t *volinfo, glusterd_brickinfo_t
                 return -1;
         }
 
-        truncate (filename, 0);
+        ret = truncate (filename, 0);
 
         fprintf (file, "volume src-posix\n");
         fprintf (file, "type storage/posix\n");
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index 1ac3f902b..30281040c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -42,6 +42,7 @@
 #include "statedump.h"
 #include "glusterd-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-store.h"
 
 static struct list_head gd_friend_sm_queue;
 
@@ -539,7 +540,7 @@ glusterd_friend_sm ()
                             GD_FRIEND_EVENT_RCVD_FRIEND_REQ == event_type)) {
                                 ret = glusterd_friend_add (NULL, port,
                                                           GD_FRIEND_STATE_DEFAULT,
-                                                          NULL, NULL, &peerinfo);
+                                                          NULL, NULL, &peerinfo, 0);
 
                                 if (ret) {
                                         gf_log ("glusterd", GF_LOG_ERROR, "Unable to add peer, "
@@ -578,6 +579,8 @@ glusterd_friend_sm ()
                                 goto out;
                         }
 
+                        ret = glusterd_store_update_peerinfo (peerinfo);
+
                         GF_FREE (event);
                 }
         }
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index 3d54c4e9d..c49e1f097 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -38,6 +38,15 @@
 //#include "glusterd.h"
 #include "rpcsvc.h"
 
+struct glusterd_store_handle_ {
+        char    *path;
+        int     fd;
+        FILE    *read;
+        FILE    *write;
+};
+
+typedef struct glusterd_store_handle_  glusterd_store_handle_t;
+
 typedef enum glusterd_friend_sm_state_ {
         GD_FRIEND_STATE_DEFAULT = 0,
         GD_FRIEND_STATE_REQ_SENT,
@@ -71,6 +80,7 @@ struct glusterd_peerinfo_ {
         struct list_head                hostnames;
         struct rpc_clnt                 *rpc;
         int                             connected;
+        glusterd_store_handle_t         *shandle;
 };
 
 typedef struct glusterd_peerinfo_ glusterd_peerinfo_t;
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 88a8140d1..592c52c34 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -406,7 +406,8 @@ glusterd_store_retrieve_value (glusterd_store_handle_t *handle,
         char            *str = NULL;
 
         GF_ASSERT (handle);
-        GF_ASSERT (handle->fd > 0);
+
+        handle->fd = open (handle->path, O_RDWR);
 
         if (!handle->read)
                 handle->read = fdopen (handle->fd, "r");
@@ -430,7 +431,8 @@ glusterd_store_retrieve_value (glusterd_store_handle_t *handle,
                         gf_log ("", GF_LOG_DEBUG, "key %s found", key);
                         iter_val = strtok (NULL, "=");
                         ret = 0;
-                        *value = gf_strdup (iter_val);
+                        if (iter_val)
+                                *value = gf_strdup (iter_val);
                         goto out;
                 }
 
@@ -440,6 +442,8 @@ glusterd_store_retrieve_value (glusterd_store_handle_t *handle,
         if (EOF == ret)
                 ret = -1;
 out:
+        if (handle->fd >= 0)
+                close (handle->fd);
 
         return ret;
 }
@@ -523,6 +527,44 @@ out:
 }
 
 int32_t
+glusterd_store_handle_destroy (glusterd_store_handle_t *handle)
+{
+        int32_t                 ret = -1;
+
+        if (!handle) {
+                ret = 0;
+                goto out;
+        }
+
+        GF_FREE (handle->path);
+
+        GF_FREE (handle);
+
+        ret = 0;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+
+int32_t
+glusterd_store_handle_truncate (glusterd_store_handle_t *handle)
+{
+        int32_t         ret = -1;
+
+        GF_ASSERT (handle);
+        GF_ASSERT (handle->path);
+
+        ret = truncate (handle->path, 0);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int32_t
 glusterd_store_uuid ()
 {
         char            str[GLUSTERD_UUID_LEN] = {0,};
@@ -686,7 +728,8 @@ glusterd_store_iter_get_next (glusterd_store_iter_t *iter,
         iter_val = strtok (NULL, "=");
         gf_log ("", GF_LOG_DEBUG, "value %s read", iter_val);
 
-        *value = gf_strdup (iter_val);
+        if (iter_val)
+                *value = gf_strdup (iter_val);
         *key   = gf_strdup (iter_key);
 
         ret = 0;
@@ -950,6 +993,185 @@ out:
         return ret;
 }
 
+int32_t
+glusterd_store_update_peerinfo (glusterd_peerinfo_t *peerinfo)
+{
+        int32_t                         ret = -1;
+        struct  stat                    stbuf = {0,};
+        glusterd_conf_t                 *priv = NULL;
+        char                            peerdir[PATH_MAX] = {0,};
+        char                            filepath[PATH_MAX] = {0,};
+        char                            str[512] = {0,};
+        char                            buf[4096] = {0,};
+        glusterd_peer_hostname_t        *hname = NULL;
+        int                             i = 0;
+        char                            hostname_path[PATH_MAX] = {0,};
+
+        GF_ASSERT (peerinfo);
+
+        priv = THIS->private;
+
+        snprintf (peerdir, PATH_MAX, "%s/peers", priv->workdir);
+
+        ret = stat (peerdir, &stbuf);
+
+        if (-1 == ret) {
+                ret = mkdir (peerdir, 0777);
+                if (ret)
+                        goto out;
+        }
+
+        if (uuid_is_null (peerinfo->uuid)) {
+
+                if (peerinfo->hostname) {
+                        snprintf (filepath, PATH_MAX, "%s/%s", peerdir,
+                                  peerinfo->hostname);
+                } else {
+                        GF_ASSERT (peerinfo->uuid || peerinfo->hostname);
+                }
+        } else {
+                uuid_unparse (peerinfo->uuid, str);
+
+                snprintf (filepath, PATH_MAX, "%s/%s", peerdir, str);
+                snprintf (hostname_path, PATH_MAX, "%s/%s",
+                          peerdir, peerinfo->hostname);
+
+                ret = stat (hostname_path, &stbuf);
+
+                if (!ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Destroying store handle");
+                        glusterd_store_handle_destroy (peerinfo->shandle);
+                        peerinfo->shandle = NULL;
+                }
+        }
+
+
+        if (!peerinfo->shandle) {
+                ret = glusterd_store_handle_new (filepath, &peerinfo->shandle);
+                if (ret)
+                        goto out;
+        } else {
+                ret = glusterd_store_handle_truncate (peerinfo->shandle);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_store_save_value (peerinfo->shandle,
+                                         GLUSTERD_STORE_KEY_PEER_UUID, str);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", peerinfo->state.state);
+        ret = glusterd_store_save_value (peerinfo->shandle,
+                                         GLUSTERD_STORE_KEY_PEER_STATE, buf);
+        if (ret)
+                goto out;
+
+        list_for_each_entry (hname, &peerinfo->hostnames, hostname_list) {
+                i++;
+                snprintf (buf, sizeof (buf), "%s%d",
+                          GLUSTERD_STORE_KEY_PEER_HOSTNAME, i);
+                ret = glusterd_store_save_value (peerinfo->shandle,
+                                                 buf, hname->hostname);
+                if (ret)
+                        goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_peers (xlator_t *this)
+{
+        int32_t                 ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        DIR                     *dir = NULL;
+        struct dirent           *entry = NULL;
+        char                    path[PATH_MAX] = {0,};
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        uuid_t                  uuid = {0,};
+        char                    *hostname = NULL;
+        int32_t                 state = 0;
+        glusterd_store_handle_t *shandle = NULL;
+        char                    filepath[PATH_MAX] = {0,};
+        glusterd_store_iter_t   *iter = NULL;
+        char                    *key = NULL;
+        char                    *value = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                  GLUSTERD_PEER_DIR_PREFIX);
+
+        dir = opendir (path);
+
+        if (!dir) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
+                ret = -1;
+                goto out;
+        }
+
+        glusterd_for_each_entry (entry, dir);
+
+        while (entry) {
+                snprintf (filepath, PATH_MAX, "%s/%s", path, entry->d_name);
+                ret = glusterd_store_handle_new (filepath, &shandle);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_store_iter_new (shandle, &iter);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_store_iter_get_next (iter, &key, &value);
+
+                while (!ret) {
+
+                        if (!strncmp (GLUSTERD_STORE_KEY_PEER_UUID, key,
+                                      strlen (GLUSTERD_STORE_KEY_PEER_UUID))) {
+                                if (value)
+                                        uuid_parse (value, uuid);
+                        } else if (!strncmp (GLUSTERD_STORE_KEY_PEER_STATE,
+                                    key,
+                                    strlen (GLUSTERD_STORE_KEY_PEER_STATE))) {
+                                state = atoi (value);
+                        } else if (!strncmp (GLUSTERD_STORE_KEY_PEER_HOSTNAME,
+                                   key,
+                                   strlen (GLUSTERD_STORE_KEY_PEER_HOSTNAME))) {
+                                hostname = gf_strdup (value);
+                        } else {
+                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
+                                        key);
+                        }
+
+                        GF_FREE (key);
+                        GF_FREE (value);
+
+                        ret = glusterd_store_iter_get_next (iter, &key, &value);
+                }
+
+                (void) glusterd_store_iter_destroy (iter);
+
+                ret = glusterd_friend_add (hostname, 0, state, &uuid,
+                                           NULL, &peerinfo, 1);
+
+                if (ret)
+                        goto out;
+
+                peerinfo->shandle = shandle;
+                glusterd_for_each_entry (entry, dir);
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
 
 int32_t
 glusterd_restore ()
@@ -964,6 +1186,10 @@ glusterd_restore ()
         if (ret)
                 goto out;
 
+        ret = glusterd_store_retrieve_peers (this);
+        if (ret)
+                goto out;
+
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index c3be21502..b9b93d9c9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -50,6 +50,10 @@
 #define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
 #define GLUSTERD_STORE_KEY_BRICK_PATH "path"
 
+#define GLUSTERD_STORE_KEY_PEER_UUID "uuid"
+#define GLUSTERD_STORE_KEY_PEER_HOSTNAME "hostname"
+#define GLUSTERD_STORE_KEY_PEER_STATE "state"
+
 #define glusterd_for_each_entry(entry, dir) \
         do {\
                 entry = readdir (dir);\
@@ -86,5 +90,8 @@ int32_t
 glusterd_retrieve_uuid ();
 
 int32_t
+glusterd_store_update_peerinfo (glusterd_peerinfo_t *peerinfo);
+
+int32_t
 glusterd_restore ();
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index dce3275e7..33850744d 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -45,14 +45,6 @@
 
 #define GLUSTERD_MAX_VOLUME_NAME        1000
 
-struct glusterd_store_handle_ {
-        char    *path;
-        int     fd;
-        FILE    *read;
-        FILE    *write;
-};
-
-typedef struct glusterd_store_handle_  glusterd_store_handle_t;
 
 struct glusterd_store_iter_ {
         int     fd;
@@ -143,6 +135,7 @@ enum glusterd_op_ret {
 #define GLUSTERD_DEFAULT_PORT   6969
 #define GLUSTERD_INFO_FILE      "glusterd.info"
 #define GLUSTERD_VOLUME_DIR_PREFIX "vols"
+#define GLUSTERD_PEER_DIR_PREFIX "peers"
 #define GLUSTERD_VOLUME_INFO_FILE "info"
 #define GLUSTERD_BRICK_INFO_DIR "bricks"
 
@@ -176,7 +169,8 @@ int
 glusterd_friend_add (const char *hoststr, int port,
                      glusterd_friend_sm_state_t state,
                      uuid_t *uuid, struct rpc_clnt    *rpc,
-                     glusterd_peerinfo_t **friend);
+                     glusterd_peerinfo_t **friend,
+                     gf_boolean_t restore);
 
 
 int