glusterd: more sanity checks during restart

* check for validity of pid file before doing pmap search, as in
  few cases, pmap_signin() may take more time.
* remove stale code from 'init()'
* update pmap->last_alloc during restore itself to handle glusterd
  restarts more agnostics to port collisions

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 971 (dynamic volume management)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=971

4 files changed, 26 insertions, 13 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h
index fcad50db84f..a87efed5a8d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.h
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h
@@ -55,7 +55,8 @@ int pmap_registry_bind (xlator_t *this, int port, const char *brickname,
                         gf_pmap_port_type_t type, void *xprt);
 int pmap_registry_remove (xlator_t *this, int port, const char *brickname,
                           gf_pmap_port_type_t type, void *xprt);
-int
-pmap_registry_search (xlator_t *this, const char *brickname,
-                      gf_pmap_port_type_t type);
+int pmap_registry_search (xlator_t *this, const char *brickname,
+                          gf_pmap_port_type_t type);
+struct pmap_registry *pmap_registry_get (xlator_t *this);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index ca19057eb46..351f2bf1a2b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -880,6 +880,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
         char                    tmpkey[4096] = {0,};
         glusterd_store_iter_t   *tmpiter = NULL;
         char                    *tmpvalue = NULL;
+        struct pmap_registry *pmap = NULL;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (volinfo->volname);
@@ -931,6 +932,11 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PORT,
                                     strlen (GLUSTERD_STORE_KEY_BRICK_PORT))) {
                                 gf_string2int (value, &brickinfo->port);
+                                /* This is required to have proper ports
+                                   assigned to bricks after restart */
+                                pmap = pmap_registry_get (THIS);
+                                if (pmap->last_alloc <= brickinfo->port)
+                                        pmap->last_alloc = brickinfo->port + 1;
                         } else {
                                 gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
                                         key);
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 3649ef8639d..0e24aca77ed 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -915,6 +915,19 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
 
         GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
                                     brickinfo->path);
+
+        file = fopen (pidfile, "r+");
+        if (file) {
+                ret = lockf (fileno (file), F_TLOCK, 0);
+                if (ret && ((EAGAIN == errno) || (EACCES == errno))) {
+                        ret = 0;
+                        gf_log ("", GF_LOG_NORMAL, "brick %s:%s "
+                                "already started", brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+        }
+
         ret = pmap_registry_search (this, brickinfo->path,
                                     GF_PMAP_PORT_BRICKSERVER);
         if (ret) {
@@ -932,6 +945,9 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
                                 is_locked = _gf_true;
                         }
                 }
+                /* This means, pmap has the entry, remove it */
+                ret = pmap_registry_remove (this, 0, brickinfo->path,
+                                            GF_PMAP_PORT_BRICKSERVER, NULL);
         }
         unlink (pidfile);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 7ec5cafbbd3..12a2486c3f9 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -228,8 +228,6 @@ init (xlator_t *this)
         glusterd_conf_t   *conf              = NULL;
         data_t            *dir_data          = NULL;
         struct stat        buf               = {0,};
-        char              *port_str          = NULL;
-        int                port_num          = 0;
         char               voldir [PATH_MAX] = {0,};
         char               dirname [PATH_MAX];
         char               cmd_log_filename [PATH_MAX] = {0,};
@@ -347,13 +345,6 @@ init (xlator_t *this)
                 goto out;
         }
 
-        glusterd1_mop_prog.options = this->options;
-        port_str = getenv ("GLUSTERD_LOCAL_PORT");
-        if (port_str) {
-                port_num = atoi (port_str);
-                glusterd1_mop_prog.progport = port_num;
-        }
-
         /*
          * only one (atmost a pair - rdma and socket) listener for
          * glusterd1_mop_prog, gluster_pmap_prog and gluster_handshake_prog.
@@ -377,7 +368,6 @@ init (xlator_t *this)
                 goto out;
         }
 
-        gluster_handshake_prog.options = this->options;
         ret = glusterd_program_register (this, rpc, &gluster_handshake_prog);
         if (ret) {
                 rpcsvc_program_unregister (rpc, &glusterd1_mop_prog);