fixes in 'gluster volume rebalance'

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1481 ()
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1481

5 files changed, 56 insertions, 18 deletions

diff --git a/cli/src/cli3_1-cops.c b/cli/src/cli3_1-cops.c
index 9b977cc3b67..6971a242d91 100644
--- a/cli/src/cli3_1-cops.c
+++ b/cli/src/cli3_1-cops.c
@@ -566,8 +566,20 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 if (rsp.op_ret == -1)
                         cli_out ("failed to get the status of defrag process");
                 else {
-                        cli_out ("rebalanced %"PRId64" files of size %"PRId64
-                                 " (total files scanned %"PRId64")",
+                        char *status = "unknown";
+                        if (rsp.op_errno == 0)
+                                status = "not started";
+                        if (rsp.op_errno == 1)
+                                status = "in progress";
+                        if (rsp.op_errno == 2)
+                                status = "stopped";
+                        if (rsp.op_errno == 3)
+                                status = "completed";
+                        if (rsp.op_errno == 4)
+                                status = "failed";
+
+                        cli_out ("rebalance %s: rebalanced %"PRId64" files of size %"PRId64
+                                 " (total files scanned %"PRId64")", status,
                                  rsp.files, rsp.size, rsp.lookedup_files);
                 }
         }
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index cafc534ad4d..b9389210b99 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -1736,7 +1736,9 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                                         "Out of memory");
                                 goto err;
                         }
-                        local->layout = layout;
+                        local->layout = layout = dht_layout_new (this,
+                                                                 conf->subvolume_cnt);
+
                         dht_selfheal_new_directory (frame, dht_fix_layout_cbk,
                                                     layout);
                         return 0;
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 8c6751184e3..00424f27dc9 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -438,7 +438,7 @@ dht_selfheal_new_directory (call_frame_t *frame,
 	local->selfheal.layout = dht_layout_ref (frame->this, layout);
 
 	dht_layout_sort_volname (layout);
-	dht_selfheal_layout_new_directory (frame, &local->loc, layout);	
+	dht_selfheal_layout_new_directory (frame, &local->loc, layout);
 	dht_selfheal_dir_xattr (frame, &local->loc, layout);
 	return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 02b5ed57638..42acfcc14f6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -657,6 +657,8 @@ glusterd_check_and_rebalance (glusterd_volinfo_t *volinfo, char *dir)
                         getxattr (full_path, "trusted.distribute.fix.layout",
                                   &value, 128);
 
+                        /* add some delay */
+                        usleep (500);
                         ret = glusterd_check_and_rebalance (volinfo, full_path);
                 }
                 if (S_ISREG (stbuf.st_mode) && ((stbuf.st_mode & 01000) == 01000)) {
@@ -737,12 +739,27 @@ glusterd_defrag_start (void *data)
         glusterd_defrag_info_t *defrag  = NULL;
         char                    cmd_str[1024] = {0,};
         int                     ret     = -1;
+        struct stat             stbuf   = {0,};
 
-        /* TODO: make it more generic.. */
         defrag = volinfo->defrag;
         if (!defrag)
                 goto out;
 
+        sleep (5);
+        ret = stat (defrag->mount, &stbuf);
+        if ((ret == -1) && (errno == ENOTCONN)) {
+                /* Wait for some more time before starting rebalance */
+                sleep (7);
+                ret = stat (defrag->mount, &stbuf);
+                if (ret == -1) {
+                        volinfo->defrag_status   = GF_DEFRAG_STATUS_FAILED;
+                        volinfo->rebalance_files = 0;
+                        volinfo->rebalance_data  = 0;
+                        volinfo->lookedup_files  = 0;
+                        goto out;
+                }
+        }
+
         ret = glusterd_check_and_rebalance (volinfo, defrag->mount);
 
         /* TODO: This should run in a thread, and finish the thread when
@@ -754,14 +771,16 @@ glusterd_defrag_start (void *data)
         volinfo->rebalance_data  = defrag->total_data;
         volinfo->lookedup_files  = defrag->num_files_lookedup;
 out:
-        gf_log ("defrag", GF_LOG_NORMAL, "defrag on %s complete",
-                defrag->mount);
+        if (defrag) {
+                gf_log ("defrag", GF_LOG_NORMAL, "defrag on %s complete",
+                        defrag->mount);
 
-        snprintf (cmd_str, 1024, "umount %s", defrag->mount);
-        ret = system (cmd_str);
+                snprintf (cmd_str, 1024, "umount -l %s", defrag->mount);
+                ret = system (cmd_str);
+                LOCK_DESTROY (&defrag->lock);
+                GF_FREE (defrag);
+        }
         volinfo->defrag = NULL;
-        LOCK_DESTROY (&defrag->lock);
-        GF_FREE (defrag);
 
         return NULL;
 }
@@ -808,6 +827,8 @@ glusterd_defrag_status_get (glusterd_volinfo_t *volinfo,
                 rsp->size  = volinfo->rebalance_data;
                 rsp->lookedup_files = volinfo->lookedup_files;
         }
+
+        rsp->op_errno = volinfo->defrag_status;
         rsp->op_ret = 0;
 out:
         return 0;
@@ -892,19 +913,21 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req)
                         goto out;
                 }
 
-                snprintf (cmd_str, 4096, "%s/sbin/glusterfs -f %s/vols/%s/%s-tcp.vol "
-                          "--xlator-option dht0.unhashed-sticky-bit=yes "
-                          "--xlator-option dht0.lookup-unhashed=yes "
-                          "--volume-name quickread %s", GFS_PREFIX,
-                          priv->workdir, cli_req.volname, cli_req.volname,
+                snprintf (cmd_str, 4096, "%s/sbin/glusterfs -s localhost "
+                          "--volfile-id %s --volume-name %s-quick-read "
+                          "--xlator-option *dht.unhashed-sticky-bit=yes "
+                          "--xlator-option *dht.lookup-unhashed=yes %s",
+                          GFS_PREFIX, cli_req.volname, cli_req.volname,
                           defrag->mount);
-                ret = system (cmd_str);
-
+                ret = gf_system (cmd_str);
                 if (ret) {
                         gf_log("glusterd", GF_LOG_DEBUG, "command: %s failed", cmd_str);
                         goto out;
                 }
+
+                volinfo->defrag_status = GF_DEFRAG_STATUS_STARTED;
                 rsp.op_ret = 0;
+
                 ret = pthread_create (&defrag->th, NULL, glusterd_defrag_start,
                                       volinfo);
                 if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index d52f713054d..df3bc9ee167 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -97,6 +97,7 @@ typedef enum gf_defrag_status_ {
         GF_DEFRAG_STATUS_STARTED,
         GF_DEFRAG_STATUS_STOPED,
         GF_DEFRAG_STATUS_COMPLETE,
+        GF_DEFRAG_STATUS_FAILED,
 } gf_defrag_status_t;
 
 struct glusterd_defrag_info_ {