glusterd: volume-start, add-brick and remove-brick to use synctask framework

- Added volume-id validation to glusterd-syncop code.
- All daemons are restarted using synctasks in init().
- glusterd_brick_start has wait/nowait variants to support
  volume commands using synctask framework and those that aren't.

Change-Id: Ieec26fe1ea7e5faac88cc7798d93e4cc2b399d34
BUG: 862834
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/3969
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

10 files changed, 193 insertions, 106 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index d0a97b1380a..5e93c061da3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -514,7 +514,7 @@ brick_val:
                                 "failed to set the new type in dict");
         }
 
-        ret = glusterd_op_begin (req, GD_OP_ADD_BRICK, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
 
 out:
         if (ret) {
@@ -531,9 +531,6 @@ out:
                 ret = 0; //sent error to cli, prevent second reply
         }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         free (cli_req.dict.dict_val); //its malloced by xdr
 
         return ret;
@@ -806,7 +803,7 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                 }
         }
 
-        ret = glusterd_op_begin (req, GD_OP_REMOVE_BRICK, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_REMOVE_BRICK, dict);
 
 out:
         if (ret) {
@@ -828,9 +825,6 @@ out:
         GF_FREE (brick_list);
         free (cli_req.dict.dict_val); //its malloced by xdr
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         return ret;
 }
 
@@ -938,7 +932,8 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                 if (ret)
                         goto out;
 
-                ret = glusterd_brick_start (volinfo, brickinfo);
+                ret = glusterd_brick_start (volinfo, brickinfo,
+                                            _gf_true);
                 if (ret)
                         goto out;
                 i++;
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 663cee307ee..cd395c49e73 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -2963,7 +2963,7 @@ struct rpcsvc_program gd_svc_mgmt_prog = {
         .progver   = GD_MGMT_VERSION,
         .numactors = GLUSTERD_MGMT_MAXVALUE,
         .actors    = gd_svc_mgmt_actors,
-	.synctask  = _gf_false,
+	.synctask  = _gf_true,
 };
 
 rpcsvc_actor_t gd_svc_peer_actors[] = {
@@ -3022,5 +3022,5 @@ struct rpcsvc_program gd_svc_cli_prog = {
         .progver   = GLUSTER_CLI_VERSION,
         .numactors = GLUSTER_CLI_MAXVALUE,
         .actors    = gd_svc_cli_actors,
-	.synctask  = _gf_false,
+	.synctask  = _gf_true,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 04b9cfe462d..d117b05af7e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -1022,7 +1022,7 @@ glusterd_start_bricks (glusterd_volinfo_t *volinfo)
         glusterd_brickinfo_t                    *brickinfo = NULL;
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_brick_start (volinfo, brickinfo))
+                if (glusterd_brick_start (volinfo, brickinfo, _gf_false))
                         return -1;
         }
 
@@ -1844,7 +1844,7 @@ out:
 }
 
 int
-glusterd_op_build_payload (dict_t **req, char **op_errstr)
+glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
 {
         int                     ret = -1;
         void                    *ctx = NULL;
@@ -1860,14 +1860,24 @@ glusterd_op_build_payload (dict_t **req, char **op_errstr)
         if (!req_dict)
                 goto out;
 
-        op  = glusterd_op_get_op ();
-        ctx = (void*)glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR, "Null Context for "
-                        "op %d", op);
-                ret = -1;
-                goto out;
+        if (!op_ctx) {
+                op  = glusterd_op_get_op ();
+                ctx = (void*)glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log ("", GF_LOG_ERROR, "Null Context for "
+                                "op %d", op);
+                        ret = -1;
+                        goto out;
+                }
+
+        } else {
+#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
+                ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, (int32_t*)&op);
+                if (ret)
+                        goto out;
+                ctx = op_ctx;
         }
+#undef GD_SYNC_OPCODE_KEY
 
         switch (op) {
                 case GD_OP_CREATE_VOLUME:
@@ -2002,7 +2012,7 @@ glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 
         op = glusterd_op_get_op ();
 
-        ret = glusterd_op_build_payload (&dict, &op_errstr);
+        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_ERROR, "Building payload failed");
                 opinfo.op_errstr = op_errstr;
@@ -2280,7 +2290,8 @@ out:
 }
 
 static int
-glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,  glusterd_commit_hook_type_t type)
+glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_commit_hook_type_t type)
 {
         glusterd_conf_t *priv                   = NULL;
         char            hookdir[PATH_MAX]       = {0, };
@@ -2353,7 +2364,7 @@ glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
         op      = glusterd_op_get_op ();
         op_dict = glusterd_op_get_ctx ();
 
-        ret = glusterd_op_build_payload (&dict, &op_errstr);
+        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_ERROR, "Building payload failed");
                 opinfo.op_errstr = op_errstr;
@@ -2767,7 +2778,7 @@ glusterd_need_brick_op (glusterd_op_t op)
         return ret;
 }
 
-static dict_t*
+dict_t*
 glusterd_op_init_commit_rsp_dict (glusterd_op_t op)
 {
         dict_t                  *rsp_dict = NULL;
@@ -4106,7 +4117,8 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
                 op = glusterd_op_get_op ();
                 req_ctx->op = op;
                 uuid_copy (req_ctx->uuid, MY_UUID);
-                ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr);
+                ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr,
+                                                 NULL);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Building payload failed");
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index 62ebcbabe6e..558c2d1f755 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -181,7 +181,7 @@ int32_t
 glusterd_op_set_op (glusterd_op_t op);
 
 int
-glusterd_op_build_payload (dict_t **req, char **op_errstr);
+glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx);
 
 int32_t
 glusterd_op_stage_validate (glusterd_op_t op, dict_t *req, char **op_errstr,
@@ -242,6 +242,10 @@ int32_t
 glusterd_handle_brick_rsp (void *pending_entry, glusterd_op_t op,
                            dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
                            gd_node_type type);
+
+dict_t*
+glusterd_op_init_commit_rsp_dict (glusterd_op_t op);
+
 int32_t
 glusterd_op_init_ctx (glusterd_op_t op);
 int32_t
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 5c6e26d3570..29c79872908 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -606,7 +606,8 @@ rb_src_brick_restart (glusterd_volinfo_t *volinfo,
         }
 
         sleep (2);
-        ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo);
+        ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo,
+                                              _gf_false);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to start "
                         "glusterfs, ret: %d", ret);
@@ -1379,7 +1380,7 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
                 goto out;
 
         if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                ret = glusterd_brick_start (volinfo, new_brickinfo);
+                ret = glusterd_brick_start (volinfo, new_brickinfo, _gf_false);
                 if (ret)
                         goto out;
         }
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index dd6659971ba..7c4ae8903cc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -310,6 +310,7 @@ out:
 
 }
 
+/*TODO: Need to add syncop for brick ops*/
 int32_t
 gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                          int count, void *myframe)
@@ -411,25 +412,80 @@ out:
 }
 
 
+static int
+glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp,
+                               char *op_errstr)
+{
+        int ret = -1;
+
+        switch (op) {
+        case GD_OP_REPLACE_BRICK:
+                ret = glusterd_rb_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_SYNC_VOLUME:
+                ret = glusterd_sync_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_GSYNC_SET:
+                ret = glusterd_gsync_use_rsp_dict (aggr, rsp, op_errstr);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_STATUS_VOLUME:
+                ret = glusterd_volume_status_copy_to_op_ctx_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = glusterd_volume_rebalance_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_HEAL_VOLUME:
+                ret = glusterd_volume_heal_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+
+        break;
+
+        default:
+        break;
+        }
+out:
+        return ret;
+}
+
 int
-gd_sync_task_begin (void *data)
+gd_sync_task_begin (dict_t *op_ctx, char **op_errstr)
 {
         int                  ret      = -1;
-        dict_t              *dict     = NULL;
+        dict_t              *req_dict     = NULL;
         dict_t              *rsp_dict = NULL;
         glusterd_peerinfo_t *peerinfo = NULL;
         glusterd_conf_t     *conf     = NULL;
         uuid_t               tmp_uuid = {0,};
-        char                *errstr   = NULL;
         glusterd_op_t        op       = 0;
         int32_t              tmp_op   = 0;
         gf_boolean_t         local_locked = _gf_false;
 
         conf = THIS->private;
 
-        dict = data;
-
-        ret = dict_get_int32 (dict, GD_SYNC_OPCODE_KEY, &tmp_op);
+        ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
         if (ret)
                 goto out;
 
@@ -455,40 +511,60 @@ gd_sync_task_begin (void *data)
                 /* TODO: Only on lock successful nodes it should unlock */
         }
 
+        ret = glusterd_op_build_payload (&req_dict, op_errstr, op_ctx);
+        if (ret)
+                goto out;
+
         /* stage op */
-        ret = glusterd_op_stage_validate (op, dict, &errstr, rsp_dict);
+        ret = glusterd_op_stage_validate (op, req_dict, op_errstr, rsp_dict);
         if (ret)
                 goto out;
 
         list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
                 ret = gd_syncop_mgmt_stage_op (peerinfo->rpc,
                                                conf->uuid, tmp_uuid,
-                                               op, dict, &rsp_dict, &errstr);
+                                               op, req_dict, &rsp_dict,
+                                               op_errstr);
                 if (ret) {
-                        if (errstr)
-                                ret = dict_set_dynstr (dict, "error", errstr);
+                        if (*op_errstr)
+                                ret = dict_set_dynstr (req_dict, "error",
+                                                       *op_errstr);
 
                         ret = -1;
                         goto out;
                 }
+
+                if (op == GD_OP_REPLACE_BRICK)
+                        (void) glusterd_syncop_aggr_rsp_dict (op, op_ctx,
+                                                              rsp_dict,
+                                                              *op_errstr);
+
+                if (rsp_dict)
+                        dict_unref (rsp_dict);
         }
 
         /* commit op */
-        ret = glusterd_op_commit_perform (op, dict, &errstr, rsp_dict);
+        ret = glusterd_op_commit_perform (op, req_dict, op_errstr, rsp_dict);
         if (ret)
                 goto out;
 
         list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
                 ret = gd_syncop_mgmt_commit_op (peerinfo->rpc,
                                                 conf->uuid, tmp_uuid,
-                                                op, dict, &rsp_dict, &errstr);
+                                                op, req_dict, &rsp_dict,
+                                                op_errstr);
                 if (ret) {
-                        if (errstr)
-                                ret = dict_set_dynstr (dict, "error", errstr);
+                        if (*op_errstr)
+                                ret = dict_set_dynstr (req_dict, "error",
+                                                       *op_errstr);
 
                         ret = -1;
                         goto out;
                 }
+                (void) glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict,
+                                                      *op_errstr);
+                if (rsp_dict)
+                        dict_unref (rsp_dict);
         }
 
         ret = 0;
@@ -509,57 +585,22 @@ out:
                 glusterd_unlock (conf->uuid);
         }
 
+        if (req_dict)
+                dict_unref (req_dict);
+
         if (rsp_dict)
                 dict_unref (rsp_dict);
 
         return ret;
 }
 
-int
-gd_sync_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
-{
-        int               ret    = 0;
-        dict_t           *dict   = NULL;
-        rpcsvc_request_t *req    = NULL;
-        int32_t           tmp_op = 0;
-        glusterd_op_t     op     = 0;
-
-        dict = data;
-
-        req = sync_frame->local;
-        sync_frame->local = NULL;
-
-        ret = dict_get_int32 (dict, GD_SYNC_OPCODE_KEY, &tmp_op);
-        if (ret)
-                goto out;
-        op = tmp_op;
-
-        ret = glusterd_op_send_cli_response (op, op_ret, 0, req, NULL,
-                                             "operation failed");
-
-out:
-        if (dict)
-                dict_unref (dict);
-
-        STACK_DESTROY (sync_frame->root);
-
-        return ret;
-}
-
-
 int32_t
 glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
                             void *dict)
 {
         int              ret = 0;
-        call_frame_t    *dummy_frame = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-
-        dummy_frame = create_frame (THIS, THIS->ctx->pool);
-        if (!dummy_frame)
-                goto out;
-
-        dummy_frame->local = req;
+        int              op_ret = 0;
+        char             *op_errstr = NULL;
 
         ret = dict_set_int32 (dict, GD_SYNC_OPCODE_KEY, op);
         if (ret) {
@@ -568,11 +609,15 @@ glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
                 goto out;
         }
 
-        ctx = THIS->ctx;
-
-        ret = synctask_new (ctx->env, gd_sync_task_begin,
-                            gd_sync_task_completion,
-                            dummy_frame, dict);
+        op_ret = gd_sync_task_begin (dict, &op_errstr);
+        glusterd_op_send_cli_response (op, op_ret, 0, req, NULL,
+                                       op_errstr);
+        ret = 0;
 out:
+        if (dict)
+            dict_unref (dict);
+        if (op_errstr)
+                GF_FREE (op_errstr);
+
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 962e2638f4a..4178ad9db43 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1155,7 +1155,8 @@ out:
 
 int32_t
 glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
-                                 glusterd_brickinfo_t  *brickinfo)
+                                 glusterd_brickinfo_t  *brickinfo,
+                                 gf_boolean_t wait)
 {
         int32_t                 ret = -1;
         xlator_t                *this = NULL;
@@ -1306,7 +1307,11 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
                 runner_add_arg (&runner, "--mem-accounting");
 
         runner_log (&runner, "", GF_LOG_DEBUG, "Starting GlusterFS");
-        ret = runner_run_nowait (&runner);
+        if (wait)
+                ret = runner_run (&runner);
+        else
+                ret = runner_run_nowait (&runner);
+
         if (ret)
                 goto out;
 
@@ -3460,7 +3465,8 @@ out:
 
 int
 glusterd_brick_start (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo)
+                      glusterd_brickinfo_t *brickinfo,
+                      gf_boolean_t wait)
 {
         int                                     ret   = -1;
         xlator_t                                *this = NULL;
@@ -3488,7 +3494,7 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo,
                 ret = 0;
                 goto out;
         }
-        ret = glusterd_volume_start_glusterfs (volinfo, brickinfo);
+        ret = glusterd_volume_start_glusterfs (volinfo, brickinfo, wait);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to start "
                         "glusterfs, ret: %d", ret);
@@ -3513,7 +3519,8 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
                 if (volinfo->status == GLUSTERD_STATUS_STARTED) {
                         list_for_each_entry (brickinfo, &volinfo->bricks,
                                              brick_list) {
-                                glusterd_brick_start (volinfo, brickinfo);
+                                glusterd_brick_start (volinfo, brickinfo,
+                                                     _gf_true);
                         }
                         start_nodesvcs = _gf_true;
                 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 2441328964c..1cda32b5719 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -124,7 +124,8 @@ glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo);
 
 int32_t
 glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
-                                 glusterd_brickinfo_t   *brickinfo);
+                                 glusterd_brickinfo_t   *brickinfo,
+                                 gf_boolean_t wait);
 
 int32_t
 glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
@@ -242,8 +243,8 @@ int32_t
 glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                              dict_t  *dict, int32_t count);
 int
-glusterd_get_brickinfo (xlator_t *this, const char *brickname, 
-                        int port, gf_boolean_t localhost, 
+glusterd_get_brickinfo (xlator_t *this, const char *brickname,
+                        int port, gf_boolean_t localhost,
                         glusterd_brickinfo_t **brickinfo);
 
 void
@@ -267,7 +268,8 @@ glusterd_all_volume_cond_check (glusterd_condition_func func, int status,
                                 void *ctx);
 int
 glusterd_brick_start (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo);
+                      glusterd_brickinfo_t *brickinfo,
+                      gf_boolean_t wait);
 int
 glusterd_brick_stop (glusterd_volinfo_t *volinfo,
                      glusterd_brickinfo_t *brickinfo,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index d167ed10022..d42694353af 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -260,14 +260,11 @@ glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
         gf_log (this->name, GF_LOG_INFO, "Received start vol req"
                 " for volume %s", volname);
 
-        ret = glusterd_op_begin (req, GD_OP_START_VOLUME, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_START_VOLUME, dict);
 
 out:
         free (cli_req.dict.dict_val); //its malloced by xdr
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         if (ret) {
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
                                                      dict, "operation failed");
@@ -1415,7 +1412,7 @@ glusterd_op_start_volume (dict_t *dict, char **op_errstr)
         if (ret)
                 goto out;
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_start (volinfo, brickinfo);
+                ret = glusterd_brick_start (volinfo, brickinfo, _gf_true);
                 if (ret)
                         goto out;
         }
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index fdcc38d8068..164009bfe4d 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -735,6 +735,29 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
         return -1;
 }
 
+static int
+glusterd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+    return ret;
+}
+
+static int
+glusterd_launch_synctask (xlator_t *this, synctask_fn_t fn)
+{
+    glusterd_conf_t *priv = NULL;
+    int              ret  = -1;
+
+    priv = this->private;
+
+    ret = synctask_new (this->ctx->env, fn,
+                        glusterd_default_synctask_cbk, NULL, priv);
+
+    if (ret)
+            gf_log (this->name, GF_LOG_CRITICAL, "Failed to create synctask"
+                    "for starting process");
+    return ret;
+}
+
 /*
  * init - called during glusterd initialization
  *
@@ -1016,16 +1039,17 @@ init (xlator_t *this)
         if (ret)
                 goto out;
 
-        glusterd_restart_bricks (conf);
-        ret = glusterd_restart_gsyncds (conf);
-        if (ret)
-                goto out;
+        glusterd_launch_synctask (this,
+                                  (synctask_fn_t) glusterd_restart_bricks);
+        glusterd_launch_synctask (this,
+                                  (synctask_fn_t) glusterd_restart_gsyncds);
+        glusterd_launch_synctask (this,
+                                  (synctask_fn_t) glusterd_restart_rebalance);
 
         ret = glusterd_hooks_spawn_worker (this);
         if (ret)
                 goto out;
 
-        glusterd_restart_rebalance (conf);
         ret = 0;
 out:
         if (ret < 0) {