glusterd: log enhancements for volume start

* changed some of the log messages to give as much information as
  available in case of failure

* added logs to identify on which machine lock/stage/commit failed

* added macros to represent error strings to maintain uniformity
  among error messages for a given kind of error

* moved error logs wherever possible, from caller to callee to avoid
  code duplication

Change-Id: I0e98d5d3ba086c99240f2fbd642451f175f51942
BUG: 812356
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: http://review.gluster.org/4353
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>
Tested-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 64 insertions, 13 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index 80d6f33f067..4a8961cf429 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -484,12 +484,20 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
         int32_t              tmp_op   = 0;
         gf_boolean_t         local_locked = _gf_false;
         char                 *op_errstr = NULL;
+        xlator_t             *this = NULL;
+        char                 *hostname = NULL;
 
-        conf = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
 
         ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume "
+                        "operation");
                 goto out;
+        }
 
         op = tmp_op;
 
@@ -501,7 +509,7 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
         /*  Lock everything */
         ret = glusterd_lock (MY_UUID);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to acquire lock");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to acquire lock");
                 gf_asprintf (&op_errstr, "Another transaction is in progress. "
                              "Please try again after sometime.");
                 goto out;
@@ -511,32 +519,51 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
 
         INIT_LIST_HEAD (&conf->xaction_peers);
         list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
+                if (!peerinfo->connected)
+                        continue;
                 if (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
                         continue;
 
                 ret = gd_syncop_mgmt_lock (peerinfo->rpc,
                                            MY_UUID, tmp_uuid);
-                if (ret == 0)
+                if (ret) {
+                        gf_asprintf (&op_errstr, "Another transaction could be "
+                                     "in progress. Please try again after "
+                                     "sometime.");
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to acquire "
+                                "lock on peer %s", peerinfo->hostname);
+                        goto out;
+                } else {
                         list_add_tail (&peerinfo->op_peers_list,
                                        &conf->xaction_peers);
+                }
         }
 
         ret = glusterd_op_build_payload (&req_dict, &op_errstr, op_ctx);
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
                 goto out;
+        }
 
         /* stage op */
         ret = glusterd_op_stage_validate (op, req_dict, &op_errstr, rsp_dict);
-        if (ret)
-                goto out;
+        if (ret) {
+                hostname = "localhost";
+                goto stage_done;
+        }
 
         list_for_each_entry (peerinfo, &conf->xaction_peers, op_peers_list) {
                 ret = gd_syncop_mgmt_stage_op (peerinfo->rpc,
                                                MY_UUID, tmp_uuid,
                                                op, req_dict, &rsp_dict,
                                                &op_errstr);
-                if (ret)
-                        goto out;
+                if (ret) {
+                        hostname = peerinfo->hostname;
+                        goto stage_done;
+                }
 
                 if (op == GD_OP_REPLACE_BRICK)
                         (void) glusterd_syncop_aggr_rsp_dict (op, op_ctx,
@@ -547,23 +574,47 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
                         dict_unref (rsp_dict);
         }
 
+stage_done:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
+                        gd_op_list[op], hostname, (op_errstr) ? ":" : " ",
+                        (op_errstr) ? op_errstr : " ");
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_STAGE_FAIL, hostname);
+                goto out;
+        }
+
         /* commit op */
         ret = glusterd_op_commit_perform (op, req_dict, &op_errstr, rsp_dict);
-        if (ret)
-                goto out;
+        if (ret) {
+                hostname = "localhost";
+                goto commit_done;
+        }
 
         list_for_each_entry (peerinfo, &conf->xaction_peers, op_peers_list) {
                 ret = gd_syncop_mgmt_commit_op (peerinfo->rpc,
                                                 MY_UUID, tmp_uuid,
                                                 op, req_dict, &rsp_dict,
                                                 &op_errstr);
-                if (ret)
-                        goto out;
+                if (ret) {
+                        hostname = peerinfo->hostname;
+                        goto commit_done;
+                }
                 (void) glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict,
                                                       op_errstr);
                 if (rsp_dict)
                         dict_unref (rsp_dict);
         }
+commit_done:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
+                        gd_op_list[op], hostname, (op_errstr) ? ":" : " ",
+                        (op_errstr) ? op_errstr : " ");
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_COMMIT_FAIL,
+                                     hostname);
+                goto out;
+         }
 
         ret = 0;
 out: