glusterd rebalance: make co-operate with all other 'op'

that way, we can share the rebalance state with other peers
and can prevent confusion/conflicts when multiple rebalances
are done by different peers.

Change-Id: I24159e69332644718df7314f6f1da7fce9ff740e
BUG: 2112
Reviewed-on: http://review.gluster.com/343
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

4 files changed, 380 insertions, 56 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 44298d663..3b30fb080 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -1559,6 +1559,7 @@ glusterd_op_build_payload (dict_t **req)
                 case GD_OP_PROFILE_VOLUME:
                 case GD_OP_LOG_LEVEL:
                 case GD_OP_STATUS_VOLUME:
+                case GD_OP_REBALANCE:
                         {
                                 dict_t  *dict = ctx;
                                 dict_copy (dict, req_dict);
@@ -2331,6 +2332,10 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_stage_status_volume (dict, op_errstr);
                         break;
 
+                case GD_OP_REBALANCE:
+                        ret = glusterd_op_stage_rebalance (dict, op_errstr);
+                        break;
+
                 default:
                         gf_log ("", GF_LOG_ERROR, "Unknown op %d",
                                 op);
@@ -2418,6 +2423,10 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                        ret = glusterd_op_status_volume (dict, op_errstr, rsp_dict);
                        break;
 
+               case GD_OP_REBALANCE:
+                       ret = glusterd_op_rebalance (dict, op_errstr, rsp_dict);
+                       break;
+
                 default:
                         gf_log ("", GF_LOG_ERROR, "Unknown op %d",
                                 op);
@@ -3414,6 +3423,7 @@ glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
                 case GD_OP_PROFILE_VOLUME:
                 case GD_OP_LOG_LEVEL:
                 case GD_OP_STATUS_VOLUME:
+                case GD_OP_REBALANCE:
                         dict_unref (ctx);
                         break;
                 case GD_OP_DELETE_VOLUME:
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 3cfa04bdd..a013d0adf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -315,11 +315,18 @@ glusterd_defrag_stop (glusterd_volinfo_t *volinfo, u_quad_t *files,
         GF_ASSERT (size);
         GF_ASSERT (op_errstr);
 
+        if (!volinfo || !volinfo->defrag) {
+                ret = -1;
+                goto out;
+        }
+
         ret = glusterd_defrag_stop_validate (volinfo, op_errstr, len);
         if (ret)
                 goto out;
-        if (!volinfo || !volinfo->defrag) {
-                ret = -1;
+
+        ret = 0;
+        if (volinfo->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+                volinfo->defrag = NULL;
                 goto out;
         }
 
@@ -339,27 +346,54 @@ out:
 
 int
 glusterd_defrag_status_get_v2 (glusterd_volinfo_t *volinfo,
-                            gf2_cli_defrag_vol_rsp *rsp)
+                               dict_t *dict)
 {
-        if (!volinfo)
+        int      ret    = 0;
+        uint64_t files  = 0;
+        uint64_t size   = 0;
+        uint64_t lookup = 0;
+
+        if (!volinfo || !dict)
+                goto out;
+
+        ret = 0;
+        if (volinfo->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
                 goto out;
 
         if (volinfo->defrag) {
                 LOCK (&volinfo->defrag->lock);
                 {
-                        rsp->files = volinfo->defrag->total_files;
-                        rsp->size = volinfo->defrag->total_data;
-                        rsp->lookedup_files = volinfo->defrag->num_files_lookedup;
+                        files  = volinfo->defrag->total_files;
+                        size   = volinfo->defrag->total_data;
+                        lookup = volinfo->defrag->num_files_lookedup;
                 }
                 UNLOCK (&volinfo->defrag->lock);
         } else {
-                rsp->files = volinfo->rebalance_files;
-                rsp->size  = volinfo->rebalance_data;
-                rsp->lookedup_files = volinfo->lookedup_files;
+                files  = volinfo->rebalance_files;
+                size   = volinfo->rebalance_data;
+                lookup = volinfo->lookedup_files;
         }
 
-        rsp->op_errno = volinfo->defrag_status;
-        rsp->op_ret = 0;
+        ret = dict_set_uint64 (dict, "files", files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set file count");
+
+        ret = dict_set_uint64 (dict, "size", size);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set size of xfer");
+
+        ret = dict_set_uint64 (dict, "lookups", lookup);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set lookedup file count");
+
+        ret = dict_set_int32 (dict, "status", volinfo->defrag_status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set status");
+
 out:
         return 0;
 }
@@ -555,23 +589,29 @@ glusterd_rebalance_cmd_validate (int cmd, char *volname,
                           "be started to perform rebalance", volname);
                 goto out;
         }
+
         ret = 0;
+
 out:
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req)
+glusterd_handle_defrag_volume (rpcsvc_request_t *req)
 {
-        int32_t                 ret           = -1;
-        gf1_cli_defrag_vol_req  cli_req       = {0,};
-        glusterd_volinfo_t     *volinfo = NULL;
-        gf2_cli_defrag_vol_rsp rsp = {0,};
+        int32_t                ret           = -1;
+        gf1_cli_defrag_vol_req cli_req       = {0,};
+        glusterd_conf_t         *priv = NULL;
+        char                   cmd_str[4096] = {0,};
+        glusterd_volinfo_t      *volinfo = NULL;
+        gf1_cli_defrag_vol_rsp rsp = {0,};
         char                    msg[2048] = {0};
 
         GF_ASSERT (req);
 
+        priv    = THIS->private;
+
         if (!xdr_to_generic (req->msg[0], &cli_req,
                              (xdrproc_t)xdr_gf1_cli_defrag_vol_req)) {
                 //failed to decode msg;
@@ -583,38 +623,45 @@ glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req)
 
         rsp.volname = cli_req.volname;
         rsp.op_ret = -1;
-        rsp.op_errstr = msg;
 
         ret = glusterd_rebalance_cmd_validate (cli_req.cmd, cli_req.volname,
                                                &volinfo, msg, sizeof (msg));
         if (ret)
                 goto out;
-
         switch (cli_req.cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_MIGRATE_DATA:
         case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
+        {
                 ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
                                                     cli_req.cmd);
                 rsp.op_ret = ret;
                 break;
+        }
         case GF_DEFRAG_CMD_STOP:
                 ret = glusterd_defrag_stop (volinfo, &rsp.files, &rsp.size,
                                             msg, sizeof (msg));
                 rsp.op_ret = ret;
                 break;
         case GF_DEFRAG_CMD_STATUS:
-                ret = glusterd_defrag_status_get_v2 (volinfo, &rsp);
+                ret = glusterd_defrag_status_get (volinfo, &rsp);
                 break;
         default:
                 break;
         }
-        glusterd_rebalance_cmd_log (cli_req.cmd, cli_req.volname, rsp.op_ret);
-out:
+        if (ret)
+                gf_log("glusterd", GF_LOG_DEBUG, "command: %s failed",cmd_str);
+
+        if (cli_req.cmd != GF_DEFRAG_CMD_STATUS) {
+                gf_cmd_log ("volume rebalance"," on volname: %s %d %s",
+                            cli_req.volname,
+                            cli_req.cmd, ((ret)?"FAILED":"SUCCESS"));
+        }
 
+out:
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf2_cli_defrag_vol_rsp);
+                                     (xdrproc_t)xdr_gf1_cli_defrag_vol_rsp);
         if (cli_req.volname)
                 free (cli_req.volname);//malloced by xdr
 
@@ -622,17 +669,17 @@ out:
 }
 
 int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req)
 {
-        int32_t                ret           = -1;
-        gf1_cli_defrag_vol_req cli_req       = {0,};
-        char                   cmd_str[4096] = {0,};
-        glusterd_volinfo_t      *volinfo = NULL;
-        gf1_cli_defrag_vol_rsp rsp = {0,};
-        char                    msg[2048] = {0};
+        int32_t                 ret           = -1;
+        gf1_cli_defrag_vol_req  cli_req       = {0,};
+        glusterd_conf_t        *priv          = NULL;
+        dict_t                 *dict          = NULL;
 
         GF_ASSERT (req);
 
+        priv = THIS->private;
+
         if (!xdr_to_generic (req->msg[0], &cli_req,
                              (xdrproc_t)xdr_gf1_cli_defrag_vol_req)) {
                 //failed to decode msg;
@@ -642,49 +689,202 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req)
 
         glusterd_rebalance_cmd_attempted_log (cli_req.cmd, cli_req.volname);
 
-        rsp.volname = cli_req.volname;
-        rsp.op_ret = -1;
+        dict = dict_new ();
+        if (!dict)
+                goto out;
 
-        ret = glusterd_rebalance_cmd_validate (cli_req.cmd, cli_req.volname,
-                                               &volinfo, msg, sizeof (msg));
+        ret = dict_set_str (dict, "volname", cli_req.volname);
         if (ret)
                 goto out;
-        switch (cli_req.cmd) {
+
+        ret = dict_set_str (dict, "node-uuid", uuid_utoa (priv->uuid));
+        if (ret)
+                goto out;
+
+        ret = dict_set_int32 (dict, "rebalance-command", cli_req.cmd);
+        if (ret)
+                goto out;
+
+        ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict);
+
+out:
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        if (ret) {
+                if (dict)
+                        dict_unref (dict);
+                ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0, req,
+                                                     NULL, "operation failed");
+        }
+        if (cli_req.volname)
+                free (cli_req.volname);//malloced by xdr
+
+        return 0;
+}
+
+
+int
+glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
+{
+        char *volname = NULL;
+        int ret = 0;
+        int32_t cmd = 0;
+        char msg[2048] = {0};
+        glusterd_volinfo_t  *volinfo = NULL;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "volname not found");
+                goto out;
+        }
+        ret = dict_get_int32 (dict, "rebalance-command", &cmd);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "cmd not found");
+                goto out;
+        }
+
+        ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
+                                               msg, sizeof (msg));
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "failed to validate");
+                goto out;
+        }
+        switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_MIGRATE_DATA:
         case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
-        {
-                ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
-                                                    cli_req.cmd);
-                rsp.op_ret = ret;
+                ret = glusterd_defrag_start_validate (volinfo,
+                                                      msg, sizeof (msg));
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "start validate failed");
+                        goto out;
+                }
+        default:
                 break;
         }
-        case GF_DEFRAG_CMD_STOP:
-                ret = glusterd_defrag_stop (volinfo, &rsp.files, &rsp.size,
-                                            msg, sizeof (msg));
-                rsp.op_ret = ret;
+
+        ret = 0;
+out:
+        if (ret && op_errstr && msg[0])
+                *op_errstr = gf_strdup (msg);
+
+        return ret;
+}
+
+
+int
+glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        char               *volname   = NULL;
+        int                 ret       = 0;
+        int32_t             cmd       = 0;
+        char                msg[2048] = {0};
+        glusterd_volinfo_t *volinfo   = NULL;
+        uint64_t            files     = 0;
+        uint64_t            size      = 0;
+        char               *uuid_str  = NULL;
+        uuid_t              node_uuid = {0,};
+        glusterd_conf_t    *priv      = NULL;
+        dict_t             *tmp_dict  = NULL;
+
+        priv = THIS->private;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "volname not given");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "rebalance-command", &cmd);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "command not given");
+                goto out;
+        }
+        ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
+                                               msg, sizeof (msg));
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "cmd validate failed");
+                goto out;
+        }
+
+        if ((cmd != GF_DEFRAG_CMD_STATUS) ||
+            (cmd != GF_DEFRAG_CMD_STOP)) {
+                ret = dict_get_str (dict, "node-uuid", &uuid_str);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG, "node-uuid not found");
+                        goto out;
+                }
+
+                ret = uuid_parse (uuid_str, node_uuid);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "uuid-unparse failed %s", uuid_str);
+                        goto out;
+                }
+
+                /* perform this on only the node which has
+                   issued the command */
+                if (uuid_compare (node_uuid, priv->uuid)) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "not the source node %s %s", uuid_str,
+                                uuid_utoa (priv->uuid));
+                        /* Need to initialize the 'volinfo->defrag' to some
+                           value to say operation is in progress */
+                        volinfo->defrag = (void *)1;
+                        goto out;
+                }
+        }
+
+        switch (cmd) {
+        case GF_DEFRAG_CMD_START:
+        case GF_DEFRAG_CMD_START_LAYOUT_FIX:
+        case GF_DEFRAG_CMD_START_MIGRATE_DATA:
+        case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
+                ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
+                                                    cmd);
+                 break;
+         case GF_DEFRAG_CMD_STOP:
+                 ret = glusterd_defrag_stop (volinfo, &files, &size,
+                                             msg, sizeof (msg));
+                if (!ret && rsp_dict) {
+                        ret = dict_set_uint64 (rsp_dict, "files", files);
+                        if (ret)
+                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        "failed to set file count");
+
+                        ret = dict_set_uint64 (rsp_dict, "size", size);
+                        if (ret)
+                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        "failed to set xfer size");
+
+                        /* Don't want to propagate errors from dict_set() */
+                        ret = 0;
+                }
                 break;
         case GF_DEFRAG_CMD_STATUS:
-                ret = glusterd_defrag_status_get (volinfo, &rsp);
+
+                if (rsp_dict)
+                        tmp_dict = rsp_dict;
+
+                /* On source node, there will be no 'rsp_dict' */
+                if (!tmp_dict)
+                        tmp_dict = glusterd_op_get_ctx (GD_OP_REBALANCE);
+
+                ret = glusterd_defrag_status_get_v2 (volinfo, tmp_dict);
                 break;
         default:
                 break;
         }
-        if (ret)
-                gf_log("glusterd", GF_LOG_DEBUG, "command: %s failed",cmd_str);
 
-        if (cli_req.cmd != GF_DEFRAG_CMD_STATUS) {
-                gf_cmd_log ("volume rebalance"," on volname: %s %d %s",
-                            cli_req.volname,
-                            cli_req.cmd, ((ret)?"FAILED":"SUCCESS"));
-        }
+        glusterd_rebalance_cmd_log (cmd, volname, ret);
 
 out:
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_defrag_vol_rsp);
-        if (cli_req.volname)
-                free (cli_req.volname);//malloced by xdr
+        if (ret && op_errstr && msg[0])
+                *op_errstr = gf_strdup (msg);
 
-        return 0;
+        return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 06cbdd793..aa493bd76 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -409,6 +409,51 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
                 xdrproc = (xdrproc_t) xdr_gf1_cli_status_volume_rsp;
                 break;
         }
+        case GD_OP_REBALANCE:
+        {
+                gf2_cli_defrag_vol_rsp rsp = {0,};
+                int32_t                status = 0;
+
+                ctx = op_ctx;
+                rsp.op_ret = op_ret;
+                rsp.op_errno = op_errno;
+                rsp.volname = "";
+                if (op_errstr)
+                        rsp.op_errstr = op_errstr;
+                else
+                        rsp.op_errstr = "";
+
+                if (ctx) {
+                        ret = dict_get_uint64 (ctx, "files", &rsp.files);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_DEBUG,
+                                        "failed to get the file count");
+                        }
+                        ret = dict_get_uint64 (ctx, "size", &rsp.size);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_DEBUG,
+                                        "failed to get the size of migration");
+                        }
+                        ret = dict_get_uint64 (ctx, "lookups", &rsp.lookedup_files);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_DEBUG,
+                                        "failed to get lookuped file count");
+                        }
+
+                        ret = dict_get_int32 (ctx, "status", &status);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_TRACE,
+                                        "failed to get status");
+                        }
+                }
+                /* needed by 'rebalance status' */
+                if (status)
+                        rsp.op_errno = status;
+
+                cli_rsp = &rsp;
+                xdrproc = (xdrproc_t)xdr_gf2_cli_defrag_vol_rsp;
+                break;
+        }
         case GD_OP_NONE:
         case GD_OP_MAX:
         {
@@ -1257,6 +1302,66 @@ out:
         return ret;
 }
 
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *rsp_dict)
+{
+        int            ret      = 0;
+        dict_t        *ctx_dict = NULL;
+        glusterd_op_t  op       = GD_OP_NONE;
+        uint64_t       value    = 0;
+        int32_t        value32  = 0;
+
+        GF_ASSERT (rsp_dict);
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (GD_OP_REBALANCE == op);
+
+        ctx_dict = glusterd_op_get_ctx (op);
+
+        if (!ctx_dict)
+                goto out;
+
+        ret = dict_get_uint64 (rsp_dict, "files", &value);
+        if (!ret) {
+                ret = dict_set_uint64 (ctx_dict, "files", value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set the file count");
+                }
+        }
+
+        ret = dict_get_uint64 (rsp_dict, "size", &value);
+        if (!ret) {
+                ret = dict_set_uint64 (ctx_dict, "size", value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set the size of migration");
+                }
+        }
+
+        ret = dict_get_uint64 (rsp_dict, "lookups", &value);
+        if (!ret) {
+                ret = dict_set_uint64 (ctx_dict, "lookups", value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set lookuped file count");
+                }
+        }
+
+        ret = dict_get_int32 (rsp_dict, "status", &value32);
+        if (!ret) {
+                ret = dict_set_int32 (ctx_dict, "status", value32);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set status");
+                }
+        }
+
+out:
+        return ret;
+}
+
+
 int32_t
 glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
@@ -1372,6 +1477,12 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                         ret = glusterd_volume_status_use_rsp_dict (dict);
                         if (ret)
                                 goto out;
+
+                case GD_OP_REBALANCE:
+                        ret = glusterd_volume_rebalance_use_rsp_dict (dict);
+                        if (ret)
+                                goto out;
+
                 break;
 
                 default:
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 699c03c4b..41fbe5faa 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -78,6 +78,7 @@ typedef enum glusterd_op_ {
         GD_OP_QUOTA,
         GD_OP_LOG_LEVEL,
         GD_OP_STATUS_VOLUME,
+        GD_OP_REBALANCE,
         GD_OP_MAX,
 } glusterd_op_t;
 
@@ -567,6 +568,8 @@ int glusterd_op_remove_brick (dict_t *dict);
 int glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr);
 int glusterd_op_stage_remove_brick (dict_t *dict);
 
+int glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr);
+int glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
 
 
 /* misc */