mgmt/glusterd: make sure quota enforcer has established connection with quotad before marking quota as enabled.

without this patch there is a window of time when quota is marked as
enabled in quota-enforcer, but connection to quotad wouldn't have been
established. Any checklimit done during this period can result in a
failed fop because of unavailability of quotad.

Change-Id: I0d509fabc434dd55ce9ec59157123524197fcc80
Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
BUG: 969461
Signed-off-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: http://review.gluster.org/6572
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

4 files changed, 68 insertions, 24 deletions

diff --git a/xlators/features/quota/src/quota-enforcer-client.c b/xlators/features/quota/src/quota-enforcer-client.c
index bfea5e42014..7d8ab937d1e 100644
--- a/xlators/features/quota/src/quota-enforcer-client.c
+++ b/xlators/features/quota/src/quota-enforcer-client.c
@@ -295,6 +295,37 @@ quota_enforcer_notify (struct rpc_clnt *rpc, void *mydata,
         return ret;
 }
 
+int
+quota_enforcer_blocking_connect (rpc_clnt_t *rpc)
+{
+        dict_t *options = NULL;
+        int     ret     = -1;
+
+        options = dict_new ();
+        if (options == NULL)
+                goto out;
+
+        ret = dict_set_str (options, "non-blocking-io", "no");
+        if (ret)
+                goto out;
+
+        rpc->conn.trans->reconfigure (rpc->conn.trans, options);
+
+        rpc_clnt_start (rpc);
+
+        ret = dict_set_str (options, "non-blocking-io", "yes");
+        if (ret)
+                goto out;
+
+        rpc->conn.trans->reconfigure (rpc->conn.trans, options);
+
+        ret = 0;
+out:
+        dict_unref (options);
+
+        return ret;
+}
+
 //Returns a started rpc_clnt. Creates a new rpc_clnt if quota_priv doesn't have
 //one already
 struct rpc_clnt *
@@ -309,9 +340,13 @@ quota_enforcer_init (xlator_t *this, dict_t *options)
                 gf_log (this->name, GF_LOG_TRACE, "quota enforcer clnt already "
                         "inited");
                 //Turns out to be a NOP if the clnt is already connected.
-                rpc_clnt_start (priv->rpc_clnt);
+                ret = quota_enforcer_blocking_connect (priv->rpc_clnt);
+                if (ret)
+                        goto out;
+
                 return priv->rpc_clnt;
         }
+
         priv->quota_enforcer = &quota_enforcer_clnt;
 
         ret = dict_set_str (options, "transport.address-family", "unix");
@@ -339,7 +374,11 @@ quota_enforcer_init (xlator_t *this, dict_t *options)
                 goto out;
         }
 
-        rpc_clnt_start (rpc);
+        ret = quota_enforcer_blocking_connect (rpc);
+        if (ret)
+                goto out;
+
+        ret = 0;
 out:
         if (ret) {
                 if (rpc)
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index e1471258b49..2812a2b13a9 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -4177,14 +4177,15 @@ err:
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
-        int32_t           ret   = -1;
-        quota_priv_t     *priv  = NULL;
+        int32_t       ret      = -1;
+        quota_priv_t *priv     = NULL;
+        gf_boolean_t  quota_on = _gf_false;
 
         priv = this->private;
 
         GF_OPTION_RECONF ("deem-statfs", priv->consider_statfs, options, bool,
                           out);
-        GF_OPTION_RECONF ("server-quota", priv->is_quota_on, options, bool,
+        GF_OPTION_RECONF ("server-quota", quota_on, options, bool,
                           out);
         GF_OPTION_RECONF ("default-soft-limit", priv->default_soft_lim,
                           options, percent, out);
@@ -4195,7 +4196,7 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("hard-timeout", priv->hard_timeout, options,
                           time, out);
 
-        if (priv->is_quota_on) {
+        if (quota_on) {
                 priv->rpc_clnt = quota_enforcer_init (this,
                                                       this->options);
                 if (priv->rpc_clnt == NULL) {
@@ -4216,6 +4217,8 @@ reconfigure (xlator_t *this, dict_t *options)
                 }
         }
 
+        priv->is_quota_on = quota_on;
+
         ret = 0;
 out:
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index ffde532bdcd..d13533aa6c1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -996,14 +996,6 @@ glusterd_quotad_op (int opcode)
                                 ret = glusterd_check_generate_start_quotad ();
                         break;
 
-                case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
-                case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
-                case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
-                case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
-
-                        ret = glusterd_reconfigure_quotad ();
-                        break;
-
                 default:
                         ret = 0;
                         break;
@@ -1131,6 +1123,12 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                         goto out;
         }
 
+        if (priv->op_version > GD_OP_VERSION_MIN) {
+                ret = glusterd_quotad_op (type);
+                if (ret)
+                        goto out;
+        }
+
         ret = glusterd_create_volfiles_and_notify_services (volinfo);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Unable to re-create "
@@ -1151,11 +1149,6 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         if (rsp_dict && start_crawl == _gf_true)
                 glusterd_quota_initiate_fs_crawl (priv, volname, type);
 
-        if (priv->op_version > GD_OP_VERSION_MIN) {
-                ret = glusterd_quotad_op (type);
-                if (ret)
-                        goto out;
-        }
         ret = 0;
 out:
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 8b99d20ee95..7c58e51addb 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -3965,7 +3965,7 @@ glusterd_nodesvc_disconnect (char *server)
 }
 
 int32_t
-glusterd_nodesvc_start (char *server)
+glusterd_nodesvc_start (char *server, gf_boolean_t wait)
 {
         int32_t                 ret                        = -1;
         xlator_t               *this                       = NULL;
@@ -4051,7 +4051,16 @@ glusterd_nodesvc_start (char *server)
         runner_log (&runner, "", GF_LOG_DEBUG,
                     "Starting the nfs/glustershd services");
 
-        ret = runner_run_nowait (&runner);
+        if (!wait) {
+                ret = runner_run_nowait (&runner);
+        } else {
+                synclock_unlock (&priv->big_lock);
+                {
+                        ret = runner_run (&runner);
+                }
+                synclock_lock (&priv->big_lock);
+        }
+
         if (ret == 0) {
                 glusterd_nodesvc_connect (server, sockfpath);
         }
@@ -4062,19 +4071,19 @@ out:
 int
 glusterd_nfs_server_start ()
 {
-        return glusterd_nodesvc_start ("nfs");
+        return glusterd_nodesvc_start ("nfs", _gf_false);
 }
 
 int
 glusterd_shd_start ()
 {
-        return glusterd_nodesvc_start ("glustershd");
+        return glusterd_nodesvc_start ("glustershd", _gf_false);
 }
 
 int
 glusterd_quotad_start ()
 {
-        return glusterd_nodesvc_start ("quotad");
+        return glusterd_nodesvc_start ("quotad", _gf_true);
 }
 
 gf_boolean_t