Quota: heal directory on newly added bricks when quota limit is reached

Problem: if a lookup is done on a newly added brick for a path on which limit
has been reached, the lookup fails to heal the directory tree due to quota.

Solution: Tag the lookup as an internal fop and ignore it in quota.
Since marking internal fop does not usually give enough contextual information.
Introducing new flags to pass the contextual info.

Adding dict_check_flag and dict_set_flag to aid flag operations.
A flag is a single bit in a bit array (currently limited to 256 bits).

Change-Id: Ifb6a68bcaffedd425dd0f01f7db24edd5394c095
fixes: bz#1505355
BUG: 1505355
Signed-off-by: Sanoj Unnikrishnan <sunnikri@redhat.com>

4 files changed, 52 insertions, 4 deletions

diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index d14020fabd0..4762f25067f 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -1411,10 +1411,25 @@ dht_selfheal_dir_mkdir_lookup_done (call_frame_t *frame, xlator_t *this)
         dht_dir_set_heal_xattr (this, local, dict, local->xattr, NULL,
                                 NULL);
 
-        if (!dict)
+        if (!dict) {
                 gf_msg (this->name, GF_LOG_WARNING, 0,
                         DHT_MSG_DICT_SET_FAILED,
                         "dict is NULL, need to make sure gfids are same");
+                dict = dict_new ();
+                if (!dict)
+                        return -1;
+        }
+        ret = dict_set_flag (dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        DHT_MSG_DICT_SET_FAILED,
+                        "Failed to set dictionary value for"
+                        " key = %s at path: %s",
+                        GF_INTERNAL_CTX_KEY, loc->path);
+                /* We can still continue. As heal can still happen
+                 * unless quota limits have reached for the dir.
+                 */
+        }
 
         cnt = layout->cnt;
         for (i = 0; i < cnt; i++) {
diff --git a/xlators/features/quota/src/quota-messages.h b/xlators/features/quota/src/quota-messages.h
index 4292c5b4dcc..85f5abf7b29 100644
--- a/xlators/features/quota/src/quota-messages.h
+++ b/xlators/features/quota/src/quota-messages.h
@@ -46,7 +46,9 @@ GLFS_MSGID(QUOTA,
         Q_MSG_INODE_CTX_GET_FAILED,
         Q_MSG_INODE_CTX_SET_FAILED,
         Q_MSG_LOOKUP_FAILED,
-        Q_MSG_RPC_SUBMIT_FAILED
+        Q_MSG_RPC_SUBMIT_FAILED,
+        Q_MSG_ENFORCEMENT_SKIPPED,
+        Q_MSG_INTERNAL_FOP_KEY_MISSING
 );
 
 #endif /* !_QUOTA_MESSAGES_H_ */
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index 068d2e8c1f1..6ef2e20cf61 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -1591,6 +1591,28 @@ out:
         return ret;
 }
 
+/*
+ * return _gf_true if enforcement is needed and _gf_false otherwise
+ */
+gf_boolean_t
+should_quota_enforce (xlator_t *this, dict_t *dict, glusterfs_fop_t fop)
+{
+        int               ret = 0;
+
+        ret = dict_check_flag(dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
+
+        if (fop == GF_FOP_MKDIR && ret == DICT_FLAG_SET) {
+                return _gf_false;
+        } else if (ret == -ENOENT) {
+                gf_msg (this->name, GF_LOG_DEBUG, EINVAL,
+			Q_MSG_INTERNAL_FOP_KEY_MISSING,
+                        "No internal fop context present");
+                goto out;
+        }
+out:
+        return _gf_true;
+}
+
 int32_t
 quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
@@ -1965,7 +1987,6 @@ unwind:
         return 0;
 }
 
-
 int32_t
 quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
              mode_t umask, dict_t *xdata)
@@ -1976,9 +1997,15 @@ quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         call_stub_t   *stub  = NULL;
 
         priv = this->private;
-
         WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
+        if (!should_quota_enforce(this, xdata, GF_FOP_MKDIR)) {
+                gf_msg (this->name, GF_LOG_DEBUG, 0,
+			Q_MSG_ENFORCEMENT_SKIPPED,
+                        "Enforcement has been skipped(internal fop).");
+                goto off;
+        }
+
         local = quota_local_new ();
         if (local == NULL) {
                 op_errno = ENOMEM;
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 816fb3587d2..6d7d8c512db 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1208,6 +1208,10 @@ posix_handle_pair (xlator_t *this, const char *real_path,
         } else if (!strncmp(key, POSIX_ACL_ACCESS_XATTR, strlen(key))
                    && stbuf && IS_DHT_LINKFILE_MODE (stbuf)) {
                 goto out;
+        } else if (!strncmp(key, GF_INTERNAL_CTX_KEY, strlen(key))) {
+                /* ignore this key value pair */
+                ret = 0;
+                goto out;
         } else {
                 sys_ret = sys_lsetxattr (real_path, key, value->data,
                                          value->len, flags);