posix: Add option to disable nftw() based deletes when purging the landfill directory

Summary:
- We may have found an issue where certain directories were being moved into .landfill and then being quickly purged via nftw().
- We would like to have an emergency option to disable these purges.

Test Plan: Build, vol-set, read logs

Reviewers: rwareing, dph

Reviewed By: dph

Subscribers: #posix_storage

Differential Revision: https://phabricator.intern.facebook.com/D4862021

Change-Id: I90b54c535930c1ca2925a928728199b6b80eadd9
Signature: t1:4862021:1491855616:51b9b5b8957b0bb97afe27766f2e5aa78ff9edd4
Reviewed-on: https://review.gluster.org/18253
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Shreyas Siravara <sshreyas@fb.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

4 files changed, 54 insertions, 7 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 94ae212bc27..bb4efdf2c21 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -2596,6 +2596,10 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .voltype     = "storage/posix",
           .op_version  = 2,
         },
+        { .key         = "storage.disable-landfill-purge",
+          .voltype     = "storage/posix",
+          .op_version  = 2,
+        },
         { .key         = "storage.max-hardlinks",
           .voltype     = "storage/posix",
           .op_version  = 2,
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 4aa39514486..6bd6bb1a0bf 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1442,15 +1442,21 @@ posix_janitor_thread_proc (void *data)
         while (1) {
                 time (&now);
                 if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) {
-                        gf_msg_trace (this->name, 0,
+                        if (priv->disable_landfill_purge) {
+                                gf_msg_debug (this->name, 0,
+                                        "Janitor would have "
+                                        "cleaned out %s, but purge"
+                                        "is disabled.",
+                                        priv->trash_path);
+                        } else {
+                                gf_msg_trace (this->name, 0,
                                       "janitor cleaning out %s",
                                       priv->trash_path);
-
-                        nftw (priv->trash_path,
-                              janitor_walker,
-                              32,
-                              FTW_DEPTH | FTW_PHYS);
-
+                                nftw (priv->trash_path,
+                                      janitor_walker,
+                                      32,
+                                      FTW_DEPTH | FTW_PHYS);
+                        }
                         priv->last_landfill_check = now;
                 }
 
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index d4b8effd6da..32b961b680d 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -2280,6 +2280,8 @@ posix_rmdir (call_frame_t *frame, xlator_t *this,
                                 "mkdir of %s failed", priv->trash_path);
                 } else {
                         sprintf (tmp_path, "%s/%s", priv->trash_path, gfid_str);
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Moving %s to %s", real_path, tmp_path);
                         op_ret = sys_rename (real_path, tmp_path);
                         pthread_cond_signal (&priv->janitor_cond);
                 }
@@ -6899,6 +6901,20 @@ struct posix_private *priv = NULL;
         GF_OPTION_RECONF ("max-hardlinks", priv->max_hardlinks,
                           options, uint32, out);
 
+        GF_OPTION_RECONF ("disable-landfill-purge",
+                          priv->disable_landfill_purge,
+                          options, bool, out);
+
+        if (priv->disable_landfill_purge) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Janitor WILL NOT purge the landfill directory. "
+                        "Your landfill directory"
+                        " may fill up this brick.");
+        } else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Janitor WILL purge the landfill directory.");
+        }
+
         ret = 0;
 out:
 	return ret;
@@ -7514,6 +7530,9 @@ init (xlator_t *this)
         GF_OPTION_INIT ("batch-fsync-delay-usec", _private->batch_fsync_delay_usec,
                         uint32, out);
 
+        GF_OPTION_INIT ("disable-landfill-purge",
+                        _private->disable_landfill_purge, bool, out);
+
         GF_OPTION_INIT ("freespace-check-interval",
                         _private->freespace_check_interval, uint32, out);
 
@@ -7527,9 +7546,19 @@ init (xlator_t *this)
         pthread_mutex_init (&_private->freespace_check_lock, NULL);
         sys_statvfs (_private->base_path, &_private->freespace_stats);
         clock_gettime (CLOCK_MONOTONIC, &_private->freespace_check_last);
+
         _private->freespace_check_passed = freespace_ok (
                 this, &_private->freespace_stats, _private->min_free_disk,
                 _gf_true);
+        if (_private->disable_landfill_purge) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Janitor WILL NOT purge the landfill directory. "
+                        "Your landfill directory"
+                        " may fill up this brick.");
+        } else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Janitor WILL purge the landfill directory.");
+        }
 out:
         return ret;
 }
@@ -7731,6 +7760,13 @@ struct volume_options options[] = {
                          "read-ahead limits",
         },
         {
+          .key = {"disable-landfill-purge"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Disable glusterfs/landfill purges. "
+                         "WARNING: This can fill up a brick."
+        },
+        {
           .key = {"max-hardlinks"},
           .type = GF_OPTION_TYPE_INT,
           .min = 0,
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index 794f06c8ada..c8abd76ffb5 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -183,6 +183,7 @@ struct posix_private {
         uint32_t freespace_check_interval;
         gf_boolean_t freespace_check_passed;
         gf_boolean_t fadvise_random;
+        gf_boolean_t disable_landfill_purge;
         uint32_t max_hardlinks;
 };