storage/posix: Added janitor thread.

The janitor thread deletes all files and directories in
the "/" GF_REPLICATE_TRASH_DIR directory. This directory
is used by replicate self-heal to dump files and
directories it deletes.

This is needed because letting replicate walk the directory
tree and delete a directory and all its children is too racy.
Instead, replicate self-heal only does an atomic rename(),
and the janitor thread takes care of actually deleting them.

Signed-off-by: Vikas Gorur <vikas@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 227 (replicate selfheal does not remove directory with contents in it)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=227

2 files changed, 119 insertions, 1 deletions

diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 325c69634ec..1219620f197 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -29,6 +29,7 @@
 #include <sys/resource.h>
 #include <errno.h>
 #include <libgen.h>
+#include <pthread.h>
 #include <ftw.h>
 
 #ifndef GF_BSD_HOST_OS
@@ -50,7 +51,7 @@
 #include "syscall.h"
 #include "statedump.h"
 #include "locking.h"
-#include <libgen.h>
+#include "timer.h"
 
 #undef HAVE_SET_FSID
 #ifdef HAVE_SET_FSID
@@ -1322,6 +1323,95 @@ posix_mknod (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
+
+static int
+janitor_walker (const char *fpath, const struct stat *sb,
+                int typeflag, struct FTW *ftwbuf)
+{
+        switch (sb->st_mode & S_IFMT) {
+        case S_IFREG:
+        case S_IFBLK:
+        case S_IFLNK:
+        case S_IFCHR:
+        case S_IFIFO:
+        case S_IFSOCK:
+                gf_log (THIS->name, GF_LOG_TRACE,
+                        "unlinking %s", fpath);
+                unlink (fpath);
+                break;
+
+        case S_IFDIR:
+                if (ftwbuf->level) { /* don't remove top level dir */
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "removing directory %s", fpath);
+
+                        rmdir (fpath);
+                }
+                break;
+        }
+
+        return FTW_CONTINUE;
+}
+
+
+#define JANITOR_SLEEP_DURATION          2
+
+static void *
+posix_janitor_thread_proc (void *data)
+{
+        xlator_t *            this = NULL;
+        struct posix_private *priv = NULL;
+
+        this = data;
+        priv = this->private;
+
+        THIS = this;
+
+        while (1) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "janitor woke up, cleaning out /" GF_REPLICATE_TRASH_DIR);
+
+                nftw (priv->trash_path,
+                      janitor_walker,
+                      32,
+                      FTW_DEPTH | FTW_PHYS);
+
+                sleep (JANITOR_SLEEP_DURATION);
+        }
+
+        return NULL;
+}
+
+
+static void
+posix_spawn_janitor_thread (xlator_t *this)
+{
+        struct posix_private *priv = NULL;
+        int ret = 0;
+
+        priv = this->private;
+
+        LOCK (&priv->lock);
+        {
+                if (!priv->janitor_present) {
+                        ret = pthread_create (&priv->janitor, NULL,
+                                              posix_janitor_thread_proc, this);
+
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "spawning janitor thread failed: %s",
+                                        strerror (errno));
+                                goto unlock;
+                        }
+
+                        priv->janitor_present = _gf_true;
+                }
+        }
+unlock:
+        UNLOCK (&priv->lock);
+}
+
+
 int32_t
 posix_mkdir (call_frame_t *frame, xlator_t *this,
              loc_t *loc, mode_t mode)
@@ -1384,6 +1474,14 @@ posix_mkdir (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+        if (!strcmp (loc->path, "/" GF_REPLICATE_TRASH_DIR)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "got mkdir %s, spawning janitor thread",
+                        loc->path);
+
+                posix_spawn_janitor_thread (this);
+        }
+
 #ifndef HAVE_SET_FSID
         op_ret = chown (real_path, frame->root->uid, gid);
         if (op_ret == -1) {
@@ -4738,6 +4836,21 @@ init (xlator_t *this)
         _private->base_path = strdup (dir_data->data);
         _private->base_path_length = strlen (_private->base_path);
 
+        _private->trash_path = CALLOC (1, _private->base_path_length
+                                       + strlen ("/")
+                                       + strlen (GF_REPLICATE_TRASH_DIR)
+                                       + 1);
+
+        if (!_private->trash_path) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory.");
+                ret = -1;
+                goto out;
+        }
+
+        strncpy (_private->trash_path, _private->base_path, _private->base_path_length);
+        strcat (_private->trash_path, "/" GF_REPLICATE_TRASH_DIR);
+
         LOCK_INIT (&_private->lock);
 
         ret = gethostname (_private->hostname, 256);
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index b16800c5c93..0a99caeb636 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -114,6 +114,11 @@ struct posix_private {
 */
         uint64_t        gen_seq;
         gf_lock_t       gen_lock;
+
+/* janitor thread which cleans up /.trash (created by replicate) */
+        pthread_t       janitor;
+        gf_boolean_t    janitor_present;
+        char *          trash_path;
 };
 
 #define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)