posix: Needs to reserve disk space to prevent the brick from getting full

Problem: Currently there is no option available at posix xlator to save the
         disk from getting full

Solution: Introduce a new option storage.reserve at posix xlator to
          configure disk threshold.posix xlator spawn a thread to update the
          disk space status in posix private structure and same flag is checked
          by every posix fop before start operation.If flag value is 1 then
          it sets op_errno to ENOSPC and goto out from the fop.

BUG: 1471366
Change-Id: I98287cd409860f4c754fc69a332e0521bfb1b67e
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
Reviewed-on: https://review.gluster.org/17780
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

5 files changed, 217 insertions, 6 deletions

diff --git a/xlators/storage/posix/src/posix-aio.c b/xlators/storage/posix/src/posix-aio.c
index b5ac1b92ded..2adafeb07b8 100644
--- a/xlators/storage/posix/src/posix-aio.c
+++ b/xlators/storage/posix/src/posix-aio.c
@@ -330,6 +330,7 @@ posix_aio_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         VALIDATE_OR_GOTO (fd, err);
 
         priv = this->private;
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_errno, op_errno, err);
 
         ret = posix_fd_ctx_get (fd, this, &pfd, &op_errno);
         if (ret < 0) {
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 2d193ccac10..1530b1192c5 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1970,6 +1970,121 @@ unlock:
         UNLOCK (&priv->lock);
 }
 
+void
+posix_disk_space_check (xlator_t *this)
+{
+        struct  posix_private *priv     = NULL;
+        char    *subvol_path            = NULL;
+        int     op_ret                  = 0;
+        int     percent                 = 0;
+        struct statvfs buf              = {0};
+        uint64_t totsz                  = 0;
+        uint64_t freesz                 = 0;
+
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO ("posix-helpers", priv, out);
+
+        subvol_path = priv->base_path;
+        percent = priv->disk_threshhold;
+
+        op_ret = sys_statvfs (subvol_path, &buf);
+
+        if (op_ret == -1) {
+                gf_msg (this->name, GF_LOG_ERROR, errno, P_MSG_STATVFS_FAILED,
+                        "statvfs failed on %s", subvol_path);
+                goto out;
+        }
+        totsz = (buf.f_blocks * buf.f_bsize);
+        freesz = (buf.f_bfree * buf.f_bsize);
+
+        if (freesz <= ((totsz * percent) / 100)) {
+                priv->disk_space_full = 1;
+        } else {
+                priv->disk_space_full = 0;
+        }
+out:
+        return;
+}
+
+
+static void *
+posix_disk_space_check_thread_proc (void *data)
+{
+        xlator_t             *this               = NULL;
+        struct posix_private *priv               = NULL;
+        uint32_t              interval           = 0;
+        int                   ret                = -1;
+
+        this = data;
+        priv = this->private;
+
+        interval = 5;
+        gf_msg_debug (this->name, 0, "disk-space thread started, "
+                      "interval = %d seconds", interval);
+        while (1) {
+                /* aborting sleep() is a request to exit this thread, sleep()
+                 * will normally not return when cancelled */
+                ret = sleep (interval);
+                if (ret > 0)
+                        break;
+                /* prevent thread errors while doing the health-check(s) */
+                pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, NULL);
+
+                /* Do the disk-check.*/
+                posix_disk_space_check (this);
+                if (!priv->disk_space_check_active)
+                        goto out;
+                pthread_setcancelstate (PTHREAD_CANCEL_ENABLE, NULL);
+        }
+
+out:
+        gf_msg_debug (this->name, 0, "disk space check thread exiting");
+        LOCK (&priv->lock);
+        {
+                priv->disk_space_check_active = _gf_false;
+        }
+        UNLOCK (&priv->lock);
+
+
+        return NULL;
+}
+
+void
+posix_spawn_disk_space_check_thread (xlator_t *xl)
+{
+        struct posix_private *priv               = NULL;
+        int                   ret                = -1;
+
+        priv = xl->private;
+
+        LOCK (&priv->lock);
+        {
+                /* cancel the running thread  */
+                if (priv->disk_space_check_active == _gf_true) {
+                        pthread_cancel (priv->disk_space_check);
+                        priv->disk_space_check_active = _gf_false;
+                }
+
+                ret = gf_thread_create (&priv->disk_space_check, NULL,
+                                        posix_disk_space_check_thread_proc,
+                                        xl, "posix_reserve");
+                if (ret < 0) {
+                        priv->disk_space_check_active = _gf_false;
+                        gf_msg (xl->name, GF_LOG_ERROR, errno,
+                                P_MSG_DISK_SPACE_CHECK_FAILED,
+                                "unable to setup disk space check thread");
+                        goto unlock;
+                }
+
+                /* run the thread detached, resources will be freed on exit */
+                pthread_detach (priv->disk_space_check);
+                priv->disk_space_check_active = _gf_true;
+        }
+unlock:
+        UNLOCK (&priv->lock);
+}
+
 int
 posix_fsyncer_pick (xlator_t *this, struct list_head *head)
 {
diff --git a/xlators/storage/posix/src/posix-messages.h b/xlators/storage/posix/src/posix-messages.h
index ee06d6f7d85..20cf1f0109b 100644
--- a/xlators/storage/posix/src/posix-messages.h
+++ b/xlators/storage/posix/src/posix-messages.h
@@ -45,7 +45,7 @@
  */
 
 #define POSIX_COMP_BASE         GLFS_MSGID_COMP_POSIX
-#define GLFS_NUM_MESSAGES       110
+#define GLFS_NUM_MESSAGES       111
 #define GLFS_MSGID_END          (POSIX_COMP_BASE + GLFS_NUM_MESSAGES + 1)
 /* Messaged with message IDs */
 #define glfs_msg_start_x POSIX_COMP_BASE, "Invalid: Start of messages"
@@ -955,6 +955,15 @@
  */
 
 
+#define P_MSG_DISK_SPACE_CHECK_FAILED             (POSIX_COMP_BASE + 112)
+
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+
 /*------------*/
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 180da9e2c08..ceb9d448025 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -735,6 +735,7 @@ posix_do_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
         struct posix_fd    *pfd    = NULL;
         gf_boolean_t        locked = _gf_false;
         posix_inode_ctx_t  *ctx    = NULL;
+        struct  posix_private *priv = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
 
@@ -744,6 +745,9 @@ posix_do_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
         VALIDATE_OR_GOTO (this, out);
         VALIDATE_OR_GOTO (fd, out);
 
+        priv = this->private;
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, ret, ret, out);
+
         ret = posix_fd_ctx_get (fd, this, &pfd, &op_errno);
         if (ret < 0) {
                 gf_msg_debug (this->name, 0, "pfd is NULL from fd=%p", fd);
@@ -1042,19 +1046,27 @@ posix_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         int32_t ret                      =  0;
         struct  iatt statpre             = {0,};
         struct  iatt statpost            = {0,};
+        struct  posix_private *priv      = NULL;
+        int     op_ret                   = -1;
+        int     op_errno                 = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+
+        priv = this->private;
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
 
         ret = posix_do_zerofill (frame, this, fd, offset, len,
                                  &statpre, &statpost, xdata);
         if (ret < 0)
-                goto err;
+                goto out;
 
         STACK_UNWIND_STRICT(zerofill, frame, 0, 0, &statpre, &statpost, NULL);
         return 0;
 
-err:
-        STACK_UNWIND_STRICT(zerofill, frame, -1, -ret, NULL, NULL, NULL);
+out:
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, NULL, NULL, NULL);
         return 0;
-
 }
 
 static int32_t
@@ -1320,6 +1332,7 @@ posix_mknod (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
         VALIDATE_OR_GOTO (priv, out);
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
 
         MAKE_ENTRY_HANDLE (real_path, par_path, this, loc, NULL);
 
@@ -1538,6 +1551,7 @@ posix_mkdir (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
         VALIDATE_OR_GOTO (priv, out);
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
 
         MAKE_ENTRY_HANDLE (real_path, par_path, this, loc, NULL);
         if (!real_path || !par_path) {
@@ -2351,6 +2365,7 @@ posix_symlink (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
         VALIDATE_OR_GOTO (priv, out);
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
 
         MAKE_ENTRY_HANDLE (real_path, par_path, this, loc, &stbuf);
 
@@ -2509,6 +2524,7 @@ posix_rename (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
         VALIDATE_OR_GOTO (priv, out);
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
 
         SET_FS_ID (frame->root->uid, frame->root->gid);
         MAKE_ENTRY_HANDLE (real_oldpath, par_oldpath, this, oldloc, NULL);
@@ -2791,6 +2807,7 @@ posix_link (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
         VALIDATE_OR_GOTO (priv, out);
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
 
         SET_FS_ID (frame->root->uid, frame->root->gid);
         MAKE_INODE_HANDLE (real_oldpath, this, oldloc, &stbuf);
@@ -2998,6 +3015,7 @@ posix_create (call_frame_t *frame, xlator_t *this,
 
         priv = this->private;
         VALIDATE_OR_GOTO (priv, out);
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
 
         MAKE_ENTRY_HANDLE (real_path, par_path, this, loc, &stbuf);
 
@@ -3185,6 +3203,9 @@ posix_open (call_frame_t *frame, xlator_t *this,
         priv = this->private;
         VALIDATE_OR_GOTO (priv, out);
 
+        if (flags & O_CREAT)
+                DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
+
         MAKE_INODE_HANDLE (real_path, this, loc, &stbuf);
         if (!real_path) {
                 op_ret = -1;
@@ -3508,6 +3529,7 @@ posix_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         priv = this->private;
 
         VALIDATE_OR_GOTO (priv, out);
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
 
         ret = posix_fd_ctx_get (fd, this, &pfd, &op_errno);
         if (ret < 0) {
@@ -3647,6 +3669,7 @@ posix_statfs (call_frame_t *frame, xlator_t *this,
         struct statvfs         buf       = {0, };
         struct posix_private * priv      = NULL;
         int                    shared_by = 1;
+        int                    percent   = 0;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
@@ -3671,6 +3694,9 @@ posix_statfs (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+        percent = priv->disk_threshhold;
+        buf.f_bfree = (buf.f_bfree - ((buf.f_blocks * percent) / 100));
+
         shared_by = priv->shared_brick_count;
         if (shared_by > 1) {
                 buf.f_blocks /= shared_by;
@@ -3828,6 +3854,7 @@ posix_fsync (call_frame_t *frame, xlator_t *this,
 #endif
 
 	priv = this->private;
+
 	if (priv->batch_fsync_mode && xdata && dict_get (xdata, "batch-fsync")) {
 		posix_batch_fsync (frame, this, fd, datasync, xdata);
 		return 0;
@@ -3932,6 +3959,7 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         ssize_t       acl_size                = 0;
         dict_t       *xattr                   = NULL;
         posix_xattr_filler_t filler = {0,};
+        struct  posix_private *priv           = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
         SET_FS_ID (frame->root->uid, frame->root->gid);
@@ -3941,6 +3969,9 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc, out);
         VALIDATE_OR_GOTO (dict, out);
 
+        priv = this->private;
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
+
         MAKE_INODE_HANDLE (real_path, this, loc, NULL);
         if (!real_path) {
                 op_ret = -1;
@@ -5278,6 +5309,7 @@ posix_fsetxattr (call_frame_t *frame, xlator_t *this,
         struct  iatt       stbuf          = {0,};
         dict_t            *xattr          = NULL;
         posix_xattr_filler_t filler       = {0,};
+        struct  posix_private *priv       = NULL;
 
         DECLARE_OLD_FS_ID_VAR;
         SET_FS_ID (frame->root->uid, frame->root->gid);
@@ -5287,6 +5319,9 @@ posix_fsetxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (fd, out);
         VALIDATE_OR_GOTO (dict, out);
 
+        priv = this->private;
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
+
         ret = posix_fd_ctx_get (fd, this, &pfd, &op_errno);
         if (ret < 0) {
                 gf_msg (this->name, GF_LOG_WARNING, op_errno, P_MSG_PFD_NULL,
@@ -5950,11 +5985,17 @@ do_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
         dict_t               *xattr_rsp = NULL;
         dict_t               *xdata_rsp = NULL;
         struct iatt           stbuf = {0};
+        struct  posix_private *priv     = NULL;
+
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (xattr, out);
         VALIDATE_OR_GOTO (this, out);
 
+        priv = this->private;
+        DISK_SPACE_CHECK_AND_GOTO (frame, priv, op_ret, op_errno, out);
+
+
         if (fd) {
                 op_ret = posix_fd_ctx_get (fd, this, &pfd, &op_errno);
                 if (op_ret < 0) {
@@ -6052,7 +6093,6 @@ posix_fxattrop (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
-
 int
 posix_access (call_frame_t *frame, xlator_t *this,
               loc_t *loc, int32_t mask, dict_t *xdata)
@@ -6876,6 +6916,11 @@ notify (xlator_t *this,
                         pthread_cancel (priv->health_check);
                         priv->health_check = 0;
                 }
+                if (priv->disk_space_check) {
+                        priv->disk_space_check_active = _gf_false;
+                        pthread_cancel (priv->disk_space_check);
+                        priv->disk_space_check = 0;
+                }
                 if (priv->janitor) {
                         (void) gf_thread_cleanup_xint (priv->janitor);
                         priv->janitor = 0;
@@ -7051,6 +7096,11 @@ reconfigure (xlator_t *this, dict_t *options)
                         " fallback to <hostname>:<export>");
         }
 
+        GF_OPTION_RECONF ("reserve", priv->disk_threshhold,
+                          options, uint32, out);
+        if (priv->disk_threshhold)
+                posix_spawn_disk_space_check_thread (this);
+
         GF_OPTION_RECONF ("health-check-interval", priv->health_check_interval,
                           options, uint32, out);
         posix_spawn_health_check_thread (this);
@@ -7660,6 +7710,13 @@ init (xlator_t *this)
                                 " fallback to <hostname>:<export>");
         }
 
+        _private->disk_space_check_active = _gf_false;
+        _private->disk_space_full          = 0;
+        GF_OPTION_INIT ("reserve",
+                        _private->disk_threshhold, uint32, out);
+        if (_private->disk_threshhold)
+                posix_spawn_disk_space_check_thread (this);
+
         _private->health_check_active = _gf_false;
         GF_OPTION_INIT ("health-check-interval",
                         _private->health_check_interval, uint32, out);
@@ -7853,6 +7910,16 @@ struct volume_options options[] = {
           .description = "Interval in seconds for a filesystem health check, "
                          "set to 0 to disable"
         },
+        {
+          .key = {"reserve"},
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .default_value = "1",
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "Value in percentage in integer form required "
+           "to set reserve disk, "
+           "set to 0 to disable"
+        },
 	{ .key = {"batch-fsync-mode"},
 	  .type = GF_OPTION_TYPE_STR,
 	  .default_value = "reverse-fsync",
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index 81158266111..d83e2678572 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -63,6 +63,18 @@
 #define GF_UNLINK_TRUE 0x0000000000000001
 #define GF_UNLINK_FALSE 0x0000000000000000
 
+#define DISK_SPACE_CHECK_AND_GOTO(frame, priv, op_ret, op_errno, out)  do {   \
+               if (frame->root->pid >= 0 && priv->disk_space_full) {          \
+                        op_ret = -1;                                          \
+                        op_errno = ENOSPC;                                    \
+                        gf_msg_debug ("posix", ENOSPC,                        \
+                                      "disk space utilization reached limits" \
+                                      " for path %s ",  priv->base_path);     \
+                        goto out;                                             \
+               }                                                              \
+        } while (0)
+
+
 /**
  * posix_fd - internal structure common to file and directory fd's
  */
@@ -167,6 +179,11 @@ struct posix_private {
         pthread_t       health_check;
         gf_boolean_t    health_check_active;
 
+        uint32_t        disk_threshhold;
+        uint32_t        disk_space_full;
+        pthread_t       disk_space_check;
+        gf_boolean_t    disk_space_check_active;
+
 #ifdef GF_DARWIN_HOST_OS
         enum {
                 XATTR_NONE = 0,
@@ -274,6 +291,8 @@ __posix_fd_set_odirect (fd_t *fd, struct posix_fd *pfd, int opflags,
 			off_t offset, size_t size);
 void posix_spawn_health_check_thread (xlator_t *this);
 
+void posix_spawn_disk_space_check_thread (xlator_t *this);
+
 void *posix_fsyncer (void *);
 int
 posix_get_ancestry (xlator_t *this, inode_t *leaf_inode,