diff options
Diffstat (limited to 'xlators/storage/posix/src/posix-helpers.c')
| -rw-r--r-- | xlators/storage/posix/src/posix-helpers.c | 575 |
1 files changed, 456 insertions, 119 deletions
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c index 06b5cedcb..e295f8850 100644 --- a/xlators/storage/posix/src/posix-helpers.c +++ b/xlators/storage/posix/src/posix-helpers.c @@ -1,22 +1,12 @@ /* - Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com> - This file is part of GlusterFS. - - GlusterFS is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - GlusterFS is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see - <http://www.gnu.org/licenses/>. -*/ + Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ #ifndef _CONFIG_H #define _CONFIG_H #include "config.h" @@ -32,6 +22,7 @@ #include <pthread.h> #include <ftw.h> #include <sys/stat.h> +#include <signal.h> #ifndef GF_BSD_HOST_OS #include <alloca.h> @@ -54,16 +45,9 @@ #include "timer.h" #include "glusterfs3-xdr.h" #include "hashfn.h" +#include "glusterfs-acl.h" #include <fnmatch.h> -typedef struct { - xlator_t *this; - const char *real_path; - dict_t *xattr; - struct iatt *stbuf; - loc_t *loc; -} posix_xattr_filler_t; - char *marker_xattrs[] = {"trusted.glusterfs.quota.*", "trusted.glusterfs.*.xtime", NULL}; @@ -117,7 +101,7 @@ out: return ignore; } -static void +static int _posix_xattr_get_set (dict_t *xattr_req, char *key, data_t *data, @@ -187,19 +171,13 @@ _posix_xattr_get_set (dict_t *xattr_req, err: if (_fd != -1) close (_fd); - if (databuf) - GF_FREE (databuf); + GF_FREE (databuf); } } else if (!strcmp (key, GLUSTERFS_OPEN_FD_COUNT)) { loc = filler->loc; - if (loc && !list_empty (&loc->inode->fd_list)) { - ret = dict_set_uint32 (filler->xattr, key, 1); - if (ret < 0) - gf_log (filler->this->name, GF_LOG_WARNING, - "Failed to set dictionary value for %s", - key); - } else { - ret = dict_set_uint32 (filler->xattr, key, 0); + if (loc) { + ret = dict_set_uint32 (filler->xattr, key, + loc->inode->fd_count); if (ret < 0) gf_log (filler->this->name, GF_LOG_WARNING, "Failed to set dictionary value for %s", @@ -212,22 +190,31 @@ _posix_xattr_get_set (dict_t *xattr_req, value = GF_CALLOC (1, xattr_size + 1, gf_posix_mt_char); if (!value) - return; + return -1; - sys_lgetxattr (filler->real_path, key, value, - xattr_size); + xattr_size = sys_lgetxattr (filler->real_path, key, value, + xattr_size); + if (xattr_size <= 0) { + gf_log (filler->this->name, GF_LOG_WARNING, + "getxattr failed. path: %s, key: %s", + filler->real_path, key); + GF_FREE (value); + return -1; + } value[xattr_size] = '\0'; ret = dict_set_bin (filler->xattr, key, value, xattr_size); - if (ret < 0) + if (ret < 0) { gf_log (filler->this->name, GF_LOG_DEBUG, "dict set failed. path: %s, key: %s", filler->real_path, key); + GF_FREE (value); + } } } out: - return; + return 0; } @@ -235,14 +222,17 @@ int posix_fill_gfid_path (xlator_t *this, const char *path, struct iatt *iatt) { int ret = 0; + ssize_t size = 0; if (!iatt) return 0; - ret = sys_lgetxattr (path, GFID_XATTR_KEY, iatt->ia_gfid, 16); + size = sys_lgetxattr (path, GFID_XATTR_KEY, iatt->ia_gfid, 16); /* Return value of getxattr */ - if ((ret == 16) || (ret == -1)) + if ((size == 16) || (size == -1)) ret = 0; + else + ret = size; return ret; } @@ -252,14 +242,17 @@ int posix_fill_gfid_fd (xlator_t *this, int fd, struct iatt *iatt) { int ret = 0; + ssize_t size = 0; if (!iatt) return 0; - ret = sys_fgetxattr (fd, GFID_XATTR_KEY, iatt->ia_gfid, 16); + size = sys_fgetxattr (fd, GFID_XATTR_KEY, iatt->ia_gfid, 16); /* Return value of getxattr */ - if ((ret == 16) || (ret == -1)) + if ((size == 16) || (size == -1)) ret = 0; + else + ret = size; return ret; } @@ -332,11 +325,21 @@ posix_istat (xlator_t *this, uuid_t gfid, const char *basename, ret = lstat (real_path, &lstatbuf); - if (ret == -1) { - if (errno != ENOENT && errno != ELOOP) - gf_log (this->name, GF_LOG_WARNING, - "lstat failed on %s (%s)", - real_path, strerror (errno)); + if (ret != 0) { + if (ret == -1) { + if (errno != ENOENT && errno != ELOOP) + gf_log (this->name, GF_LOG_WARNING, + "lstat failed on %s (%s)", + real_path, strerror (errno)); + } else { + // may be some backend filesystem issue + gf_log (this->name, GF_LOG_ERROR, "lstat failed on " + "%s and return value is %d instead of -1. " + "Please see dmesg output to check whether the " + "failure is due to backend filesystem issue", + real_path, ret); + ret = -1; + } goto out; } @@ -380,11 +383,21 @@ posix_pstat (xlator_t *this, uuid_t gfid, const char *path, ret = lstat (path, &lstatbuf); - if (ret == -1) { - if (errno != ENOENT) - gf_log (this->name, GF_LOG_WARNING, - "lstat failed on %s (%s)", - path, strerror (errno)); + if (ret != 0) { + if (ret == -1) { + if (errno != ENOENT) + gf_log (this->name, GF_LOG_WARNING, + "lstat failed on %s (%s)", + path, strerror (errno)); + } else { + // may be some backend filesytem issue + gf_log (this->name, GF_LOG_ERROR, "lstat failed on " + "%s and return value is %d instead of -1. " + "Please see dmesg output to check whether the " + "failure is due to backend filesystem issue", + path, ret); + ret = -1; + } goto out; } @@ -443,6 +456,7 @@ posix_gfid_set (xlator_t *this, const char *path, loc_t *loc, dict_t *xattr_req) void *uuid_req = NULL; uuid_t uuid_curr; int ret = 0; + ssize_t size = 0; struct stat stat = {0, }; @@ -452,8 +466,8 @@ posix_gfid_set (xlator_t *this, const char *path, loc_t *loc, dict_t *xattr_req) if (sys_lstat (path, &stat) != 0) goto out; - ret = sys_lgetxattr (path, GFID_XATTR_KEY, uuid_curr, 16); - if (ret == 16) { + size = sys_lgetxattr (path, GFID_XATTR_KEY, uuid_curr, 16); + if (size == 16) { ret = 0; goto verify_handle; } @@ -487,8 +501,8 @@ out: int -posix_set_file_contents (xlator_t *this, const char *path, data_pair_t *trav, - int flags) +posix_set_file_contents (xlator_t *this, const char *path, char *keyp, + data_t *value, int flags) { char * key = NULL; char real_path[PATH_MAX]; @@ -500,7 +514,7 @@ posix_set_file_contents (xlator_t *this, const char *path, data_pair_t *trav, /* XXX: does not handle assigning GFID to created files */ return -1; - key = &(trav->key[15]); + key = &(keyp[15]); sprintf (real_path, "%s/%s", path, key); if (flags & XATTR_REPLACE) { @@ -512,9 +526,8 @@ posix_set_file_contents (xlator_t *this, const char *path, data_pair_t *trav, goto create; } - if (trav->value->len) { - ret = write (file_fd, trav->value->data, - trav->value->len); + if (value->len) { + ret = write (file_fd, value->data, value->len); if (ret == -1) { op_ret = -errno; gf_log (this->name, GF_LOG_ERROR, @@ -546,7 +559,7 @@ posix_set_file_contents (xlator_t *this, const char *path, data_pair_t *trav, goto out; } - ret = write (file_fd, trav->value->data, trav->value->len); + ret = write (file_fd, value->data, value->len); if (ret == -1) { op_ret = -errno; gf_log (this->name, GF_LOG_ERROR, @@ -630,8 +643,7 @@ posix_get_file_contents (xlator_t *this, uuid_t pargfid, out: if (op_ret < 0) { - if (*contents) - GF_FREE (*contents); + GF_FREE (*contents); if (file_fd != -1) close (file_fd); } @@ -643,17 +655,17 @@ static int gf_xattr_enotsup_log; int posix_handle_pair (xlator_t *this, const char *real_path, - data_pair_t *trav, int flags) + char *key, data_t *value, int flags) { int sys_ret = -1; int ret = 0; - if (ZR_FILE_CONTENT_REQUEST(trav->key)) { - ret = posix_set_file_contents (this, real_path, trav, flags); + if (ZR_FILE_CONTENT_REQUEST(key)) { + ret = posix_set_file_contents (this, real_path, key, value, + flags); } else { - sys_ret = sys_lsetxattr (real_path, trav->key, - trav->value->data, - trav->value->len, flags); + sys_ret = sys_lsetxattr (real_path, key, value->data, + value->len, flags); if (sys_ret < 0) { if (errno == ENOTSUP) { @@ -663,12 +675,13 @@ posix_handle_pair (xlator_t *this, const char *real_path, "supported (try remounting " "brick with 'user_xattr' " "flag)"); - } else if (errno == ENOENT && - !posix_special_xattr (marker_xattrs, - trav->key)) { - gf_log (this->name, GF_LOG_ERROR, - "setxattr on %s failed: %s", real_path, - strerror (errno)); + } else if (errno == ENOENT) { + if (!posix_special_xattr (marker_xattrs, + key)) { + gf_log (this->name, GF_LOG_ERROR, + "setxattr on %s failed: %s", + real_path, strerror (errno)); + } } else { #ifdef GF_DARWIN_HOST_OS @@ -676,12 +689,12 @@ posix_handle_pair (xlator_t *this, const char *real_path, ((errno == EINVAL) ? GF_LOG_DEBUG : GF_LOG_ERROR), "%s: key:%s error:%s", - real_path, trav->key, + real_path, key, strerror (errno)); #else /* ! DARWIN */ gf_log (this->name, GF_LOG_ERROR, "%s: key:%s error:%s", - real_path, trav->key, + real_path, key, strerror (errno)); #endif /* DARWIN */ } @@ -696,13 +709,13 @@ out: int posix_fhandle_pair (xlator_t *this, int fd, - data_pair_t *trav, int flags) + char *key, data_t *value, int flags) { int sys_ret = -1; int ret = 0; - sys_ret = sys_fsetxattr (fd, trav->key, trav->value->data, - trav->value->len, flags); + sys_ret = sys_fsetxattr (fd, key, value->data, + value->len, flags); if (sys_ret < 0) { if (errno == ENOTSUP) { @@ -723,13 +736,11 @@ posix_fhandle_pair (xlator_t *this, int fd, ((errno == EINVAL) ? GF_LOG_DEBUG : GF_LOG_ERROR), "fd=%d: key:%s error:%s", - fd, trav->key, - strerror (errno)); + fd, key, strerror (errno)); #else /* ! DARWIN */ gf_log (this->name, GF_LOG_ERROR, "fd=%d: key:%s error:%s", - fd, trav->key, - strerror (errno)); + fd, key, strerror (errno)); #endif /* DARWIN */ } @@ -833,7 +844,7 @@ posix_janitor_thread_proc (void *data) time (&now); if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) { gf_log (this->name, GF_LOG_TRACE, - "janitor cleaning out /" GF_REPLICATE_TRASH_DIR); + "janitor cleaning out %s", priv->trash_path); nftw (priv->trash_path, janitor_walker, @@ -874,8 +885,8 @@ posix_spawn_janitor_thread (xlator_t *this) LOCK (&priv->lock); { if (!priv->janitor_present) { - ret = pthread_create (&priv->janitor, NULL, - posix_janitor_thread_proc, this); + ret = gf_thread_create (&priv->janitor, NULL, + posix_janitor_thread_proc, this); if (ret < 0) { gf_log (this->name, GF_LOG_ERROR, @@ -891,6 +902,74 @@ unlock: UNLOCK (&priv->lock); } +static int +is_fresh_file (struct stat *stat) +{ + struct timeval tv; + + gettimeofday (&tv, NULL); + + if ((stat->st_ctime >= (tv.tv_sec - 1)) + && (stat->st_ctime <= tv.tv_sec)) + return 1; + + return 0; +} + + +int +posix_gfid_heal (xlator_t *this, const char *path, loc_t *loc, dict_t *xattr_req) +{ + /* The purpose of this function is to prevent a race + where an inode creation FOP (like mkdir/mknod/create etc) + races with lookup in the following way: + + {create thread} | {lookup thread} + | + t0 + mkdir ("name") | + t1 + | posix_gfid_set ("name", 2); + t2 + posix_gfid_set ("name", 1); | + t3 + lstat ("name"); | lstat ("name"); + + In the above case mkdir FOP would have resulted with GFID 2 while + it should have been GFID 1. It matters in the case where GFID would + have gotten set to 1 on other subvolumes of replciate/distribute + + The "solution" here is that, if we detect lookup is attempting to + set a GFID on a file which is created very recently, but does not + yet have a GFID (i.e, between t1 and t2), then "fake" it as though + posix_gfid_heal was called at t0 instead. + */ + + uuid_t uuid_curr; + int ret = 0; + struct stat stat = {0, }; + + if (!xattr_req) + goto out; + + if (sys_lstat (path, &stat) != 0) + goto out; + + ret = sys_lgetxattr (path, GFID_XATTR_KEY, uuid_curr, 16); + if (ret != 16) { + if (is_fresh_file (&stat)) { + ret = -1; + errno = ENOENT; + goto out; + } + } + + ret = posix_gfid_set (this, path, loc, xattr_req); +out: + return ret; +} + + int posix_acl_xattr_set (xlator_t *this, const char *path, dict_t *xattr_req) { @@ -904,17 +983,17 @@ posix_acl_xattr_set (xlator_t *this, const char *path, dict_t *xattr_req) if (sys_lstat (path, &stat) != 0) goto out; - data = dict_get (xattr_req, "system.posix_acl_access"); + data = dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR); if (data) { - ret = sys_lsetxattr (path, "system.posix_acl_access", + ret = sys_lsetxattr (path, POSIX_ACL_ACCESS_XATTR, data->data, data->len, 0); if (ret != 0) goto out; } - data = dict_get (xattr_req, "system.posix_acl_default"); + data = dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR); if (data) { - ret = sys_lsetxattr (path, "system.posix_acl_default", + ret = sys_lsetxattr (path, POSIX_ACL_DEFAULT_XATTR, data->data, data->len, 0); if (ret != 0) goto out; @@ -924,37 +1003,47 @@ out: return ret; } +static int +_handle_entry_create_keyvalue_pair (dict_t *d, char *k, data_t *v, + void *tmp) +{ + int ret = -1; + posix_xattr_filler_t *filler = NULL; + + filler = tmp; + + if (!strcmp (GFID_XATTR_KEY, k) || + !strcmp ("gfid-req", k) || + !strcmp (POSIX_ACL_DEFAULT_XATTR, k) || + !strcmp (POSIX_ACL_ACCESS_XATTR, k) || + ZR_FILE_CONTENT_REQUEST(k)) { + return 0; + } + + ret = posix_handle_pair (filler->this, filler->real_path, k, v, + XATTR_CREATE); + if (ret < 0) { + errno = -ret; + return -1; + } + return 0; +} + int posix_entry_create_xattr_set (xlator_t *this, const char *path, dict_t *dict) { - data_pair_t *trav = NULL; int ret = -1; + posix_xattr_filler_t filler = {0,}; + if (!dict) goto out; - trav = dict->members_list; - while (trav) { - if (!strcmp (GFID_XATTR_KEY, trav->key) || - !strcmp ("gfid-req", trav->key) || - !strcmp ("system.posix_acl_default", trav->key) || - !strcmp ("system.posix_acl_access", trav->key) || - ZR_FILE_CONTENT_REQUEST(trav->key)) { - trav = trav->next; - continue; - } - - ret = posix_handle_pair (this, path, trav, XATTR_CREATE); - if (ret < 0) { - errno = -ret; - ret = -1; - goto out; - } - trav = trav->next; - } + filler.this = this; + filler.real_path = path; - ret = 0; + ret = dict_foreach (dict, _handle_entry_create_keyvalue_pair, &filler); out: return ret; @@ -978,7 +1067,7 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p) goto out; } - if (fd->pid != -1) + if (!fd_is_anonymous(fd)) /* anonymous fd */ goto out; @@ -1045,10 +1134,258 @@ posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd) return ret; } +static void * +posix_health_check_thread_proc (void *data) +{ + xlator_t *this = NULL; + struct posix_private *priv = NULL; + uint32_t interval = 0; + int ret = -1; + struct stat sb = {0, }; + + this = data; + priv = this->private; + + /* prevent races when the interval is updated */ + interval = priv->health_check_interval; + if (interval == 0) + goto out; + + gf_log (this->name, GF_LOG_DEBUG, "health-check thread started, " + "interval = %d seconds", interval); + + while (1) { + /* aborting sleep() is a request to exit this thread, sleep() + * will normally not return when cancelled */ + ret = sleep (interval); + if (ret > 0) + break; + + /* prevent thread errors while doing the health-check(s) */ + pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, NULL); + + /* Do the health-check, it should be moved to its own function + * in case it gets more complex. */ + ret = stat (priv->base_path, &sb); + if (ret < 0) { + gf_log (this->name, GF_LOG_WARNING, + "stat() on %s returned: %s", priv->base_path, + strerror (errno)); + goto abort; + } + + pthread_setcancelstate (PTHREAD_CANCEL_ENABLE, NULL); + } + +out: + gf_log (this->name, GF_LOG_DEBUG, "health-check thread exiting"); + + LOCK (&priv->lock); + { + priv->health_check_active = _gf_false; + } + UNLOCK (&priv->lock); + + return NULL; + +abort: + /* health-check failed */ + gf_log (this->name, GF_LOG_EMERG, "health-check failed, going down"); + xlator_notify (this->parents->xlator, GF_EVENT_CHILD_DOWN, this); + + ret = sleep (30); + if (ret == 0) { + gf_log (this->name, GF_LOG_EMERG, "still alive! -> SIGTERM"); + kill (getpid(), SIGTERM); + } + + ret = sleep (30); + if (ret == 0) { + gf_log (this->name, GF_LOG_EMERG, "still alive! -> SIGKILL"); + kill (getpid(), SIGKILL); + } + + return NULL; +} + +void +posix_spawn_health_check_thread (xlator_t *xl) +{ + struct posix_private *priv = NULL; + int ret = -1; + + priv = xl->private; + + LOCK (&priv->lock); + { + /* cancel the running thread */ + if (priv->health_check_active == _gf_true) { + pthread_cancel (priv->health_check); + priv->health_check_active = _gf_false; + } + + /* prevent scheduling a check in a tight loop */ + if (priv->health_check_interval == 0) + goto unlock; + + ret = gf_thread_create (&priv->health_check, NULL, + posix_health_check_thread_proc, xl); + if (ret < 0) { + priv->health_check_interval = 0; + priv->health_check_active = _gf_false; + gf_log (xl->name, GF_LOG_ERROR, + "unable to setup health-check thread: %s", + strerror (errno)); + goto unlock; + } + + /* run the thread detached, resources will be freed on exit */ + pthread_detach (priv->health_check); + priv->health_check_active = _gf_true; + } +unlock: + UNLOCK (&priv->lock); +} int -posix_fd_ctx_get_off (fd_t *fd, xlator_t *this, struct posix_fd **pfd, - off_t offset) +posix_fsyncer_pick (xlator_t *this, struct list_head *head) +{ + struct posix_private *priv = NULL; + int count = 0; + + priv = this->private; + pthread_mutex_lock (&priv->fsync_mutex); + { + while (list_empty (&priv->fsyncs)) + pthread_cond_wait (&priv->fsync_cond, + &priv->fsync_mutex); + + count = priv->fsync_queue_count; + priv->fsync_queue_count = 0; + list_splice_init (&priv->fsyncs, head); + } + pthread_mutex_unlock (&priv->fsync_mutex); + + return count; +} + + +void +posix_fsyncer_process (xlator_t *this, call_stub_t *stub, gf_boolean_t do_fsync) +{ + struct posix_fd *pfd = NULL; + int ret = -1; + struct posix_private *priv = NULL; + + priv = this->private; + + ret = posix_fd_ctx_get (stub->args.fd, this, &pfd); + if (ret < 0) { + gf_log (this->name, GF_LOG_ERROR, + "could not get fdctx for fd(%s)", + uuid_utoa (stub->args.fd->inode->gfid)); + call_unwind_error (stub, -1, EINVAL); + return; + } + + if (do_fsync) { +#ifdef HAVE_FDATASYNC + if (stub->args.datasync) + ret = fdatasync (pfd->fd); + else +#endif + ret = fsync (pfd->fd); + } else { + ret = 0; + } + + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "could not fstat fd(%s)", + uuid_utoa (stub->args.fd->inode->gfid)); + call_unwind_error (stub, -1, errno); + return; + } + + call_unwind_error (stub, 0, 0); +} + + +static void +posix_fsyncer_syncfs (xlator_t *this, struct list_head *head) +{ + call_stub_t *stub = NULL; + struct posix_fd *pfd = NULL; + int ret = -1; + + stub = list_entry (head->prev, call_stub_t, list); + ret = posix_fd_ctx_get (stub->args.fd, this, &pfd); + if (ret) + return; + +#ifdef GF_LINUX_HOST_OS + /* syncfs() is not "declared" in RHEL's glibc even though + the kernel has support. + */ +#include <sys/syscall.h> +#include <unistd.h> +#ifdef SYS_syncfs + syscall (SYS_syncfs, pfd->fd); +#else + sync(); +#endif +#else + sync(); +#endif +} + + +void * +posix_fsyncer (void *d) { - return posix_fd_ctx_get (fd, this, pfd); + xlator_t *this = d; + struct posix_private *priv = NULL; + call_stub_t *stub = NULL; + call_stub_t *tmp = NULL; + struct list_head list; + int count = 0; + gf_boolean_t do_fsync = _gf_true; + + priv = this->private; + + for (;;) { + INIT_LIST_HEAD (&list); + + count = posix_fsyncer_pick (this, &list); + + usleep (priv->batch_fsync_delay_usec); + + gf_log (this->name, GF_LOG_DEBUG, + "picked %d fsyncs", count); + + switch (priv->batch_fsync_mode) { + case BATCH_NONE: + case BATCH_REVERSE_FSYNC: + break; + case BATCH_SYNCFS: + case BATCH_SYNCFS_SINGLE_FSYNC: + case BATCH_SYNCFS_REVERSE_FSYNC: + posix_fsyncer_syncfs (this, &list); + break; + } + + if (priv->batch_fsync_mode == BATCH_SYNCFS) + do_fsync = _gf_false; + else + do_fsync = _gf_true; + + list_for_each_entry_safe_reverse (stub, tmp, &list, list) { + list_del_init (&stub->list); + + posix_fsyncer_process (this, stub, do_fsync); + + if (priv->batch_fsync_mode == BATCH_SYNCFS_SINGLE_FSYNC) + do_fsync = _gf_false; + } + } } |
