/* Copyright (c) 2008-2012 Red Hat, Inc. This file is part of GlusterFS. This file is licensed to you under your choice of the GNU Lesser General Public License, version 3 or any later version (LGPLv3 or later), or the GNU General Public License, version 2 (GPLv2), in all cases as published by the Free Software Foundation. */ #include #include #include #include #include #include #include #ifndef _CONFIG_H #define _CONFIG_H #include "config.h" #endif #include "glusterfs.h" #include "dict.h" #include "xlator.h" #include "hashfn.h" #include "logging.h" #include "stack.h" #include "list.h" #include "call-stub.h" #include "defaults.h" #include "common-utils.h" #include "compat-errno.h" #include "compat.h" #include "checksum.h" #include "afr.h" #include "afr-self-heal.h" #include "afr-self-heal-common.h" int afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret, int32_t op_errno, int32_t sh_failed) { afr_local_t *local = NULL; local = frame->local; afr_set_opendir_done (this, local->fd->inode); AFR_STACK_UNWIND (opendir, frame, local->op_ret, local->op_errno, local->fd, NULL); return 0; } gf_boolean_t __checksums_differ (uint32_t *checksum, int child_count, unsigned char *child_up) { int ret = _gf_false; int i = 0; uint32_t cksum = 0; gf_boolean_t activate_check = _gf_false; for (i = 0; i < child_count; i++) { if (!child_up[i]) continue; if (_gf_false == activate_check) { cksum = checksum[i]; activate_check = _gf_true; continue; } if (cksum != checksum[i]) { ret = _gf_true; break; } cksum = checksum[i]; } return ret; } int32_t afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata) { afr_private_t * priv = NULL; afr_local_t * local = NULL; afr_self_heal_t * sh = NULL; gf_dirent_t * entry = NULL; gf_dirent_t * tmp = NULL; char *reason = NULL; int child_index = 0; uint32_t entry_cksum = 0; int call_count = 0; off_t last_offset = 0; inode_t *inode = NULL; priv = this->private; local = frame->local; sh = &local->self_heal; inode = local->fd->inode; child_index = (long) cookie; if (op_ret == -1) { gf_log (this->name, GF_LOG_INFO, "%s: failed to do opendir on %s", local->loc.path, priv->children[child_index]->name); local->op_ret = -1; local->op_ret = op_errno; goto out; } if (op_ret == 0) { gf_log (this->name, GF_LOG_DEBUG, "%s: no entries found in %s", local->loc.path, priv->children[child_index]->name); goto out; } list_for_each_entry_safe (entry, tmp, &entries->list, list) { entry_cksum = gf_rsync_weak_checksum ((unsigned char *)entry->d_name, strlen (entry->d_name)); local->cont.opendir.checksum[child_index] ^= entry_cksum; } list_for_each_entry (entry, &entries->list, list) { last_offset = entry->d_off; } /* read more entries */ STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk, (void *) (long) child_index, priv->children[child_index], priv->children[child_index]->fops->readdir, local->fd, 131072, last_offset, NULL); return 0; out: call_count = afr_frame_return (frame); if (call_count == 0) { if (__checksums_differ (local->cont.opendir.checksum, priv->child_count, local->child_up)) { sh->do_entry_self_heal = _gf_true; sh->forced_merge = _gf_true; reason = "checksums of directory differ"; afr_launch_self_heal (frame, this, inode, _gf_false, inode->ia_type, reason, NULL, afr_examine_dir_sh_unwind); } else { afr_set_opendir_done (this, inode); AFR_STACK_UNWIND (opendir, frame, local->op_ret, local->op_errno, local->fd, NULL); } } return 0; } int afr_examine_dir (call_frame_t *frame, xlator_t *this) { afr_private_t * priv = NULL; afr_local_t * local = NULL; int i = 0; int call_count = 0; local = frame->local; priv = this->private; local->cont.opendir.checksum = GF_CALLOC (priv->child_count, sizeof (*local->cont.opendir.checksum), gf_afr_mt_int32_t); call_count = afr_up_children_count (local->child_up, priv->child_count); local->call_count = call_count; for (i = 0; i < priv->child_count; i++) { if (local->child_up[i]) { STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk, (void *) (long) i, priv->children[i], priv->children[i]->fops->readdir, local->fd, 131072, 0, NULL); if (!--call_count) break; } } return 0; } int32_t afr_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata) { afr_private_t *priv = NULL; afr_local_t *local = NULL; int32_t up_children_count = 0; int ret = -1; int call_count = -1; int32_t child_index = 0; priv = this->private; local = frame->local; child_index = (long) cookie; up_children_count = afr_up_children_count (local->child_up, priv->child_count); LOCK (&frame->lock); { if (op_ret >= 0) { local->op_ret = op_ret; ret = afr_child_fd_ctx_set (this, fd, child_index, 0); if (ret) { local->op_ret = -1; local->op_errno = -ret; goto unlock; } } local->op_errno = op_errno; } unlock: UNLOCK (&frame->lock); call_count = afr_frame_return (frame); if (call_count == 0) { if (local->op_ret != 0) goto out; if (!afr_is_opendir_done (this, local->fd->inode) && up_children_count > 1) { /* * This is the first opendir on this inode. We need * to check if the directory's entries are the same * on all subvolumes. This is needed in addition * to regular entry self-heal because the readdir * call is sent only to the first subvolume, and * thus files that exist only there will never be healed * otherwise (assuming changelog shows no anomalies). */ gf_log (this->name, GF_LOG_TRACE, "reading contents of directory %s looking for mismatch", local->loc.path); afr_examine_dir (frame, this); } else { /* do the unwind */ goto out; } } return 0; out: AFR_STACK_UNWIND (opendir, frame, local->op_ret, local->op_errno, local->fd, NULL); return 0; } int32_t afr_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd) { afr_private_t * priv = NULL; afr_local_t * local = NULL; int child_count = 0; int i = 0; int ret = -1; int call_count = -1; int32_t op_errno = 0; VALIDATE_OR_GOTO (frame, out); VALIDATE_OR_GOTO (this, out); VALIDATE_OR_GOTO (this->private, out); priv = this->private; child_count = priv->child_count; AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out); local = frame->local; ret = afr_local_init (local, priv, &op_errno); if (ret < 0) goto out; loc_copy (&local->loc, loc); local->fd = fd_ref (fd); call_count = local->call_count; for (i = 0; i < child_count; i++) { if (local->child_up[i]) { STACK_WIND_COOKIE (frame, afr_opendir_cbk, (void*) (long) i, priv->children[i], priv->children[i]->fops->opendir, loc, fd, NULL); if (!--call_count) break; } } ret = 0; out: if (ret < 0) AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL); return 0; } /** * Common algorithm for directory read calls: * * - Try the fop on the first child that is up * - if we have failed due to ENOTCONN: * try the next child * * Applicable to: readdir */ struct entry_name { char *name; struct list_head list; }; static gf_boolean_t remembered_name (const char *name, struct list_head *entries) { struct entry_name *e = NULL; gf_boolean_t ret = _gf_false; list_for_each_entry (e, entries, list) { if (!strcmp (name, e->name)) { ret = _gf_true; goto out; } } out: return ret; } static void afr_remember_entries (gf_dirent_t *entries, fd_t *fd) { struct entry_name *n = NULL; gf_dirent_t *entry = NULL; int ret = 0; uint64_t ctx = 0; afr_fd_ctx_t *fd_ctx = NULL; ret = fd_ctx_get (fd, THIS, &ctx); if (ret < 0) { gf_log (THIS->name, GF_LOG_INFO, "could not get fd ctx for fd=%p", fd); return; } fd_ctx = (afr_fd_ctx_t *)(long) ctx; list_for_each_entry (entry, &entries->list, list) { n = GF_CALLOC (1, sizeof (*n), gf_afr_mt_entry_name); n->name = gf_strdup (entry->d_name); INIT_LIST_HEAD (&n->list); list_add (&n->list, &fd_ctx->entries); } } static off_t afr_filter_entries (gf_dirent_t *entries, fd_t *fd) { gf_dirent_t *entry = NULL; gf_dirent_t *tmp = NULL; int ret = 0; uint64_t ctx = 0; afr_fd_ctx_t *fd_ctx = NULL; off_t offset = 0; ret = fd_ctx_get (fd, THIS, &ctx); if (ret < 0) { gf_log (THIS->name, GF_LOG_INFO, "could not get fd ctx for fd=%p", fd); return -1; } fd_ctx = (afr_fd_ctx_t *)(long) ctx; list_for_each_entry_safe (entry, tmp, &entries->list, list) { offset = entry->d_off; if (remembered_name (entry->d_name, &fd_ctx->entries)) { list_del (&entry->list); GF_FREE (entry); } } return offset; } static void afr_forget_entries (fd_t *fd) { struct entry_name *entry = NULL; struct entry_name *tmp = NULL; int ret = 0; uint64_t ctx = 0; afr_fd_ctx_t *fd_ctx = NULL; ret = fd_ctx_get (fd, THIS, &ctx); if (ret < 0) { gf_log (THIS->name, GF_LOG_INFO, "could not get fd ctx for fd=%p", fd); return; } fd_ctx = (afr_fd_ctx_t *)(long) ctx; list_for_each_entry_safe (entry, tmp, &fd_ctx->entries, list) { GF_FREE (entry->name); list_del (&entry->list); GF_FREE (entry); } } int32_t afr_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata) { AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, NULL); return 0; } int32_t afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata) { afr_private_t * priv = NULL; afr_local_t * local = NULL; xlator_t ** children = NULL; int32_t next_call_child = -1; int ret = 0; int32_t *last_index = NULL; int32_t read_child = -1; int32_t *fresh_children = NULL; uint64_t ctx = 0; afr_fd_ctx_t *fd_ctx = NULL; off_t offset = 0; int32_t call_child = -1; priv = this->private; children = priv->children; local = frame->local; if ((priv->readdir_failover == _gf_false) && (op_ret < 0)) goto out; read_child = (long) cookie; last_index = &local->cont.readdir.last_index; fresh_children = local->fresh_children; /* the value of the last_index changes if afr_next_call_child is * called. So to find the call_child of this callback use last_index * before the next_call_child call. */ if (*last_index == -1) call_child = read_child; else call_child = fresh_children[*last_index]; if (priv->strict_readdir) { ret = fd_ctx_get (local->fd, this, &ctx); if (ret < 0) { gf_log (this->name, GF_LOG_INFO, "could not get fd ctx for fd=%p", local->fd); op_ret = -1; op_errno = -ret; goto out; } fd_ctx = (afr_fd_ctx_t *)(long) ctx; if (op_ret == -1) { next_call_child = afr_next_call_child (fresh_children, local->child_up, priv->child_count, last_index, read_child); if (next_call_child < 0) goto out; gf_log (this->name, GF_LOG_TRACE, "starting readdir afresh on child %d, offset %"PRId64, next_call_child, (uint64_t) 0); fd_ctx->failed_over = _gf_true; STACK_WIND_COOKIE (frame, afr_readdirp_cbk, (void *) (long) read_child, children[next_call_child], children[next_call_child]->fops->readdirp, local->fd, local->cont.readdir.size, 0, local->cont.readdir.dict); return 0; } } if (priv->strict_readdir) { if (fd_ctx->failed_over) { if (list_empty (&entries->list)) { gf_log (this->name, GF_LOG_DEBUG, "no entries found"); goto out; } offset = afr_filter_entries (entries, local->fd); afr_remember_entries (entries, local->fd); if (list_empty (&entries->list)) { /* All the entries we got were duplicate. We shouldn't send an empty list now, because that will make the application stop reading. So try to get more entries */ gf_log (this->name, GF_LOG_TRACE, "trying to fetch non-duplicate entries " "from offset %"PRId64", child %s", offset, children[call_child]->name); STACK_WIND_COOKIE (frame, afr_readdirp_cbk, (void *) (long) read_child, children[call_child], children[call_child]->fops->readdirp, local->fd, local->cont.readdir.size, offset, local->cont.readdir.dict); return 0; } } else { afr_remember_entries (entries, local->fd); } } out: AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, NULL); return 0; } int32_t afr_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t offset, int whichop, dict_t *dict) { afr_private_t *priv = NULL; xlator_t **children = NULL; int call_child = 0; afr_local_t *local = NULL; afr_fd_ctx_t *fd_ctx = NULL; int ret = -1; int32_t op_errno = 0; uint64_t read_child = 0; VALIDATE_OR_GOTO (frame, out); VALIDATE_OR_GOTO (this, out); VALIDATE_OR_GOTO (this->private, out); priv = this->private; children = priv->children; AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out); local = frame->local; ret = afr_local_init (local, priv, &op_errno); if (ret < 0) goto out; local->fresh_children = afr_children_create (priv->child_count); if (!local->fresh_children) { op_errno = ENOMEM; goto out; } read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children); ret = afr_get_call_child (this, local->child_up, read_child, local->fresh_children, &call_child, &local->cont.readdir.last_index); if (ret < 0) { op_errno = -ret; goto out; } fd_ctx = afr_fd_ctx_get (fd, this); if (!fd_ctx) { op_errno = EBADF; goto out; } if ((offset == 0) || (fd_ctx->call_child == -1)) { fd_ctx->call_child = call_child; } else if ((priv->readdir_failover == _gf_false) && (call_child != fd_ctx->call_child)) { op_errno = EBADF; goto out; } local->fd = fd_ref (fd); local->cont.readdir.size = size; local->cont.readdir.dict = (dict)? dict_ref (dict) : NULL; if (priv->strict_readdir) { if (fd_ctx->last_tried != call_child) { gf_log (this->name, GF_LOG_TRACE, "first up child has changed from %d to %d, " "restarting readdir from offset 0", fd_ctx->last_tried, call_child); fd_ctx->failed_over = _gf_true; offset = 0; } fd_ctx->last_tried = call_child; } if (whichop == GF_FOP_READDIR) STACK_WIND_COOKIE (frame, afr_readdir_cbk, (void *) (long) call_child, children[call_child], children[call_child]->fops->readdir, fd, size, offset, dict); else STACK_WIND_COOKIE (frame, afr_readdirp_cbk, (void *) (long) call_child, children[call_child], children[call_child]->fops->readdirp, fd, size, offset, dict); return 0; out: AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL); return 0; } int32_t afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t offset, dict_t *xdata) { afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata); return 0; } int32_t afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t offset, dict_t *dict) { afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict); return 0; } int32_t afr_releasedir (xlator_t *this, fd_t *fd) { afr_forget_entries (fd); afr_cleanup_fd_ctx (this, fd); return 0; }