path: root/xlators/cluster/afr/src/afr-self-heal-name.c

/*
  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
  This file is part of GlusterFS.

  This file is licensed to you under your choice of the GNU Lesser
  General Public License, version 3 or any later version (LGPLv3 or
  later), or the GNU General Public License, version 2 (GPLv2), in all
  cases as published by the Free Software Foundation.
*/

#include <glusterfs/events.h>
#include "afr.h"
#include "afr-self-heal.h"
#include "afr-messages.h"

int
__afr_selfheal_assign_gfid(xlator_t *this, inode_t *parent, uuid_t pargfid,
                           const char *bname, inode_t *inode,
                           struct afr_reply *replies, void *gfid,
                           unsigned char *locked_on, int source,
                           unsigned char *sources, gf_boolean_t is_gfid_absent,
                           int *gfid_idx)
{
    int ret = 0;
    int up_count = 0;
    int locked_count = 0;
    afr_private_t *priv = NULL;

    priv = this->private;

    gf_uuid_copy(parent->gfid, pargfid);

    if (is_gfid_absent) {
        /* Ensure all children of AFR are up before performing gfid heal, to
         * guard against the possibility of gfid split brain. */

        up_count = AFR_COUNT(priv->child_up, priv->child_count);
        if (up_count != priv->child_count) {
            ret = -EIO;
            goto out;
        }

        locked_count = AFR_COUNT(locked_on, priv->child_count);
        if (locked_count != priv->child_count) {
            ret = -EIO;
            goto out;
        }
    }

    ret = afr_lookup_and_heal_gfid(this, parent, bname, inode, replies, source,
                                   sources, gfid, gfid_idx);

out:
    return ret;
}

int
__afr_selfheal_name_impunge(call_frame_t *frame, xlator_t *this,
                            inode_t *parent, uuid_t pargfid, const char *bname,
                            inode_t *inode, struct afr_reply *replies,
                            int gfid_idx)
{
    int i = 0;
    afr_private_t *priv = NULL;
    int ret = 0;
    unsigned char *sources = NULL;

    priv = this->private;

    sources = alloca0(priv->child_count);

    gf_uuid_copy(parent->gfid, pargfid);

    for (i = 0; i < priv->child_count; i++) {
        if (!replies[i].valid || replies[i].op_ret != 0)
            continue;

        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
                            replies[gfid_idx].poststat.ia_gfid) == 0) {
            sources[i] = 1;
            continue;
        }
    }

    for (i = 0; i < priv->child_count; i++) {
        if (sources[i])
            continue;

        ret |= afr_selfheal_recreate_entry(frame, i, gfid_idx, sources, parent,
                                           bname, inode, replies);
    }

    return ret;
}

int
__afr_selfheal_name_expunge(xlator_t *this, inode_t *parent, uuid_t pargfid,
                            const char *bname, inode_t *inode,
                            struct afr_reply *replies)
{
    loc_t loc = {
        0,
    };
    int i = 0;
    afr_private_t *priv = NULL;
    char g[64];
    int ret = 0;

    priv = this->private;

    loc.parent = inode_ref(parent);
    gf_uuid_copy(loc.pargfid, pargfid);
    loc.name = bname;
    loc.inode = inode_ref(inode);

    for (i = 0; i < priv->child_count; i++) {
        if (!replies[i].valid)
            continue;

        if (replies[i].op_ret)
            continue;

        switch (replies[i].poststat.ia_type) {
            case IA_IFDIR:
                gf_msg(this->name, GF_LOG_WARNING, 0,
                       AFR_MSG_EXPUNGING_FILE_OR_DIR,
                       "expunging dir %s/%s (%s) on %s", uuid_utoa(pargfid),
                       bname, uuid_utoa_r(replies[i].poststat.ia_gfid, g),
                       priv->children[i]->name);

                ret |= syncop_rmdir(priv->children[i], &loc, 1, NULL, NULL);
                break;
            default:
                gf_msg(this->name, GF_LOG_WARNING, 0,
                       AFR_MSG_EXPUNGING_FILE_OR_DIR,
                       "expunging file %s/%s (%s) on %s", uuid_utoa(pargfid),
                       bname, uuid_utoa_r(replies[i].poststat.ia_gfid, g),
                       priv->children[i]->name);

                ret |= syncop_unlink(priv->children[i], &loc, NULL, NULL);
                break;
        }
    }

    loc_wipe(&loc);

    return ret;
}

static gf_boolean_t
afr_selfheal_name_need_heal_check(xlator_t *this, struct afr_reply *replies)
{
    int i = 0;
    int first_idx = -1;
    gf_boolean_t need_heal = _gf_false;
    afr_private_t *priv = NULL;

    priv = this->private;

    for (i = 0; i < priv->child_count; i++) {
        if (!replies[i].valid)
            continue;

        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA))
            need_heal = _gf_true;

        if (first_idx == -1) {
            first_idx = i;
            continue;
        }

        if (replies[i].op_ret != replies[first_idx].op_ret)
            need_heal = _gf_true;

        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
                            replies[first_idx].poststat.ia_gfid))
            need_heal = _gf_true;

        if ((replies[i].op_ret == 0) &&
            (gf_uuid_is_null(replies[i].poststat.ia_gfid)))
            need_heal = _gf_true;
    }

    return need_heal;
}

static int
afr_selfheal_name_type_mismatch_check(xlator_t *this, struct afr_reply *replies,
                                      int source, unsigned char *sources,
                                      uuid_t pargfid, const char *bname)
{
    int i = 0;
    int type_idx = -1;
    ia_type_t inode_type = IA_INVAL;
    ia_type_t inode_type1 = IA_INVAL;
    afr_private_t *priv = NULL;

    priv = this->private;

    for (i = 0; i < priv->child_count; i++) {
        if (!replies[i].valid || replies[i].op_ret != 0)
            continue;

        if (replies[i].poststat.ia_type == IA_INVAL)
            continue;

        if (inode_type == IA_INVAL) {
            inode_type = replies[i].poststat.ia_type;
            type_idx = i;
            continue;
        }
        inode_type1 = replies[i].poststat.ia_type;
        if (sources[i] || source == -1) {
            if ((sources[type_idx] || source == -1) &&
                (inode_type != inode_type1)) {
                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN,
                       "Type mismatch for <gfid:%s>/%s: "
                       "%s on %s and %s on %s",
                       uuid_utoa(pargfid), bname,
                       gf_inode_type_to_str(inode_type1),
                       priv->children[i]->name,
                       gf_inode_type_to_str(inode_type),
                       priv->children[type_idx]->name);
                gf_event(EVENT_AFR_SPLIT_BRAIN,
                         "subvol=%s;type=file;"
                         "file=<gfid:%s>/%s;count=2;"
                         "child-%d=%s;type-%d=%s;child-%d=%s;"
                         "type-%d=%s",
                         this->name, uuid_utoa(pargfid), bname, i,
                         priv->children[i]->name, i,
                         gf_inode_type_to_str(inode_type1), type_idx,
                         priv->children[type_idx]->name, type_idx,
                         gf_inode_type_to_str(inode_type));
                return -EIO;
            }
            inode_type = replies[i].poststat.ia_type;
            type_idx = i;
        }
    }
    return 0;
}

static int
afr_selfheal_name_gfid_mismatch_check(xlator_t *this, struct afr_reply *replies,
                                      int source, unsigned char *sources,
                                      int *gfid_idx, uuid_t pargfid,
                                      const char *bname, inode_t *inode,
                                      unsigned char *locked_on, dict_t *xdata)
{
    int i = 0;
    int gfid_idx_iter = -1;
    int ret = -1;
    void *gfid = NULL;
    void *gfid1 = NULL;
    afr_private_t *priv = NULL;

    priv = this->private;

    for (i = 0; i < priv->child_count; i++) {
        if (!replies[i].valid || replies[i].op_ret != 0)
            continue;

        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
            continue;

        if (!gfid) {
            gfid = &replies[i].poststat.ia_gfid;
            gfid_idx_iter = i;
            continue;
        }

        gfid1 = &replies[i].poststat.ia_gfid;
        if (sources[i] || source == -1) {
            if ((sources[gfid_idx_iter] || source == -1) &&
                gf_uuid_compare(gfid, gfid1)) {
                ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
                                                  bname, gfid_idx_iter, i,
                                                  locked_on, gfid_idx, xdata);
                if (!ret && *gfid_idx >= 0) {
                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
                                                   "GFID split-brain resolved");
                    if (ret)
                        gf_msg(this->name, GF_LOG_ERROR, 0,
                               AFR_MSG_DICT_SET_FAILED,
                               "Error setting gfid-"
                               "heal-msg dict");
                }
                return ret;
            }
            gfid = &replies[i].poststat.ia_gfid;
            gfid_idx_iter = i;
        }
    }

    *gfid_idx = gfid_idx_iter;
    return 0;
}

static gf_boolean_t
afr_selfheal_name_source_empty_check(xlator_t *this, struct afr_reply *replies,
                                     unsigned char *sources, int source)
{
    int i = 0;
    afr_private_t *priv = NULL;
    gf_boolean_t source_is_empty = _gf_true;

    priv = this->private;

    if (source == -1) {
        source_is_empty = _gf_false;
        goto out;
    }

    for (i = 0; i < priv->child_count; i++) {
        if (!sources[i])
            continue;

        if (replies[i].op_ret == -1 && replies[i].op_errno == ENOENT)
            continue;

        source_is_empty = _gf_false;
        break;
    }
out:
    return source_is_empty;
}

int
__afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
                       uuid_t pargfid, const char *bname, inode_t *inode,
                       unsigned char *sources, unsigned char *sinks,
                       unsigned char *healed_sinks, int source,
                       unsigned char *locked_on, struct afr_reply *replies,
                       void *gfid_req, dict_t *xdata)
{
    int gfid_idx = -1;
    int ret = -1;
    void *gfid = NULL;
    gf_boolean_t source_is_empty = _gf_true;
    gf_boolean_t need_heal = _gf_false;
    gf_boolean_t is_gfid_absent = _gf_false;

    need_heal = afr_selfheal_name_need_heal_check(this, replies);
    if (!need_heal)
        return 0;

    source_is_empty = afr_selfheal_name_source_empty_check(this, replies,
                                                           sources, source);
    if (source_is_empty) {
        ret = __afr_selfheal_name_expunge(this, parent, pargfid, bname, inode,
                                          replies);
        if (ret == -EIO)
            ret = -1;
        return ret;
    }

    ret = afr_selfheal_name_type_mismatch_check(this, replies, source, sources,
                                                pargfid, bname);
    if (ret)
        return ret;

    ret = afr_selfheal_name_gfid_mismatch_check(this, replies, source, sources,
                                                &gfid_idx, pargfid, bname,
                                                inode, locked_on, xdata);
    if (ret)
        return ret;

    if (gfid_idx == -1) {
        if (!gfid_req || gf_uuid_is_null(gfid_req))
            return -1;
        gfid = gfid_req;
    } else {
        gfid = &replies[gfid_idx].poststat.ia_gfid;
        if (source == -1)
            /* Either entry split-brain or dirty xattrs are present on parent.*/
            source = gfid_idx;
    }

    is_gfid_absent = (gfid_idx == -1) ? _gf_true : _gf_false;
    ret = __afr_selfheal_assign_gfid(this, parent, pargfid, bname, inode,
                                     replies, gfid, locked_on, source, sources,
                                     is_gfid_absent, &gfid_idx);
    if (ret)
        return ret;

    ret = __afr_selfheal_name_impunge(frame, this, parent, pargfid, bname,
                                      inode, replies, gfid_idx);
    if (ret == -EIO)
        ret = -1;

    return ret;
}

int
__afr_selfheal_name_finalize_source(xlator_t *this, unsigned char *sources,
                                    unsigned char *healed_sinks,
                                    unsigned char *locked_on, uint64_t *witness)
{
    int i = 0;
    afr_private_t *priv = NULL;
    int source = -1;
    int sources_count = 0;

    priv = this->private;

    sources_count = AFR_COUNT(sources, priv->child_count);

    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
        !sources_count || afr_does_witness_exist(this, witness)) {
        memset(sources, 0, sizeof(*sources) * priv->child_count);
        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
        return -1;
    }

    for (i = 0; i < priv->child_count; i++) {
        if (sources[i]) {
            source = i;
            break;
        }
    }

    return source;
}

int
__afr_selfheal_name_prepare(call_frame_t *frame, xlator_t *this,
                            inode_t *parent, uuid_t pargfid,
                            unsigned char *locked_on, unsigned char *sources,
                            unsigned char *sinks, unsigned char *healed_sinks,
                            int *source_p)
{
    int ret = -1;
    int source = -1;
    afr_private_t *priv = NULL;
    struct afr_reply *replies = NULL;
    uint64_t *witness = NULL;

    priv = this->private;

    replies = alloca0(priv->child_count * sizeof(*replies));

    ret = afr_selfheal_unlocked_discover(frame, parent, pargfid, replies);
    if (ret)
        goto out;

    witness = alloca0(sizeof(*witness) * priv->child_count);
    ret = afr_selfheal_find_direction(frame, this, replies,
                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
                                      sinks, witness, NULL);
    if (ret)
        goto out;

    /* Initialize the healed_sinks[] array optimistically to
       the intersection of to-be-healed (i.e sinks[]) and
       the list of servers which are up (i.e locked_on[]).

       As we encounter failures in the healing process, we
       will unmark the respective servers in the healed_sinks[]
       array.
    */
    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);

    source = __afr_selfheal_name_finalize_source(this, sources, healed_sinks,
                                                 locked_on, witness);
    if (source < 0) {
        /* If source is < 0 (typically split-brain), we perform a
           conservative merge of entries rather than erroring out */
    }
    *source_p = source;

out:
    if (replies)
        afr_replies_wipe(replies, priv->child_count);

    return ret;
}

int
afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
                     uuid_t pargfid, const char *bname, void *gfid_req,
                     dict_t *xdata)
{
    afr_private_t *priv = NULL;
    unsigned char *sources = NULL;
    unsigned char *sinks = NULL;
    unsigned char *healed_sinks = NULL;
    unsigned char *locked_on = NULL;
    int source = -1;
    struct afr_reply *replies = NULL;
    int ret = -1;
    inode_t *inode = NULL;
    dict_t *xattr = NULL;

    xattr = dict_new();
    if (!xattr)
        return -ENOMEM;

    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
    if (ret) {
        dict_unref(xattr);
        return -1;
    }

    priv = this->private;

    locked_on = alloca0(priv->child_count);
    sources = alloca0(priv->child_count);
    sinks = alloca0(priv->child_count);
    healed_sinks = alloca0(priv->child_count);

    replies = alloca0(priv->child_count * sizeof(*replies));

    ret = afr_selfheal_entrylk(frame, this, parent, this->name, bname,
                               locked_on);
    {
        if (ret < AFR_SH_MIN_PARTICIPANTS) {
            ret = -ENOTCONN;
            goto unlock;
        }

        ret = __afr_selfheal_name_prepare(frame, this, parent, pargfid,
                                          locked_on, sources, sinks,
                                          healed_sinks, &source);
        if (ret)
            goto unlock;

        inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
                                                locked_on, xattr);
        if (!inode) {
            ret = -ENOMEM;
            goto unlock;
        }

        ret = __afr_selfheal_name_do(frame, this, parent, pargfid, bname, inode,
                                     sources, sinks, healed_sinks, source,
                                     locked_on, replies, gfid_req, xdata);
    }
unlock:
    afr_selfheal_unentrylk(frame, this, parent, this->name, bname, locked_on,
                           NULL);
    if (inode)
        inode_unref(inode);

    if (replies)
        afr_replies_wipe(replies, priv->child_count);
    if (xattr)
        dict_unref(xattr);

    return ret;
}

int
afr_selfheal_name_unlocked_inspect(call_frame_t *frame, xlator_t *this,
                                   inode_t *parent, uuid_t pargfid,
                                   const char *bname, gf_boolean_t *need_heal)
{
    afr_private_t *priv = NULL;
    int i = 0;
    struct afr_reply *replies = NULL;
    inode_t *inode = NULL;
    int first_idx = -1;

    priv = this->private;

    replies = alloca0(sizeof(*replies) * priv->child_count);

    inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
                                            priv->child_up, NULL);
    if (!inode)
        return -ENOMEM;

    for (i = 0; i < priv->child_count; i++) {
        if (!replies[i].valid)
            continue;

        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
            *need_heal = _gf_true;
            break;
        }

        if (first_idx == -1) {
            first_idx = i;
            continue;
        }

        if (replies[i].op_ret != replies[first_idx].op_ret) {
            *need_heal = _gf_true;
            break;
        }

        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
                            replies[first_idx].poststat.ia_gfid)) {
            *need_heal = _gf_true;
            break;
        }
    }

    if (inode)
        inode_unref(inode);
    if (replies)
        afr_replies_wipe(replies, priv->child_count);
    return 0;
}

int
afr_selfheal_name(xlator_t *this, uuid_t pargfid, const char *bname,
                  void *gfid_req, dict_t *xdata)
{
    inode_t *parent = NULL;
    call_frame_t *frame = NULL;
    int ret = -1;
    gf_boolean_t need_heal = _gf_false;

    parent = afr_inode_find(this, pargfid);
    if (!parent)
        goto out;

    frame = afr_frame_create(this, NULL);
    if (!frame)
        goto out;

    ret = afr_selfheal_name_unlocked_inspect(frame, this, parent, pargfid,
                                             bname, &need_heal);
    if (ret)
        goto out;

    if (need_heal) {
        ret = afr_selfheal_name_do(frame, this, parent, pargfid, bname,
                                   gfid_req, xdata);
        if (ret)
            goto out;
    }

    ret = 0;
out:
    if (parent)
        inode_unref(parent);
    if (frame)
        AFR_STACK_DESTROY(frame);

    return ret;
}