diff options
Diffstat (limited to 'xlators/cluster/dht/src/dht-rebalance.c')
| -rw-r--r-- | xlators/cluster/dht/src/dht-rebalance.c | 8900 |
1 files changed, 4361 insertions, 4539 deletions
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c index 9983429acec..7a2539dc5f5 100644 --- a/xlators/cluster/dht/src/dht-rebalance.c +++ b/xlators/cluster/dht/src/dht-rebalance.c @@ -8,7 +8,6 @@ cases as published by the Free Software Foundation. */ - #include "tier.h" #include "dht-common.h" #include "xlator.h" @@ -18,244 +17,230 @@ #include <signal.h> #include "events.h" -#define GF_DISK_SECTOR_SIZE 512 -#define DHT_REBALANCE_PID 4242 /* Change it if required */ -#define DHT_REBALANCE_BLKSIZE (1024 * 1024) /* 1 MB */ -#define MAX_MIGRATE_QUEUE_COUNT 500 -#define MIN_MIGRATE_QUEUE_COUNT 200 -#define MAX_REBAL_TYPE_SIZE 16 -#define FILE_CNT_INTERVAL 600 /* 10 mins */ -#define ESTIMATE_START_INTERVAL 600 /* 10 mins */ -#define HARDLINK_MIG_INPROGRESS -2 -#define SKIP_MIGRATION_FD_POSITIVE -3 +#define GF_DISK_SECTOR_SIZE 512 +#define DHT_REBALANCE_PID 4242 /* Change it if required */ +#define DHT_REBALANCE_BLKSIZE (1024 * 1024) /* 1 MB */ +#define MAX_MIGRATE_QUEUE_COUNT 500 +#define MIN_MIGRATE_QUEUE_COUNT 200 +#define MAX_REBAL_TYPE_SIZE 16 +#define FILE_CNT_INTERVAL 600 /* 10 mins */ +#define ESTIMATE_START_INTERVAL 600 /* 10 mins */ +#define HARDLINK_MIG_INPROGRESS -2 +#define SKIP_MIGRATION_FD_POSITIVE -3 #ifndef MAX -#define MAX(a, b) (((a) > (b))?(a):(b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) #endif - -#define GF_CRAWL_INDEX_MOVE(idx, sv_cnt) { \ - idx++; \ - idx %= sv_cnt; \ - } +#define GF_CRAWL_INDEX_MOVE(idx, sv_cnt) \ + { \ + idx++; \ + idx %= sv_cnt; \ + } uint64_t g_totalfiles = 0; uint64_t g_totalsize = 0; - void -gf_defrag_free_dir_dfmeta (struct dir_dfmeta *meta, int local_subvols_cnt) +gf_defrag_free_dir_dfmeta(struct dir_dfmeta *meta, int local_subvols_cnt) { - int i = 0; + int i = 0; - if (meta) { - for (i = 0; i < local_subvols_cnt; i++) { - gf_dirent_free (&meta->equeue[i]); - } - - GF_FREE (meta->equeue); - GF_FREE (meta->head); - GF_FREE (meta->iterator); - GF_FREE (meta->offset_var); - GF_FREE (meta->fetch_entries); - GF_FREE (meta); + if (meta) { + for (i = 0; i < local_subvols_cnt; i++) { + gf_dirent_free(&meta->equeue[i]); } + + GF_FREE(meta->equeue); + GF_FREE(meta->head); + GF_FREE(meta->iterator); + GF_FREE(meta->offset_var); + GF_FREE(meta->fetch_entries); + GF_FREE(meta); + } } void -gf_defrag_free_container (struct dht_container *container) +gf_defrag_free_container(struct dht_container *container) { - if (container) { - gf_dirent_entry_free (container->df_entry); + if (container) { + gf_dirent_entry_free(container->df_entry); - if (container->parent_loc) { - loc_wipe (container->parent_loc); - } + if (container->parent_loc) { + loc_wipe(container->parent_loc); + } - GF_FREE (container->parent_loc); + GF_FREE(container->parent_loc); - GF_FREE (container); - } + GF_FREE(container); + } } void -dht_set_global_defrag_error (gf_defrag_info_t *defrag, int ret) +dht_set_global_defrag_error(gf_defrag_info_t *defrag, int ret) { - LOCK (&defrag->lock); - { - defrag->global_error = ret; - } - UNLOCK (&defrag->lock); - return; + LOCK(&defrag->lock); + { + defrag->global_error = ret; + } + UNLOCK(&defrag->lock); + return; } - static gf_boolean_t -dht_is_tier_command (int cmd) { - - gf_boolean_t is_tier = _gf_false; +dht_is_tier_command(int cmd) +{ + gf_boolean_t is_tier = _gf_false; - switch (cmd) { + switch (cmd) { case GF_DEFRAG_CMD_START_TIER: case GF_DEFRAG_CMD_STATUS_TIER: case GF_DEFRAG_CMD_START_DETACH_TIER: case GF_DEFRAG_CMD_STOP_DETACH_TIER: case GF_DEFRAG_CMD_PAUSE_TIER: case GF_DEFRAG_CMD_RESUME_TIER: - is_tier = _gf_true; - break; + is_tier = _gf_true; + break; default: - break; - } - return is_tier; - + break; + } + return is_tier; } - static int -dht_send_rebalance_event (xlator_t *this, int cmd, gf_defrag_status_t status) +dht_send_rebalance_event(xlator_t *this, int cmd, gf_defrag_status_t status) { - int ret = -1; - char *volname = NULL; - char *tmpstr = NULL; - char *ptr = NULL; - char *suffix = "-dht"; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; - int len = 0; - - eventtypes_t event = EVENT_LAST; - - switch (status) { + int ret = -1; + char *volname = NULL; + char *tmpstr = NULL; + char *ptr = NULL; + char *suffix = "-dht"; + dht_conf_t *conf = NULL; + gf_defrag_info_t *defrag = NULL; + int len = 0; + + eventtypes_t event = EVENT_LAST; + + switch (status) { case GF_DEFRAG_STATUS_COMPLETE: - event = EVENT_VOLUME_REBALANCE_COMPLETE; - break; + event = EVENT_VOLUME_REBALANCE_COMPLETE; + break; case GF_DEFRAG_STATUS_FAILED: - event = EVENT_VOLUME_REBALANCE_FAILED; - break; + event = EVENT_VOLUME_REBALANCE_FAILED; + break; case GF_DEFRAG_STATUS_STOPPED: - event = EVENT_VOLUME_REBALANCE_STOP; - break; + event = EVENT_VOLUME_REBALANCE_STOP; + break; default: - break; - - } - - if (dht_is_tier_command (cmd)) { - /* We should have the tier volume name*/ - conf = this->private; - defrag = conf->defrag; - volname = defrag->tier_conf.volname; - } else { - /* DHT volume */ - len = strlen (this->name) - strlen (suffix); - tmpstr = gf_strdup (this->name); - if (tmpstr) { - ptr = tmpstr + len; - if (!strcmp (ptr, suffix)) { - tmpstr[len] = '\0'; - volname = tmpstr; - } - } - } + break; + } - if (!volname) { - /* Better than nothing */ - volname = this->name; - } - - if (event != EVENT_LAST) { - gf_event (event, "volume=%s", volname); - } - - GF_FREE (tmpstr); - return ret; + if (dht_is_tier_command(cmd)) { + /* We should have the tier volume name*/ + conf = this->private; + defrag = conf->defrag; + volname = defrag->tier_conf.volname; + } else { + /* DHT volume */ + len = strlen(this->name) - strlen(suffix); + tmpstr = gf_strdup(this->name); + if (tmpstr) { + ptr = tmpstr + len; + if (!strcmp(ptr, suffix)) { + tmpstr[len] = '\0'; + volname = tmpstr; + } + } + } + + if (!volname) { + /* Better than nothing */ + volname = this->name; + } + + if (event != EVENT_LAST) { + gf_event(event, "volume=%s", volname); + } + + GF_FREE(tmpstr); + return ret; } - static void -dht_strip_out_acls (dict_t *dict) +dht_strip_out_acls(dict_t *dict) { - if (dict) { - dict_del (dict, "trusted.SGI_ACL_FILE"); - dict_del (dict, POSIX_ACL_ACCESS_XATTR); - } + if (dict) { + dict_del(dict, "trusted.SGI_ACL_FILE"); + dict_del(dict, POSIX_ACL_ACCESS_XATTR); + } } - - static int -dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count, - int32_t size, off_t offset, struct iobref *iobref, - int *fop_errno) +dht_write_with_holes(xlator_t *to, fd_t *fd, struct iovec *vec, int count, + int32_t size, off_t offset, struct iobref *iobref, + int *fop_errno) { - int i = 0; - int ret = -1; - int start_idx = 0; - int tmp_offset = 0; - int write_needed = 0; - int buf_len = 0; - int size_pending = 0; - char *buf = NULL; - - /* loop through each vector */ - for (i = 0; i < count; i++) { - buf = vec[i].iov_base; - buf_len = vec[i].iov_len; - - for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len; - start_idx += GF_DISK_SECTOR_SIZE) { - - if (mem_0filled (buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) { - write_needed = 1; - continue; - } - - if (write_needed) { - ret = syncop_write (to, fd, (buf + tmp_offset), - (start_idx - tmp_offset), - (offset + tmp_offset), - iobref, 0, NULL, NULL); - /* 'path' will be logged in calling function */ - if (ret < 0) { - gf_log (THIS->name, GF_LOG_WARNING, - "failed to write (%s)", - strerror (-ret)); - *fop_errno = -ret; - ret = -1; - goto out; - } - - write_needed = 0; - } - tmp_offset = start_idx + GF_DISK_SECTOR_SIZE; + int i = 0; + int ret = -1; + int start_idx = 0; + int tmp_offset = 0; + int write_needed = 0; + int buf_len = 0; + int size_pending = 0; + char *buf = NULL; + + /* loop through each vector */ + for (i = 0; i < count; i++) { + buf = vec[i].iov_base; + buf_len = vec[i].iov_len; + + for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len; + start_idx += GF_DISK_SECTOR_SIZE) { + if (mem_0filled(buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) { + write_needed = 1; + continue; + } + + if (write_needed) { + ret = syncop_write( + to, fd, (buf + tmp_offset), (start_idx - tmp_offset), + (offset + tmp_offset), iobref, 0, NULL, NULL); + /* 'path' will be logged in calling function */ + if (ret < 0) { + gf_log(THIS->name, GF_LOG_WARNING, "failed to write (%s)", + strerror(-ret)); + *fop_errno = -ret; + ret = -1; + goto out; } - if ((start_idx < buf_len) || write_needed) { - /* This means, last chunk is not yet written.. write it */ - ret = syncop_write (to, fd, (buf + tmp_offset), - (buf_len - tmp_offset), - (offset + tmp_offset), iobref, 0, - NULL, NULL); - if (ret < 0) { - /* 'path' will be logged in calling function */ - gf_log (THIS->name, GF_LOG_WARNING, - "failed to write (%s)", - strerror (-ret)); - *fop_errno = -ret; - ret = -1; - goto out; - } - } + write_needed = 0; + } + tmp_offset = start_idx + GF_DISK_SECTOR_SIZE; + } - size_pending = (size - buf_len); - if (!size_pending) - break; + if ((start_idx < buf_len) || write_needed) { + /* This means, last chunk is not yet written.. write it */ + ret = syncop_write(to, fd, (buf + tmp_offset), + (buf_len - tmp_offset), (offset + tmp_offset), + iobref, 0, NULL, NULL); + if (ret < 0) { + /* 'path' will be logged in calling function */ + gf_log(THIS->name, GF_LOG_WARNING, "failed to write (%s)", + strerror(-ret)); + *fop_errno = -ret; + ret = -1; + goto out; + } } - ret = size; -out: - return ret; + size_pending = (size - buf_len); + if (!size_pending) + break; + } + ret = size; +out: + return ret; } /* @@ -294,323 +279,318 @@ be converted to "0" in dht_migrate_file. */ int32_t -gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, int *fop_errno) +gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno) { - int32_t ret = -1; - xlator_t *cached_subvol = NULL; - xlator_t *hashed_subvol = NULL; - xlator_t *linkto_subvol = NULL; - data_t *data = NULL; - struct iatt iatt = {0,}; - int32_t op_errno = 0; - dht_conf_t *conf = NULL; - gf_loglevel_t loglevel = 0; - dict_t *link_xattr = NULL; - dict_t *dict = NULL; - dict_t *xattr_rsp = NULL; - struct iatt stbuf = {0,}; - + int32_t ret = -1; + xlator_t *cached_subvol = NULL; + xlator_t *hashed_subvol = NULL; + xlator_t *linkto_subvol = NULL; + data_t *data = NULL; + struct iatt iatt = { + 0, + }; + int32_t op_errno = 0; + dht_conf_t *conf = NULL; + gf_loglevel_t loglevel = 0; + dict_t *link_xattr = NULL; + dict_t *dict = NULL; + dict_t *xattr_rsp = NULL; + struct iatt stbuf = { + 0, + }; + + *fop_errno = EINVAL; + + GF_VALIDATE_OR_GOTO("defrag", loc, out); + GF_VALIDATE_OR_GOTO("defrag", loc->name, out); + GF_VALIDATE_OR_GOTO("defrag", this, out); + GF_VALIDATE_OR_GOTO("defrag", this->private, out); + + conf = this->private; + + if (gf_uuid_is_null(loc->pargfid)) { + gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed :" + "loc->pargfid is NULL for %s", + loc->path); *fop_errno = EINVAL; + ret = -1; + goto out; + } + + if (gf_uuid_is_null(loc->gfid)) { + gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed :" + "loc->gfid is NULL for %s", + loc->path); + *fop_errno = EINVAL; + ret = -1; + goto out; + } - GF_VALIDATE_OR_GOTO ("defrag", loc, out); - GF_VALIDATE_OR_GOTO ("defrag", loc->name, out); - GF_VALIDATE_OR_GOTO ("defrag", this, out); - GF_VALIDATE_OR_GOTO ("defrag", this->private, out); - - conf = this->private; - - if (gf_uuid_is_null (loc->pargfid)) { - gf_msg ("", GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed :" - "loc->pargfid is NULL for %s", loc->path); - *fop_errno = EINVAL; - ret = -1; - goto out; - } - - if (gf_uuid_is_null (loc->gfid)) { - gf_msg ("", GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed :" - "loc->gfid is NULL for %s", loc->path); - *fop_errno = EINVAL; - ret = -1; - goto out; - } - - link_xattr = dict_new (); - if (!link_xattr) { - ret = -1; - *fop_errno = ENOMEM; - goto out; - } - - /* - Parallel migration can lead to migration of the hard link multiple - times which can lead to data loss. Hence, adding a fresh lookup to - decide whether migration is required or not. - - Elaborating the scenario for let say 10 hardlinks [link{1..10}]: - Let say the first hard link "link1" does the setxattr of the - new hashed subvolume info on the cached file. As there are multiple - threads working, we might have already all the links created on the - new hashed by the time we reach hardlink let say link5. Now the - number of links on hashed is equal to that of cached. Hence, file - migration will happen for link6. - - Cached Hashed - --------T link6 rwxrwxrwx link6 - - Now post above state all the link file on the cached will be zero - byte linkto files. Hence, if we still do migration for the following - files link{7..10}, we will end up migrating 0 data leading to data - loss. - Hence, a lookup can make sure whether we need to migrate the - file or not. - */ - - dict = dict_new (); - if (!dict) { - ret = -1; - *fop_errno = ENOMEM; - gf_msg (this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY, - "could not allocate memory for dict"); - goto out; - } - - ret = dict_set_int32 (dict, conf->link_xattr_name, 256); + link_xattr = dict_new(); + if (!link_xattr) { + ret = -1; + *fop_errno = ENOMEM; + goto out; + } + + /* + Parallel migration can lead to migration of the hard link multiple + times which can lead to data loss. Hence, adding a fresh lookup to + decide whether migration is required or not. + + Elaborating the scenario for let say 10 hardlinks [link{1..10}]: + Let say the first hard link "link1" does the setxattr of the + new hashed subvolume info on the cached file. As there are multiple + threads working, we might have already all the links created on the + new hashed by the time we reach hardlink let say link5. Now the + number of links on hashed is equal to that of cached. Hence, file + migration will happen for link6. + + Cached Hashed + --------T link6 rwxrwxrwx link6 + + Now post above state all the link file on the cached will be zero + byte linkto files. Hence, if we still do migration for the following + files link{7..10}, we will end up migrating 0 data leading to data + loss. + Hence, a lookup can make sure whether we need to migrate the + file or not. + */ + + dict = dict_new(); + if (!dict) { + ret = -1; + *fop_errno = ENOMEM; + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY, + "could not allocate memory for dict"); + goto out; + } + + ret = dict_set_int32(dict, conf->link_xattr_name, 256); + if (ret) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: failed to set 'linkto' key in dict", + loc->path); + goto out; + } + + ret = syncop_lookup(this, loc, &stbuf, NULL, dict, &xattr_rsp); + if (ret) { + /*Ignore ENOENT and ESTALE as file might have been + migrated already*/ + if (-ret == ENOENT || -ret == ESTALE) { + ret = -2; + goto out; + } + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:%s lookup failed with ret = %d", loc->path, + ret); + *fop_errno = -ret; + ret = -1; + goto out; + } + + cached_subvol = dht_subvol_get_cached(this, loc->inode); + if (!cached_subvol) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed :" + "Failed to get cached subvol" + " for %s on %s", + loc->name, this->name); + *fop_errno = EINVAL; + ret = -1; + goto out; + } + + hashed_subvol = dht_subvol_get_hashed(this, loc); + if (!hashed_subvol) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed :" + "Failed to get hashed subvol" + " for %s on %s", + loc->name, this->name); + *fop_errno = EINVAL; + ret = -1; + goto out; + } + + /* Hardlink migration happens only with remove-brick. So this condition will + * be true only when the migration has happened. In case hardlinks are + * migrated for rebalance case, remove this check. Having this check here + * avoid redundant calls below*/ + if (hashed_subvol == cached_subvol) { + ret = -2; + goto out; + } + + gf_log(this->name, GF_LOG_INFO, + "Attempting to migrate hardlink %s " + "with gfid %s from %s -> %s", + loc->name, uuid_utoa(loc->gfid), cached_subvol->name, + hashed_subvol->name); + + data = dict_get(xattr_rsp, conf->link_xattr_name); + /* set linkto on cached -> hashed if not present, else link it */ + if (!data) { + ret = dict_set_str(link_xattr, conf->link_xattr_name, + hashed_subvol->name); if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: failed to set 'linkto' key in dict", loc->path); - goto out; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed :" + "Failed to set dictionary value:" + " key = %s for %s", + conf->link_xattr_name, loc->name); + *fop_errno = ENOMEM; + ret = -1; + goto out; } - ret = syncop_lookup (this, loc, &stbuf, NULL, dict, &xattr_rsp); + ret = syncop_setxattr(cached_subvol, loc, link_xattr, 0, NULL, NULL); if (ret) { - /*Ignore ENOENT and ESTALE as file might have been - migrated already*/ - if (-ret == ENOENT || -ret == ESTALE) { - ret = -2; - goto out; - } - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:%s lookup failed with ret = %d", - loc->path, ret); - *fop_errno = -ret; - ret = -1; - goto out; - } - - cached_subvol = dht_subvol_get_cached (this, loc->inode); - if (!cached_subvol) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed :" - "Failed to get cached subvol" - " for %s on %s", loc->name, this->name); - *fop_errno = EINVAL; - ret = -1; - goto out; + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed :" + "Linkto setxattr failed %s -> %s", + cached_subvol->name, loc->name); + *fop_errno = -ret; + ret = -1; + goto out; } - hashed_subvol = dht_subvol_get_hashed (this, loc); - if (!hashed_subvol) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed :" - "Failed to get hashed subvol" - " for %s on %s", loc->name, this->name); - *fop_errno = EINVAL; - ret = -1; - goto out; - } - - /* Hardlink migration happens only with remove-brick. So this condition will - * be true only when the migration has happened. In case hardlinks are migrated - * for rebalance case, remove this check. Having this check here avoid redundant - * calls below*/ - if (hashed_subvol == cached_subvol) { - ret = -2; - goto out; - } - - gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s " - "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid), - cached_subvol->name, hashed_subvol->name); - - data = dict_get (xattr_rsp, conf->link_xattr_name); - /* set linkto on cached -> hashed if not present, else link it */ - if (!data) { - ret = dict_set_str (link_xattr, conf->link_xattr_name, - hashed_subvol->name); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed :" - "Failed to set dictionary value:" - " key = %s for %s", - conf->link_xattr_name, loc->name); - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - - ret = syncop_setxattr (cached_subvol, loc, link_xattr, 0, NULL, - NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed :" - "Linkto setxattr failed %s -> %s", - cached_subvol->name, - loc->name); - *fop_errno = -ret; - ret = -1; - goto out; - } + gf_msg_debug(this->name, 0, + "hardlink target subvol created on %s " + ",cached %s, file %s", + hashed_subvol->name, cached_subvol->name, loc->path); - gf_msg_debug (this->name, 0, "hardlink target subvol created on %s " - ",cached %s, file %s", - hashed_subvol->name, cached_subvol->name, loc->path); - - ret = -2; - goto out; + ret = -2; + goto out; + } else { + linkto_subvol = dht_linkfile_subvol(this, NULL, NULL, xattr_rsp); + if (!linkto_subvol) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_ERROR, + "Failed to get " + "linkto subvol for %s", + loc->name); } else { - linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattr_rsp); - if (!linkto_subvol) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_SUBVOL_ERROR, - "Failed to get " - "linkto subvol for %s", loc->name); - } else { - hashed_subvol = linkto_subvol; - } - - ret = syncop_link (hashed_subvol, loc, loc, &iatt, NULL, NULL); - if (ret) { - op_errno = -ret; - ret = -1; - - loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : \ - GF_LOG_ERROR; - gf_msg (this->name, loglevel, op_errno, - DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED, - "link of %s -> %s" - " failed on subvol %s", loc->name, - uuid_utoa(loc->gfid), - hashed_subvol->name); - if (op_errno != EEXIST) { - *fop_errno = op_errno; - goto out; - } - } else { - gf_msg_debug (this->name, 0, "syncop_link successful for" - " hardlink %s on subvol %s, cached %s", loc->path, - hashed_subvol->name, cached_subvol->name); - - } + hashed_subvol = linkto_subvol; } - ret = syncop_lookup (hashed_subvol, loc, &iatt, NULL, NULL, NULL); + ret = syncop_link(hashed_subvol, loc, loc, &iatt, NULL, NULL); if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed :Failed lookup %s on %s ", - loc->name, hashed_subvol->name); - - *fop_errno = -ret; - ret = -1; - goto out; + op_errno = -ret; + ret = -1; + + loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_ERROR; + gf_msg(this->name, loglevel, op_errno, + DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED, + "link of %s -> %s" + " failed on subvol %s", + loc->name, uuid_utoa(loc->gfid), hashed_subvol->name); + if (op_errno != EEXIST) { + *fop_errno = op_errno; + goto out; + } + } else { + gf_msg_debug(this->name, 0, + "syncop_link successful for" + " hardlink %s on subvol %s, cached %s", + loc->path, hashed_subvol->name, cached_subvol->name); } + } - /* There is a race where on the target subvol for the hardlink - * (note: hash subvol for the hardlink might differ from this), some - * other client(non-rebalance) would have created a linkto file for that - * hardlink as part of lookup. So let say there are 10 hardlinks, on the - * 5th hardlink it self the hardlinks might have migrated. Now for - * (6..10th) hardlinks the cached and target would be same as the file - * has already migrated. Hence this check is needed */ - if (cached_subvol == hashed_subvol) { - gf_msg_debug (this->name, 0, "source %s and destination %s " - "for hardlink %s are same", cached_subvol->name, - hashed_subvol->name, loc->path); - ret = -2; - goto out; - } + ret = syncop_lookup(hashed_subvol, loc, &iatt, NULL, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed :Failed lookup %s on %s ", loc->name, + hashed_subvol->name); - if (iatt.ia_nlink == stbuf.ia_nlink) { - ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol, - GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS, - fop_errno); - if (ret) { - goto out; - } - } + *fop_errno = -ret; + ret = -1; + goto out; + } + + /* There is a race where on the target subvol for the hardlink + * (note: hash subvol for the hardlink might differ from this), some + * other client(non-rebalance) would have created a linkto file for that + * hardlink as part of lookup. So let say there are 10 hardlinks, on the + * 5th hardlink it self the hardlinks might have migrated. Now for + * (6..10th) hardlinks the cached and target would be same as the file + * has already migrated. Hence this check is needed */ + if (cached_subvol == hashed_subvol) { + gf_msg_debug(this->name, 0, + "source %s and destination %s " + "for hardlink %s are same", + cached_subvol->name, hashed_subvol->name, loc->path); ret = -2; + goto out; + } + + if (iatt.ia_nlink == stbuf.ia_nlink) { + ret = dht_migrate_file(this, loc, cached_subvol, hashed_subvol, + GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS, fop_errno); + if (ret) { + goto out; + } + } + ret = -2; out: - if (link_xattr) - dict_unref (link_xattr); + if (link_xattr) + dict_unref(link_xattr); - if (xattr_rsp) - dict_unref (xattr_rsp); + if (xattr_rsp) + dict_unref(xattr_rsp); - if (dict) - dict_unref (dict); + if (dict) + dict_unref(dict); - return ret; + return ret; } - - static int -__check_file_has_hardlink (xlator_t *this, loc_t *loc, - struct iatt *stbuf, dict_t *xattrs, int flags, - gf_defrag_info_t *defrag, dht_conf_t *conf, int *fop_errno) +__check_file_has_hardlink(xlator_t *this, loc_t *loc, struct iatt *stbuf, + dict_t *xattrs, int flags, gf_defrag_info_t *defrag, + dht_conf_t *conf, int *fop_errno) { - int ret = 0; + int ret = 0; - if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) { - ret = 0; - return ret; - } - if (stbuf->ia_nlink > 1) { - /* support for decomission */ - if (flags == GF_DHT_MIGRATE_HARDLINK) { - synclock_lock (&conf->link_lock); - ret = gf_defrag_handle_hardlink - (this, loc, fop_errno); - synclock_unlock (&conf->link_lock); - /* - Returning zero will force the file to be remigrated. - Checkout gf_defrag_handle_hardlink for more information. - */ - if (ret && ret != -2) { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: failed to migrate file with link", - loc->path); - } - } else { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migration skipped for:" - "%s: file has hardlinks", loc->path); - *fop_errno = ENOTSUP; - ret = 1; - } - } + if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) { + ret = 0; + return ret; + } + if (stbuf->ia_nlink > 1) { + /* support for decomission */ + if (flags == GF_DHT_MIGRATE_HARDLINK) { + synclock_lock(&conf->link_lock); + ret = gf_defrag_handle_hardlink(this, loc, fop_errno); + synclock_unlock(&conf->link_lock); + /* + Returning zero will force the file to be remigrated. + Checkout gf_defrag_handle_hardlink for more information. + */ + if (ret && ret != -2) { + gf_msg(this->name, GF_LOG_WARNING, 0, + DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: failed to migrate file with link", + loc->path); + } + } else { + gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migration skipped for:" + "%s: file has hardlinks", + loc->path); + *fop_errno = ENOTSUP; + ret = 1; + } + } - return ret; + return ret; } - /* return values 0 : File will be migrated @@ -620,964 +600,911 @@ __check_file_has_hardlink (xlator_t *this, loc_t *loc, -1 : failure */ static int -__is_file_migratable (xlator_t *this, loc_t *loc, - struct iatt *stbuf, dict_t *xattrs, int flags, - gf_defrag_info_t *defrag, dht_conf_t *conf, - int *fop_errno) +__is_file_migratable(xlator_t *this, loc_t *loc, struct iatt *stbuf, + dict_t *xattrs, int flags, gf_defrag_info_t *defrag, + dht_conf_t *conf, int *fop_errno) { - int ret = -1; - int lock_count = 0; - - if (IA_ISDIR (stbuf->ia_type)) { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: migrate-file called on directory", loc->path); - *fop_errno = EISDIR; - ret = -1; - goto out; - } - - if (!conf->lock_migration_enabled) { - ret = dict_get_int32 (xattrs, GLUSTERFS_POSIXLK_COUNT, - &lock_count); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: Unable to get lock count for file", - loc->path); - *fop_errno = EINVAL; - ret = -1; - goto out; - } - - if (lock_count) { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed: %s: File has locks." - " Skipping file migration", loc->path); - *fop_errno = ENOTSUP; - ret = 1; - goto out; - } - } + int ret = -1; + int lock_count = 0; + + if (IA_ISDIR(stbuf->ia_type)) { + gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: migrate-file called on directory", + loc->path); + *fop_errno = EISDIR; + ret = -1; + goto out; + } - /* Check if file has hardlink*/ - ret = __check_file_has_hardlink (this, loc, stbuf, xattrs, - flags, defrag, conf, fop_errno); + if (!conf->lock_migration_enabled) { + ret = dict_get_int32(xattrs, GLUSTERFS_POSIXLK_COUNT, &lock_count); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: Unable to get lock count for file", + loc->path); + *fop_errno = EINVAL; + ret = -1; + goto out; + } + + if (lock_count) { + gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed: %s: File has locks." + " Skipping file migration", + loc->path); + *fop_errno = ENOTSUP; + ret = 1; + goto out; + } + } + + /* Check if file has hardlink*/ + ret = __check_file_has_hardlink(this, loc, stbuf, xattrs, flags, defrag, + conf, fop_errno); out: - return ret; + return ret; } - static int -__dht_rebalance_create_dst_file (xlator_t *this, xlator_t *to, xlator_t *from, - loc_t *loc, struct iatt *stbuf, fd_t **dst_fd, - int *fop_errno) +__dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from, + loc_t *loc, struct iatt *stbuf, fd_t **dst_fd, + int *fop_errno) { - int ret = -1; - int ret2 = -1; - fd_t *fd = NULL; - struct iatt new_stbuf = {0,}; - struct iatt check_stbuf= {0,}; - dht_conf_t *conf = NULL; - dict_t *dict = NULL; - dict_t *xdata = NULL; - - conf = this->private; + int ret = -1; + int ret2 = -1; + fd_t *fd = NULL; + struct iatt new_stbuf = { + 0, + }; + struct iatt check_stbuf = { + 0, + }; + dht_conf_t *conf = NULL; + dict_t *dict = NULL; + dict_t *xdata = NULL; + + conf = this->private; + + dict = dict_new(); + if (!dict) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY, + "dictionary allocation failed for" + "path:%s", + loc->path); + goto out; + } + ret = dict_set_gfuuid(dict, "gfid-req", stbuf->ia_gfid, true); + if (ret) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, + "%s: failed to set dictionary value: key = gfid-req", loc->path); + goto out; + } + + ret = dict_set_str(dict, conf->link_xattr_name, from->name); + if (ret) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, + "%s: failed to set dictionary value: key = %s ", loc->path, + conf->link_xattr_name); + goto out; + } + + fd = fd_create(loc->inode, DHT_REBALANCE_PID); + if (!fd) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: fd create failed (destination)", loc->path); + goto out; + } - dict = dict_new (); - if (!dict) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, ENOMEM, - DHT_MSG_NO_MEMORY, "dictionary allocation failed for" - "path:%s", loc->path); - goto out; - } - ret = dict_set_gfuuid (dict, "gfid-req", stbuf->ia_gfid, true); - if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_DICT_SET_FAILED, - "%s: failed to set dictionary value: key = gfid-req", - loc->path); - goto out; + if (!!dht_is_tier_xlator(this)) { + xdata = dict_new(); + if (!xdata) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, + DHT_MSG_MIGRATE_FILE_FAILED, "%s: dict_new failed)", + loc->path); + goto out; } - ret = dict_set_str (dict, conf->link_xattr_name, from->name); + ret = dict_set_int32(xdata, GF_CLEAN_WRITE_PROTECTION, 1); if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_DICT_SET_FAILED, - "%s: failed to set dictionary value: key = %s ", - loc->path, conf->link_xattr_name); - goto out; - } - - fd = fd_create (loc->inode, DHT_REBALANCE_PID); - if (!fd) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, ENOMEM, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: fd create failed (destination)", - loc->path); - goto out; - } - - if (!!dht_is_tier_xlator (this)) { - xdata = dict_new (); - if (!xdata) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, ENOMEM, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: dict_new failed)", - loc->path); - goto out; - } - - ret = dict_set_int32 (xdata, GF_CLEAN_WRITE_PROTECTION, 1); - if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_DICT_SET_FAILED, - "%s: failed to set dictionary value: key = %s ", - loc->path, GF_CLEAN_WRITE_PROTECTION); - goto out; - } - } - - ret = syncop_lookup (to, loc, &new_stbuf, NULL, xdata, NULL); - if (!ret) { - /* File exits in the destination, check if gfid matches */ - if (gf_uuid_compare (stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_GFID_MISMATCH, - "file %s exists in %s with different gfid", - loc->path, to->name); - *fop_errno = EINVAL; - ret = -1; - goto out; - } - } - if ((ret < 0) && (-ret != ENOENT)) { - /* File exists in destination, but not accessible */ - gf_msg (THIS->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to lookup file", - loc->path); - *fop_errno = -ret; - ret = -1; - goto out; - } - - /* Create the destination with LINKFILE mode, and linkto xattr, - if the linkfile already exists, just open the file */ - if (!ret) { - /* - * File already present, just open the file. - */ - ret = syncop_open (to, loc, O_RDWR, fd, NULL, NULL); - if (ret < 0) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to open %s on %s", - loc->path, to->name); - *fop_errno = -ret; - ret = -1; - goto out; - } - } else { - ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd, - &new_stbuf, dict, NULL); - if (ret < 0) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to create %s on %s", - loc->path, to->name); - *fop_errno = -ret; - ret = -1; - goto out; - } - - } - - fd_bind (fd); + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, + "%s: failed to set dictionary value: key = %s ", loc->path, + GF_CLEAN_WRITE_PROTECTION); + goto out; + } + } + + ret = syncop_lookup(to, loc, &new_stbuf, NULL, xdata, NULL); + if (!ret) { + /* File exits in the destination, check if gfid matches */ + if (gf_uuid_compare(stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, + "file %s exists in %s with different gfid", loc->path, + to->name); + *fop_errno = EINVAL; + ret = -1; + goto out; + } + } + if ((ret < 0) && (-ret != ENOENT)) { + /* File exists in destination, but not accessible */ + gf_msg(THIS->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to lookup file", loc->path); + *fop_errno = -ret; + ret = -1; + goto out; + } - /*Reason of doing lookup after create again: - *In the create, there is some time-gap between opening fd at the - *server (posix_layer) and binding it in server (incrementing fd count), - *so if in that time-gap, if other process sends unlink considering it - *as a linkto file, because inode->fd count will be 0, so file will be - *unlinked at the backend. And because further operations are performed - *on fd, so though migration will be done but will end with no file - *at the backend. + /* Create the destination with LINKFILE mode, and linkto xattr, + if the linkfile already exists, just open the file */ + if (!ret) { + /* + * File already present, just open the file. */ - - ret = syncop_lookup (to, loc, &check_stbuf, NULL, NULL, NULL); - if (!ret) { - - if (gf_uuid_compare (stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_GFID_MISMATCH, - "file %s exists in %s with different gfid," - "found in lookup after create", - loc->path, to->name); - *fop_errno = EINVAL; - ret = -1; - goto out; + ret = syncop_open(to, loc, O_RDWR, fd, NULL, NULL); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "failed to open %s on %s", loc->path, to->name); + *fop_errno = -ret; + ret = -1; + goto out; + } + } else { + ret = syncop_create(to, loc, O_RDWR, DHT_LINKFILE_MODE, fd, &new_stbuf, + dict, NULL); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "failed to create %s on %s", loc->path, to->name); + *fop_errno = -ret; + ret = -1; + goto out; + } + } + + fd_bind(fd); + + /*Reason of doing lookup after create again: + *In the create, there is some time-gap between opening fd at the + *server (posix_layer) and binding it in server (incrementing fd count), + *so if in that time-gap, if other process sends unlink considering it + *as a linkto file, because inode->fd count will be 0, so file will be + *unlinked at the backend. And because further operations are performed + *on fd, so though migration will be done but will end with no file + *at the backend. + */ + + ret = syncop_lookup(to, loc, &check_stbuf, NULL, NULL, NULL); + if (!ret) { + if (gf_uuid_compare(stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, + "file %s exists in %s with different gfid," + "found in lookup after create", + loc->path, to->name); + *fop_errno = EINVAL; + ret = -1; + goto out; + } + } + + if (-ret == ENOENT) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: file does not exist" + "on %s", + loc->path, to->name); + *fop_errno = -ret; + ret = -1; + goto out; + } + + ret = syncop_fsetattr(to, fd, stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), + NULL, NULL, NULL, NULL); + if (ret < 0) { + *fop_errno = -ret; + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "chown failed for %s on %s", loc->path, to->name); + } + + /* No need to bother about 0 byte size files */ + if (stbuf->ia_size > 0) { + if (conf->use_fallocate) { + ret = syncop_fallocate(to, fd, 0, 0, stbuf->ia_size, NULL, NULL); + if (ret < 0) { + if (ret == -EOPNOTSUPP || ret == -EINVAL || ret == -ENOSYS) { + conf->use_fallocate = _gf_false; + } else { + gf_msg(this->name, GF_LOG_ERROR, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "fallocate failed for %s on %s", loc->path, + to->name); + + *fop_errno = -ret; + + /* fallocate does not release the space + * in some cases + */ + ret2 = syncop_ftruncate(to, fd, 0, NULL, NULL, NULL, NULL); + if (ret2 < 0) { + gf_msg(this->name, GF_LOG_WARNING, -ret2, + DHT_MSG_MIGRATE_FILE_FAILED, + "ftruncate failed for " + "%s on %s", + loc->path, to->name); + } + goto out; } - + } } - if (-ret == ENOENT) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, "%s: file does not exist" - "on %s", loc->path, to->name); + if (!conf->use_fallocate) { + ret = syncop_ftruncate(to, fd, stbuf->ia_size, NULL, NULL, NULL, + NULL); + if (ret < 0) { *fop_errno = -ret; - ret = -1; - goto out; + gf_msg(this->name, GF_LOG_WARNING, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "ftruncate failed for %s on %s", loc->path, to->name); + } } + } - ret = syncop_fsetattr (to, fd, stbuf, - (GF_SET_ATTR_UID | GF_SET_ATTR_GID), - NULL, NULL, NULL, NULL); - if (ret < 0) { - *fop_errno = -ret; - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "chown failed for %s on %s", - loc->path, to->name); - } + /* success */ + ret = 0; - /* No need to bother about 0 byte size files */ - if (stbuf->ia_size > 0) { - if (conf->use_fallocate) { - ret = syncop_fallocate (to, fd, 0, 0, stbuf->ia_size, - NULL, NULL); - if (ret < 0) { - if (ret == -EOPNOTSUPP || ret == -EINVAL || - ret == -ENOSYS) { - conf->use_fallocate = _gf_false; - } else { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "fallocate failed for %s on %s", - loc->path, to->name); - - *fop_errno = -ret; - - /* fallocate does not release the space - * in some cases - */ - ret2 = syncop_ftruncate (to, fd, 0, - NULL, NULL, - NULL, NULL); - if (ret2 < 0) { - gf_msg (this->name, - GF_LOG_WARNING, -ret2, - DHT_MSG_MIGRATE_FILE_FAILED, - "ftruncate failed for " - "%s on %s", - loc->path, to->name); - } - goto out; - } - } - } - - if (!conf->use_fallocate) { - ret = syncop_ftruncate (to, fd, stbuf->ia_size, NULL, - NULL, NULL, NULL); - if (ret < 0) { - *fop_errno = -ret; - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "ftruncate failed for %s on %s", - loc->path, to->name); - } - } - } - - /* success */ - ret = 0; - - if (dst_fd) - *dst_fd = fd; + if (dst_fd) + *dst_fd = fd; out: - if (ret) { - if (fd) { - fd_unref (fd); - } + if (ret) { + if (fd) { + fd_unref(fd); } - if (dict) - dict_unref (dict); - - if (xdata) - dict_unref (dict); + } + if (dict) + dict_unref(dict); + if (xdata) + dict_unref(dict); - return ret; + return ret; } static int -__dht_check_free_space (xlator_t *this, xlator_t *to, xlator_t *from, - loc_t *loc, struct iatt *stbuf, int flag, - dht_conf_t *conf, gf_boolean_t *target_changed, - xlator_t **new_subvol, int *fop_errno) +__dht_check_free_space(xlator_t *this, xlator_t *to, xlator_t *from, loc_t *loc, + struct iatt *stbuf, int flag, dht_conf_t *conf, + gf_boolean_t *target_changed, xlator_t **new_subvol, + int *fop_errno) { - struct statvfs src_statfs = {0,}; - struct statvfs dst_statfs = {0,}; - int ret = -1; - dict_t *xdata = NULL; - dht_layout_t *layout = NULL; - uint64_t src_statfs_blocks = 1; - uint64_t dst_statfs_blocks = 1; - double dst_post_availspacepercent = 0; - double src_post_availspacepercent = 0; - uint64_t file_blocks = 0; - uint64_t src_total_blocks = 0; - uint64_t dst_total_blocks = 0; - - xdata = dict_new (); - if (!xdata) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, ENOMEM, - DHT_MSG_NO_MEMORY, - "failed to allocate dictionary"); - goto out; - } - - ret = dict_set_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "Failed to set " - GF_INTERNAL_IGNORE_DEEM_STATFS" in dict"); - ret = -1; - *fop_errno = ENOMEM; - goto out; - } - - ret = syncop_statfs (from, loc, &src_statfs, xdata, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to get statfs of %s on %s", - loc->path, from->name); - *fop_errno = -ret; - ret = -1; - goto out; - } - - ret = syncop_statfs (to, loc, &dst_statfs, xdata, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to get statfs of %s on %s", - loc->path, to->name); - *fop_errno = -ret; - ret = -1; - goto out; - } - - gf_msg_debug (this->name, 0, "min_free_disk - %f , block available - " - "%lu , block size - %lu ", conf->min_free_disk, - dst_statfs.f_bavail, dst_statfs.f_bsize); - - dst_statfs_blocks = dst_statfs.f_bavail * - (dst_statfs.f_frsize / - GF_DISK_SECTOR_SIZE); - - src_statfs_blocks = src_statfs.f_bavail * - (src_statfs.f_frsize / - GF_DISK_SECTOR_SIZE); - - dst_total_blocks = dst_statfs.f_blocks * - (dst_statfs.f_frsize / - GF_DISK_SECTOR_SIZE); - - src_total_blocks = src_statfs.f_blocks * - (src_statfs.f_frsize / - GF_DISK_SECTOR_SIZE); - - /* if force option is given, do not check for space @ dst. - * Check only if space is avail for the file */ - if (flag != GF_DHT_MIGRATE_DATA) - goto check_avail_space; - - /* Check: - During rebalance `migrate-data` - Destination subvol experiences - a `reduction` in 'blocks' of free space, at the same time source - subvol gains certain 'blocks' of free space. A valid check is - necessary here to avoid erroneous move to destination where - the space could be scantily available. - With heterogeneous brick support, an actual space comparison could - prevent any files being migrated to newly added bricks if they are - smaller then the free space available on the existing bricks. - */ - if (stbuf) { - if (!conf->use_fallocate) { - file_blocks = stbuf->ia_size + GF_DISK_SECTOR_SIZE - 1; - file_blocks /= GF_DISK_SECTOR_SIZE; - - if (file_blocks >= dst_statfs_blocks) { - dst_statfs_blocks = 0; - } else { - dst_statfs_blocks -= file_blocks; - } - } - - src_post_availspacepercent = - ((src_statfs_blocks + file_blocks) * 100) / src_total_blocks; - - dst_post_availspacepercent = - (dst_statfs_blocks * 100) / dst_total_blocks; - - if (dst_post_availspacepercent < src_post_availspacepercent) { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "data movement of file " - "{blocks:%"PRIu64" name:(%s)} would result in " - "dst node (%s:%"PRIu64") having lower disk " - "space than the source node (%s:%"PRIu64")" - ".Skipping file.", stbuf->ia_blocks, loc->path, - to->name, dst_statfs_blocks, from->name, - src_statfs_blocks); - - /* this is not a 'failure', but we don't want to - consider this as 'success' too :-/ */ - *fop_errno = ENOSPC; - ret = 1; - goto out; - } - } + struct statvfs src_statfs = { + 0, + }; + struct statvfs dst_statfs = { + 0, + }; + int ret = -1; + dict_t *xdata = NULL; + dht_layout_t *layout = NULL; + uint64_t src_statfs_blocks = 1; + uint64_t dst_statfs_blocks = 1; + double dst_post_availspacepercent = 0; + double src_post_availspacepercent = 0; + uint64_t file_blocks = 0; + uint64_t src_total_blocks = 0; + uint64_t dst_total_blocks = 0; + + xdata = dict_new(); + if (!xdata) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY, + "failed to allocate dictionary"); + goto out; + } + + ret = dict_set_int8(xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict"); + ret = -1; + *fop_errno = ENOMEM; + goto out; + } + + ret = syncop_statfs(from, loc, &src_statfs, xdata, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "failed to get statfs of %s on %s", loc->path, from->name); + *fop_errno = -ret; + ret = -1; + goto out; + } + + ret = syncop_statfs(to, loc, &dst_statfs, xdata, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "failed to get statfs of %s on %s", loc->path, to->name); + *fop_errno = -ret; + ret = -1; + goto out; + } + + gf_msg_debug(this->name, 0, + "min_free_disk - %f , block available - " + "%lu , block size - %lu ", + conf->min_free_disk, dst_statfs.f_bavail, dst_statfs.f_bsize); + + dst_statfs_blocks = dst_statfs.f_bavail * + (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE); + + src_statfs_blocks = src_statfs.f_bavail * + (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE); + + dst_total_blocks = dst_statfs.f_blocks * + (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE); + + src_total_blocks = src_statfs.f_blocks * + (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE); + + /* if force option is given, do not check for space @ dst. + * Check only if space is avail for the file */ + if (flag != GF_DHT_MIGRATE_DATA) + goto check_avail_space; + + /* Check: + During rebalance `migrate-data` - Destination subvol experiences + a `reduction` in 'blocks' of free space, at the same time source + subvol gains certain 'blocks' of free space. A valid check is + necessary here to avoid erroneous move to destination where + the space could be scantily available. + With heterogeneous brick support, an actual space comparison could + prevent any files being migrated to newly added bricks if they are + smaller then the free space available on the existing bricks. + */ + if (stbuf) { + if (!conf->use_fallocate) { + file_blocks = stbuf->ia_size + GF_DISK_SECTOR_SIZE - 1; + file_blocks /= GF_DISK_SECTOR_SIZE; + + if (file_blocks >= dst_statfs_blocks) { + dst_statfs_blocks = 0; + } else { + dst_statfs_blocks -= file_blocks; + } + } + + src_post_availspacepercent = ((src_statfs_blocks + file_blocks) * 100) / + src_total_blocks; + + dst_post_availspacepercent = (dst_statfs_blocks * 100) / + dst_total_blocks; + + if (dst_post_availspacepercent < src_post_availspacepercent) { + gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "data movement of file " + "{blocks:%" PRIu64 + " name:(%s)} would result in " + "dst node (%s:%" PRIu64 + ") having lower disk " + "space than the source node (%s:%" PRIu64 + ")" + ".Skipping file.", + stbuf->ia_blocks, loc->path, to->name, dst_statfs_blocks, + from->name, src_statfs_blocks); + + /* this is not a 'failure', but we don't want to + consider this as 'success' too :-/ */ + *fop_errno = ENOSPC; + ret = 1; + goto out; + } + } check_avail_space: - if (conf->disk_unit == 'p' && dst_statfs.f_blocks) { - dst_post_availspacepercent = - (dst_statfs_blocks * 100) / dst_total_blocks; - - gf_msg_debug (this->name, 0, "file : %s, post_availspacepercent" - " : %lf f_bavail : %lu min-free-disk: %lf", - loc->path, dst_post_availspacepercent, - dst_statfs.f_bavail, conf->min_free_disk); - - if (dst_post_availspacepercent < conf->min_free_disk) { - gf_msg (this->name, GF_LOG_WARNING, 0, 0, - "Write will cross min-free-disk for " - "file - %s on subvol - %s. Looking " - "for new subvol", loc->path, to->name); - - goto find_new_subvol; - } else { - ret = 0; - goto out; - } + if (conf->disk_unit == 'p' && dst_statfs.f_blocks) { + dst_post_availspacepercent = (dst_statfs_blocks * 100) / + dst_total_blocks; + + gf_msg_debug(this->name, 0, + "file : %s, post_availspacepercent" + " : %lf f_bavail : %lu min-free-disk: %lf", + loc->path, dst_post_availspacepercent, dst_statfs.f_bavail, + conf->min_free_disk); + + if (dst_post_availspacepercent < conf->min_free_disk) { + gf_msg(this->name, GF_LOG_WARNING, 0, 0, + "Write will cross min-free-disk for " + "file - %s on subvol - %s. Looking " + "for new subvol", + loc->path, to->name); + + goto find_new_subvol; + } else { + ret = 0; + goto out; } + } - if (conf->disk_unit != 'p') { - if ((dst_statfs_blocks * GF_DISK_SECTOR_SIZE) < - conf->min_free_disk) { - gf_msg_debug (this->name, 0, "file : %s, destination " - "frsize: %lu f_bavail : %lu " - "min-free-disk: %lf", loc->path, - dst_statfs.f_frsize, dst_statfs.f_bavail, - conf->min_free_disk); + if (conf->disk_unit != 'p') { + if ((dst_statfs_blocks * GF_DISK_SECTOR_SIZE) < conf->min_free_disk) { + gf_msg_debug(this->name, 0, + "file : %s, destination " + "frsize: %lu f_bavail : %lu " + "min-free-disk: %lf", + loc->path, dst_statfs.f_frsize, dst_statfs.f_bavail, + conf->min_free_disk); - gf_msg (this->name, GF_LOG_WARNING, 0, 0, "write will" - " cross min-free-disk for file - %s on subvol -" - " %s. looking for new subvol", loc->path, - to->name); + gf_msg(this->name, GF_LOG_WARNING, 0, 0, + "write will" + " cross min-free-disk for file - %s on subvol -" + " %s. looking for new subvol", + loc->path, to->name); - goto find_new_subvol; + goto find_new_subvol; - } else { - ret = 0; - goto out; - } + } else { + ret = 0; + goto out; } + } find_new_subvol: - layout = dht_layout_get (this, loc->parent); - if (!layout) { - gf_log (this->name, GF_LOG_ERROR, "Layout is NULL"); - *fop_errno = EINVAL; - ret = -1; - goto out; - } - - *new_subvol = dht_subvol_with_free_space_inodes (this, to, from, layout, - stbuf->ia_size); - if ((!(*new_subvol)) || (*new_subvol == from)) { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_SUBVOL_INSUFF_SPACE, "Could not find any subvol" - " with space accommodating the file - %s. Consider " - "adding bricks", loc->path); - - *target_changed = _gf_false; - *fop_errno = ENOSPC; - ret = -1; - } else { - gf_msg (this->name, GF_LOG_INFO, 0, 0, "new target found - %s" - " for file - %s", (*new_subvol)->name, loc->path); - *target_changed = _gf_true; - ret = 0; - } + layout = dht_layout_get(this, loc->parent); + if (!layout) { + gf_log(this->name, GF_LOG_ERROR, "Layout is NULL"); + *fop_errno = EINVAL; + ret = -1; + goto out; + } + + *new_subvol = dht_subvol_with_free_space_inodes(this, to, from, layout, + stbuf->ia_size); + if ((!(*new_subvol)) || (*new_subvol == from)) { + gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE, + "Could not find any subvol" + " with space accommodating the file - %s. Consider " + "adding bricks", + loc->path); + + *target_changed = _gf_false; + *fop_errno = ENOSPC; + ret = -1; + } else { + gf_msg(this->name, GF_LOG_INFO, 0, 0, + "new target found - %s" + " for file - %s", + (*new_subvol)->name, loc->path); + *target_changed = _gf_true; + ret = 0; + } out: - if (xdata) - dict_unref (xdata); - return ret; + if (xdata) + dict_unref(xdata); + return ret; } static int -__dht_rebalance_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, - xlator_t *from, xlator_t *to, fd_t *src, - fd_t *dst, uint64_t ia_size, int hole_exists, - int *fop_errno) +__dht_rebalance_migrate_data(xlator_t *this, gf_defrag_info_t *defrag, + xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst, + uint64_t ia_size, int hole_exists, int *fop_errno) { - int ret = 0; - int count = 0; - off_t offset = 0; - struct iovec *vector = NULL; - struct iobref *iobref = NULL; - uint64_t total = 0; - size_t read_size = 0; - dict_t *xdata = NULL; - dht_conf_t *conf = NULL; - - conf = this->private; - /* if file size is '0', no need to enter this loop */ - while (total < ia_size) { - read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) ? - DHT_REBALANCE_BLKSIZE : (ia_size - total)); - - ret = syncop_readv (from, src, read_size, - offset, 0, &vector, &count, &iobref, NULL, - NULL, NULL); - if (!ret || (ret < 0)) { - *fop_errno = -ret; - break; - } - - if (hole_exists) { - ret = dht_write_with_holes (to, dst, vector, count, - ret, offset, iobref, - fop_errno); - } else { - if (!conf->force_migration && - !dht_is_tier_xlator (this)) { - xdata = dict_new (); - if (!xdata) { - gf_msg ("dht", GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "insufficient memory"); - ret = -1; - *fop_errno = ENOMEM; - break; - } - - /* Fail this write and abort rebalance if we - * detect a write from client since migration of - * this file started. This is done to avoid - * potential data corruption due to out of order - * writes from rebalance and client to the same - * region (as compared between src and dst - * files). See - * https://github.com/gluster/glusterfs/issues/308 - * for more details. - */ - ret = dict_set_int32 (xdata, - GF_AVOID_OVERWRITE, 1); - if (ret) { - gf_msg ("dht", GF_LOG_ERROR, 0, - ENOMEM, "failed to set dict"); - ret = -1; - *fop_errno = ENOMEM; - break; - } - - } - - ret = syncop_writev (to, dst, vector, count, - offset, iobref, 0, NULL, NULL, - xdata, NULL); - if (ret < 0) { - *fop_errno = -ret; - } - } - - if ((defrag && defrag->cmd == GF_DEFRAG_CMD_START_TIER) && - (gf_defrag_get_pause_state (&defrag->tier_conf) != TIER_RUNNING)) { - gf_msg ("tier", GF_LOG_INFO, 0, - DHT_MSG_TIER_PAUSED, - "Migrate file paused"); - ret = -1; + int ret = 0; + int count = 0; + off_t offset = 0; + struct iovec *vector = NULL; + struct iobref *iobref = NULL; + uint64_t total = 0; + size_t read_size = 0; + dict_t *xdata = NULL; + dht_conf_t *conf = NULL; + + conf = this->private; + /* if file size is '0', no need to enter this loop */ + while (total < ia_size) { + read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) + ? DHT_REBALANCE_BLKSIZE + : (ia_size - total)); + + ret = syncop_readv(from, src, read_size, offset, 0, &vector, &count, + &iobref, NULL, NULL, NULL); + if (!ret || (ret < 0)) { + *fop_errno = -ret; + break; + } + + if (hole_exists) { + ret = dht_write_with_holes(to, dst, vector, count, ret, offset, + iobref, fop_errno); + } else { + if (!conf->force_migration && !dht_is_tier_xlator(this)) { + xdata = dict_new(); + if (!xdata) { + gf_msg("dht", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "insufficient memory"); + ret = -1; + *fop_errno = ENOMEM; + break; } - if (ret < 0) { - break; + /* Fail this write and abort rebalance if we + * detect a write from client since migration of + * this file started. This is done to avoid + * potential data corruption due to out of order + * writes from rebalance and client to the same + * region (as compared between src and dst + * files). See + * https://github.com/gluster/glusterfs/issues/308 + * for more details. + */ + ret = dict_set_int32(xdata, GF_AVOID_OVERWRITE, 1); + if (ret) { + gf_msg("dht", GF_LOG_ERROR, 0, ENOMEM, + "failed to set dict"); + ret = -1; + *fop_errno = ENOMEM; + break; } + } - offset += ret; - total += ret; - - GF_FREE (vector); - if (iobref) - iobref_unref (iobref); - iobref = NULL; - vector = NULL; - } - if (iobref) - iobref_unref (iobref); - GF_FREE (vector); - - if (ret >= 0) - ret = 0; - else - ret = -1; - - if (xdata) { - dict_unref (xdata); + ret = syncop_writev(to, dst, vector, count, offset, iobref, 0, NULL, + NULL, xdata, NULL); + if (ret < 0) { + *fop_errno = -ret; + } } - return ret; -} - - -static int -__dht_rebalance_open_src_file (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc, - struct iatt *stbuf, fd_t **src_fd, - gf_boolean_t *clean_src, int *fop_errno) -{ - - int ret = 0; - fd_t *fd = NULL; - dict_t *dict = NULL; - struct iatt iatt = {0,}; - dht_conf_t *conf = NULL; - - conf = this->private; - - *clean_src = _gf_false; - - fd = fd_create (loc->inode, DHT_REBALANCE_PID); - if (!fd) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: fd create failed (source)", loc->path); - *fop_errno = ENOMEM; - ret = -1; - goto out; + if ((defrag && defrag->cmd == GF_DEFRAG_CMD_START_TIER) && + (gf_defrag_get_pause_state(&defrag->tier_conf) != TIER_RUNNING)) { + gf_msg("tier", GF_LOG_INFO, 0, DHT_MSG_TIER_PAUSED, + "Migrate file paused"); + ret = -1; } - ret = syncop_open (from, loc, O_RDWR, fd, NULL, NULL); if (ret < 0) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to open file %s on %s", - loc->path, from->name); - *fop_errno = -ret; - ret = -1; - goto out; + break; } - fd_bind (fd); - - if (src_fd) - *src_fd = fd; + offset += ret; + total += ret; + GF_FREE(vector); + if (iobref) + iobref_unref(iobref); + iobref = NULL; + vector = NULL; + } + if (iobref) + iobref_unref(iobref); + GF_FREE(vector); + + if (ret >= 0) + ret = 0; + else ret = -1; - dict = dict_new (); - if (!dict) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: Could not allocate memory for dict", loc->path); - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - ret = dict_set_str (dict, conf->link_xattr_name, to->name); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "failed to set xattr in dict for %s (linkto:%s)", - loc->path, to->name); - *fop_errno = ENOMEM; - ret = -1; - goto out; - } + if (xdata) { + dict_unref(xdata); + } - /* Once the migration starts, the source should have 'linkto' key set - to show which is the target, so other clients can work around it */ - ret = syncop_setxattr (from, loc, dict, 0, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to set xattr on %s in %s", - loc->path, from->name); - *fop_errno = -ret; - ret = -1; - goto out; - } + return ret; +} - /* Reset source mode/xattr if migration fails*/ - *clean_src = _gf_true; +static int +__dht_rebalance_open_src_file(xlator_t *this, xlator_t *from, xlator_t *to, + loc_t *loc, struct iatt *stbuf, fd_t **src_fd, + gf_boolean_t *clean_src, int *fop_errno) +{ + int ret = 0; + fd_t *fd = NULL; + dict_t *dict = NULL; + struct iatt iatt = { + 0, + }; + dht_conf_t *conf = NULL; + + conf = this->private; + + *clean_src = _gf_false; + + fd = fd_create(loc->inode, DHT_REBALANCE_PID); + if (!fd) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: fd create failed (source)", loc->path); + *fop_errno = ENOMEM; + ret = -1; + goto out; + } + + ret = syncop_open(from, loc, O_RDWR, fd, NULL, NULL); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "failed to open file %s on %s", loc->path, from->name); + *fop_errno = -ret; + ret = -1; + goto out; + } - /* mode should be (+S+T) to indicate migration is in progress */ - iatt.ia_prot = stbuf->ia_prot; - iatt.ia_type = stbuf->ia_type; - iatt.ia_prot.sticky = 1; - iatt.ia_prot.sgid = 1; + fd_bind(fd); - ret = syncop_setattr (from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL, - NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to set mode on %s in %s", - loc->path, from->name); - *fop_errno = -ret; - ret = -1; - goto out; - } + if (src_fd) + *src_fd = fd; - /* success */ - ret = 0; + ret = -1; + dict = dict_new(); + if (!dict) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: Could not allocate memory for dict", loc->path); + *fop_errno = ENOMEM; + ret = -1; + goto out; + } + + ret = dict_set_str(dict, conf->link_xattr_name, to->name); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "failed to set xattr in dict for %s (linkto:%s)", loc->path, + to->name); + *fop_errno = ENOMEM; + ret = -1; + goto out; + } + + /* Once the migration starts, the source should have 'linkto' key set + to show which is the target, so other clients can work around it */ + ret = syncop_setxattr(from, loc, dict, 0, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "failed to set xattr on %s in %s", loc->path, from->name); + *fop_errno = -ret; + ret = -1; + goto out; + } + + /* Reset source mode/xattr if migration fails*/ + *clean_src = _gf_true; + + /* mode should be (+S+T) to indicate migration is in progress */ + iatt.ia_prot = stbuf->ia_prot; + iatt.ia_type = stbuf->ia_type; + iatt.ia_prot.sticky = 1; + iatt.ia_prot.sgid = 1; + + ret = syncop_setattr(from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL, NULL, + NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "failed to set mode on %s in %s", loc->path, from->name); + *fop_errno = -ret; + ret = -1; + goto out; + } + + /* success */ + ret = 0; out: - if (dict) - dict_unref (dict); + if (dict) + dict_unref(dict); - return ret; + return ret; } int -migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc, - struct iatt *buf, int *fop_errno) +migrate_special_files(xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc, + struct iatt *buf, int *fop_errno) { - int ret = -1; - dict_t *rsp_dict = NULL; - dict_t *dict = NULL; - char *link = NULL; - struct iatt stbuf = {0,}; - dht_conf_t *conf = this->private; - - dict = dict_new (); - if (!dict) { - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - ret = dict_set_int32 (dict, conf->link_xattr_name, 256); - if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_log (this->name, GF_LOG_ERROR, - "%s: failed to set 'linkto' key in dict", loc->path); - goto out; - } - - /* check in the destination if the file is link file */ - ret = syncop_lookup (to, loc, &stbuf, NULL, dict, &rsp_dict); - if ((ret < 0) && (-ret != ENOENT)) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: lookup failed", - loc->path); - *fop_errno = -ret; - ret = -1; - goto out; - } - - /* we no more require this key */ - dict_del (dict, conf->link_xattr_name); - - /* file exists in target node, only if it is 'linkfile' its valid, - otherwise, error out */ - if (!ret) { - if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict, - conf->link_xattr_name)) { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: file exists in destination", loc->path); - *fop_errno = EINVAL; - ret = -1; - goto out; - } - - /* as file is linkfile, delete it */ - ret = syncop_unlink (to, loc, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to delete the linkfile", - loc->path); - *fop_errno = -ret; - ret = -1; - goto out; - } - } - - /* Set the gfid of the source file in dict */ - ret = dict_set_gfuuid (dict, "gfid-req", buf->ia_gfid, true); + int ret = -1; + dict_t *rsp_dict = NULL; + dict_t *dict = NULL; + char *link = NULL; + struct iatt stbuf = { + 0, + }; + dht_conf_t *conf = this->private; + + dict = dict_new(); + if (!dict) { + *fop_errno = ENOMEM; + ret = -1; + goto out; + } + ret = dict_set_int32(dict, conf->link_xattr_name, 256); + if (ret) { + *fop_errno = ENOMEM; + ret = -1; + gf_log(this->name, GF_LOG_ERROR, + "%s: failed to set 'linkto' key in dict", loc->path); + goto out; + } + + /* check in the destination if the file is link file */ + ret = syncop_lookup(to, loc, &stbuf, NULL, dict, &rsp_dict); + if ((ret < 0) && (-ret != ENOENT)) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: lookup failed", loc->path); + *fop_errno = -ret; + ret = -1; + goto out; + } + + /* we no more require this key */ + dict_del(dict, conf->link_xattr_name); + + /* file exists in target node, only if it is 'linkfile' its valid, + otherwise, error out */ + if (!ret) { + if (!check_is_linkfile(loc->inode, &stbuf, rsp_dict, + conf->link_xattr_name)) { + gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: file exists in destination", loc->path); + *fop_errno = EINVAL; + ret = -1; + goto out; + } + + /* as file is linkfile, delete it */ + ret = syncop_unlink(to, loc, NULL, NULL); if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_log (this->name, GF_LOG_ERROR, - "%s: failed to set gfid in dict for create", loc->path); - goto out; - } - - /* Create the file in target */ - if (IA_ISLNK (buf->ia_type)) { - /* Handle symlinks separately */ - ret = syncop_readlink (from, loc, &link, buf->ia_size, NULL, - NULL); - if (ret < 0) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: readlink on symlink failed", - loc->path); - *fop_errno = -ret; - ret = -1; - goto out; - } - - ret = syncop_symlink (to, loc, link, 0, dict, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: creating symlink failed", - loc->path); - *fop_errno = -ret; - ret = -1; - goto out; - } - - goto done; + gf_msg(this->name, GF_LOG_WARNING, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to delete the linkfile", loc->path); + *fop_errno = -ret; + ret = -1; + goto out; + } + } + + /* Set the gfid of the source file in dict */ + ret = dict_set_gfuuid(dict, "gfid-req", buf->ia_gfid, true); + if (ret) { + *fop_errno = ENOMEM; + ret = -1; + gf_log(this->name, GF_LOG_ERROR, + "%s: failed to set gfid in dict for create", loc->path); + goto out; + } + + /* Create the file in target */ + if (IA_ISLNK(buf->ia_type)) { + /* Handle symlinks separately */ + ret = syncop_readlink(from, loc, &link, buf->ia_size, NULL, NULL); + if (ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "%s: readlink on symlink failed", loc->path); + *fop_errno = -ret; + ret = -1; + goto out; } - ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot, - buf->ia_type), - makedev (ia_major (buf->ia_rdev), - ia_minor (buf->ia_rdev)), 0, dict, NULL); + ret = syncop_symlink(to, loc, link, 0, dict, NULL); if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: mknod failed", - loc->path); - *fop_errno = -ret; - ret = -1; - goto out; - } + gf_msg(this->name, GF_LOG_WARNING, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, "%s: creating symlink failed", + loc->path); + *fop_errno = -ret; + ret = -1; + goto out; + } + + goto done; + } + + ret = syncop_mknod(to, loc, st_mode_from_ia(buf->ia_prot, buf->ia_type), + makedev(ia_major(buf->ia_rdev), ia_minor(buf->ia_rdev)), + 0, dict, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: mknod failed", loc->path); + *fop_errno = -ret; + ret = -1; + goto out; + } done: - ret = syncop_setattr (to, loc, buf, - (GF_SET_ATTR_MTIME | - GF_SET_ATTR_UID | GF_SET_ATTR_GID | - GF_SET_ATTR_MODE), NULL, NULL, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to perform setattr on %s", - loc->path, to->name); - *fop_errno = -ret; - } - - ret = syncop_unlink (from, loc, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: unlink failed", - loc->path); - *fop_errno = -ret; - ret = -1; - } + ret = syncop_setattr(to, loc, buf, + (GF_SET_ATTR_MTIME | GF_SET_ATTR_UID | + GF_SET_ATTR_GID | GF_SET_ATTR_MODE), + NULL, NULL, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to perform setattr on %s", loc->path, to->name); + *fop_errno = -ret; + } + + ret = syncop_unlink(from, loc, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: unlink failed", loc->path); + *fop_errno = -ret; + ret = -1; + } out: - GF_FREE (link); - if (dict) - dict_unref (dict); + GF_FREE(link); + if (dict) + dict_unref(dict); - if (rsp_dict) - dict_unref (rsp_dict); + if (rsp_dict) + dict_unref(rsp_dict); - return ret; + return ret; } static int -__dht_migration_cleanup_src_file (xlator_t *this, loc_t *loc, fd_t *fd, - xlator_t *from, ia_prot_t *src_ia_prot) +__dht_migration_cleanup_src_file(xlator_t *this, loc_t *loc, fd_t *fd, + xlator_t *from, ia_prot_t *src_ia_prot) { - int ret = -1; - dht_conf_t *conf = NULL; - struct iatt new_stbuf = {0,}; - - if (!this || !fd || !from || !src_ia_prot) { - goto out; - } - - conf = this->private; - - /*Revert source mode and xattr changes*/ - ret = syncop_fstat (from, fd, &new_stbuf, NULL, NULL); - if (ret < 0) { - /* Failed to get the stat info */ - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file cleanup failed: failed to fstat " - "file %s on %s ", loc->path, from->name); - ret = -1; - goto out; - } - - - /* Remove the sticky bit and sgid bit set, reset it to 0*/ - if (!src_ia_prot->sticky) - new_stbuf.ia_prot.sticky = 0; + int ret = -1; + dht_conf_t *conf = NULL; + struct iatt new_stbuf = { + 0, + }; + + if (!this || !fd || !from || !src_ia_prot) { + goto out; + } + + conf = this->private; + + /*Revert source mode and xattr changes*/ + ret = syncop_fstat(from, fd, &new_stbuf, NULL, NULL); + if (ret < 0) { + /* Failed to get the stat info */ + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file cleanup failed: failed to fstat " + "file %s on %s ", + loc->path, from->name); + ret = -1; + goto out; + } - if (!src_ia_prot->sgid) - new_stbuf.ia_prot.sgid = 0; + /* Remove the sticky bit and sgid bit set, reset it to 0*/ + if (!src_ia_prot->sticky) + new_stbuf.ia_prot.sticky = 0; - ret = syncop_fsetattr (from, fd, &new_stbuf, - (GF_SET_ATTR_GID | GF_SET_ATTR_MODE), - NULL, NULL, NULL, NULL); + if (!src_ia_prot->sgid) + new_stbuf.ia_prot.sgid = 0; - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file cleanup failed:" - "%s: failed to perform fsetattr on %s ", - loc->path, from->name); - ret = -1; - goto out; - } + ret = syncop_fsetattr(from, fd, &new_stbuf, + (GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL, + NULL, NULL); - ret = syncop_fremovexattr (from, fd, conf->link_xattr_name, 0, NULL); - if (ret) { - gf_log (this->name, GF_LOG_WARNING, - "%s: failed to remove linkto xattr on %s (%s)", - loc->path, from->name, strerror (-ret)); - ret = -1; - goto out; - } + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file cleanup failed:" + "%s: failed to perform fsetattr on %s ", + loc->path, from->name); + ret = -1; + goto out; + } + + ret = syncop_fremovexattr(from, fd, conf->link_xattr_name, 0, NULL); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, + "%s: failed to remove linkto xattr on %s (%s)", loc->path, + from->name, strerror(-ret)); + ret = -1; + goto out; + } - ret = 0; + ret = 0; out: - return ret; + return ret; } - - /* return values: @@ -1586,1098 +1513,1078 @@ out: 1 : not a failure, but we can't migrate data as of now */ int -dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to, - int flag, int *fop_errno) +dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to, + int flag, int *fop_errno) { - int ret = -1; - struct iatt new_stbuf = {0,}; - struct iatt stbuf = {0,}; - struct iatt empty_iatt = {0,}; - ia_prot_t src_ia_prot = {0,}; - fd_t *src_fd = NULL; - fd_t *dst_fd = NULL; - dict_t *dict = NULL; - dict_t *xattr = NULL; - dict_t *xattr_rsp = NULL; - int file_has_holes = 0; - dht_conf_t *conf = this->private; - int rcvd_enoent_from_src = 0; - struct gf_flock flock = {0, }; - struct gf_flock plock = {0, }; - loc_t tmp_loc = {0, }; - loc_t parent_loc = {0, }; - gf_boolean_t inodelk_locked = _gf_false; - gf_boolean_t entrylk_locked = _gf_false; - gf_boolean_t p_locked = _gf_false; - int lk_ret = -1; - gf_defrag_info_t *defrag = NULL; - gf_boolean_t clean_src = _gf_false; - gf_boolean_t clean_dst = _gf_false; - int log_level = GF_LOG_INFO; - gf_boolean_t delete_src_linkto = _gf_true; - lock_migration_info_t locklist; - dict_t *meta_dict = NULL; - gf_boolean_t meta_locked = _gf_false; - gf_boolean_t target_changed = _gf_false; - xlator_t *new_target = NULL; - xlator_t *old_target = NULL; - xlator_t *hashed_subvol = NULL; - fd_t *linkto_fd = NULL; - - - if (from == to) { - gf_msg_debug (this->name, 0, "destination and source are same. file %s" - " might have migrated already", loc->path); - ret = 0; - goto out; - } + int ret = -1; + struct iatt new_stbuf = { + 0, + }; + struct iatt stbuf = { + 0, + }; + struct iatt empty_iatt = { + 0, + }; + ia_prot_t src_ia_prot = { + 0, + }; + fd_t *src_fd = NULL; + fd_t *dst_fd = NULL; + dict_t *dict = NULL; + dict_t *xattr = NULL; + dict_t *xattr_rsp = NULL; + int file_has_holes = 0; + dht_conf_t *conf = this->private; + int rcvd_enoent_from_src = 0; + struct gf_flock flock = { + 0, + }; + struct gf_flock plock = { + 0, + }; + loc_t tmp_loc = { + 0, + }; + loc_t parent_loc = { + 0, + }; + gf_boolean_t inodelk_locked = _gf_false; + gf_boolean_t entrylk_locked = _gf_false; + gf_boolean_t p_locked = _gf_false; + int lk_ret = -1; + gf_defrag_info_t *defrag = NULL; + gf_boolean_t clean_src = _gf_false; + gf_boolean_t clean_dst = _gf_false; + int log_level = GF_LOG_INFO; + gf_boolean_t delete_src_linkto = _gf_true; + lock_migration_info_t locklist; + dict_t *meta_dict = NULL; + gf_boolean_t meta_locked = _gf_false; + gf_boolean_t target_changed = _gf_false; + xlator_t *new_target = NULL; + xlator_t *old_target = NULL; + xlator_t *hashed_subvol = NULL; + fd_t *linkto_fd = NULL; + + if (from == to) { + gf_msg_debug(this->name, 0, + "destination and source are same. file %s" + " might have migrated already", + loc->path); + ret = 0; + goto out; + } - /* If defrag is NULL, it should be assumed that migration is triggered - * from client */ - defrag = conf->defrag; + /* If defrag is NULL, it should be assumed that migration is triggered + * from client */ + defrag = conf->defrag; - /* migration of files from clients is restricted to non-tiered clients - * for now */ - if (!defrag && dht_is_tier_xlator (this)) { - ret = ENOTSUP; - goto out; - } + /* migration of files from clients is restricted to non-tiered clients + * for now */ + if (!defrag && dht_is_tier_xlator(this)) { + ret = ENOTSUP; + goto out; + } - if (defrag && defrag->tier_conf.is_tier) - log_level = GF_LOG_TRACE; + if (defrag && defrag->tier_conf.is_tier) + log_level = GF_LOG_TRACE; - gf_log (this->name, - log_level, "%s: attempting to move from %s to %s", - loc->path, from->name, to->name); + gf_log(this->name, log_level, "%s: attempting to move from %s to %s", + loc->path, from->name, to->name); - dict = dict_new (); - if (!dict) { - ret = -1; - *fop_errno = ENOMEM; - gf_msg (this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY, - "Could not allocate memory for dict"); - goto out; - } - ret = dict_set_int32 (dict, conf->link_xattr_name, 256); + dict = dict_new(); + if (!dict) { + ret = -1; + *fop_errno = ENOMEM; + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY, + "Could not allocate memory for dict"); + goto out; + } + ret = dict_set_int32(dict, conf->link_xattr_name, 256); + if (ret) { + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: failed to set 'linkto' key in dict", + loc->path); + goto out; + } + + /* Do not migrate file in case lock migration is not enabled on the + * volume*/ + if (!conf->lock_migration_enabled) { + ret = dict_set_int32(dict, GLUSTERFS_POSIXLK_COUNT, sizeof(int32_t)); if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: failed to set 'linkto' key in dict", loc->path); - goto out; - } - - /* Do not migrate file in case lock migration is not enabled on the - * volume*/ - if (!conf->lock_migration_enabled) { - ret = dict_set_int32 (dict, - GLUSTERFS_POSIXLK_COUNT, sizeof(int32_t)); - if (ret) { - *fop_errno = ENOMEM; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed: %s: failed to " - "set "GLUSTERFS_POSIXLK_COUNT" key in dict", - loc->path); - goto out; - } - } else { - gf_msg (this->name, GF_LOG_INFO, 0, 0, "locks will be migrated" - " for file: %s", loc->path); - } - - ret = dht_build_parent_loc (this, &parent_loc, loc, fop_errno); - if (ret < 0) { - ret = -1; - gf_msg (this->name, GF_LOG_WARNING, *fop_errno, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to build parent loc, which is needed to " - "acquire entrylk to synchronize with renames on this " - "path. Skipping migration", loc->path); - goto out; - } - - hashed_subvol = dht_subvol_get_hashed (this, loc); - if (hashed_subvol == NULL) { - ret = -1; - gf_msg (this->name, GF_LOG_WARNING, EINVAL, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: cannot find hashed subvol which is needed to " - "synchronize with renames on this path. " - "Skipping migration", loc->path); - goto out; - } - - flock.l_type = F_WRLCK; - - tmp_loc.inode = inode_ref (loc->inode); - gf_uuid_copy (tmp_loc.gfid, loc->gfid); - tmp_loc.path = gf_strdup(loc->path); - - /* this inodelk happens with flock.owner being zero. But to synchronize - * hardlink migration we need to have different lkowner for each migration - * Filed a bug here: https://bugzilla.redhat.com/show_bug.cgi?id=1468202 to - * track the fix for this. Currently synclock takes care of synchronizing - * hardlink migration. Once this bug is fixed we can avoid taking synclock */ - ret = syncop_inodelk (from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW, - &flock, NULL, NULL); - if (ret < 0) { - *fop_errno = -ret; - ret = -1; - gf_msg (this->name, GF_LOG_WARNING, *fop_errno, - DHT_MSG_MIGRATE_FILE_FAILED, - "migrate file failed: " - "%s: failed to lock file on %s", - loc->path, from->name); - goto out; - } - - inodelk_locked = _gf_true; - - /* dht_rename has changed to use entrylk on hashed subvol for - * synchronization. So, rebalance too has to acquire an entrylk on - * hashed subvol. - */ - ret = syncop_entrylk (hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, &parent_loc, - loc->name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL, - NULL); - if (ret < 0) { - *fop_errno = -ret; - ret = -1; - gf_msg (this->name, GF_LOG_WARNING, *fop_errno, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to acquire entrylk on subvol %s", - loc->path, hashed_subvol->name); - goto out; - } - - entrylk_locked = _gf_true; - - /* Phase 1 - Data migration is in progress from now on */ - ret = syncop_lookup (from, loc, &stbuf, NULL, dict, &xattr_rsp); + *fop_errno = ENOMEM; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed: %s: failed to " + "set " GLUSTERFS_POSIXLK_COUNT " key in dict", + loc->path); + goto out; + } + } else { + gf_msg(this->name, GF_LOG_INFO, 0, 0, + "locks will be migrated" + " for file: %s", + loc->path); + } + + ret = dht_build_parent_loc(this, &parent_loc, loc, fop_errno); + if (ret < 0) { + ret = -1; + gf_msg(this->name, GF_LOG_WARNING, *fop_errno, + DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to build parent loc, which is needed to " + "acquire entrylk to synchronize with renames on this " + "path. Skipping migration", + loc->path); + goto out; + } + + hashed_subvol = dht_subvol_get_hashed(this, loc); + if (hashed_subvol == NULL) { + ret = -1; + gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: cannot find hashed subvol which is needed to " + "synchronize with renames on this path. " + "Skipping migration", + loc->path); + goto out; + } + + flock.l_type = F_WRLCK; + + tmp_loc.inode = inode_ref(loc->inode); + gf_uuid_copy(tmp_loc.gfid, loc->gfid); + tmp_loc.path = gf_strdup(loc->path); + + /* this inodelk happens with flock.owner being zero. But to synchronize + * hardlink migration we need to have different lkowner for each migration + * Filed a bug here: https://bugzilla.redhat.com/show_bug.cgi?id=1468202 to + * track the fix for this. Currently synclock takes care of synchronizing + * hardlink migration. Once this bug is fixed we can avoid taking synclock + */ + ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW, + &flock, NULL, NULL); + if (ret < 0) { + *fop_errno = -ret; + ret = -1; + gf_msg(this->name, GF_LOG_WARNING, *fop_errno, + DHT_MSG_MIGRATE_FILE_FAILED, + "migrate file failed: " + "%s: failed to lock file on %s", + loc->path, from->name); + goto out; + } + + inodelk_locked = _gf_true; + + /* dht_rename has changed to use entrylk on hashed subvol for + * synchronization. So, rebalance too has to acquire an entrylk on + * hashed subvol. + */ + ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, &parent_loc, + loc->name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL, NULL); + if (ret < 0) { + *fop_errno = -ret; + ret = -1; + gf_msg(this->name, GF_LOG_WARNING, *fop_errno, + DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to acquire entrylk on subvol %s", loc->path, + hashed_subvol->name); + goto out; + } + + entrylk_locked = _gf_true; + + /* Phase 1 - Data migration is in progress from now on */ + ret = syncop_lookup(from, loc, &stbuf, NULL, dict, &xattr_rsp); + if (ret) { + *fop_errno = -ret; + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, *fop_errno, + DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: lookup failed on %s", + loc->path, from->name); + goto out; + } + + /* preserve source mode, so set the same to the destination */ + src_ia_prot = stbuf.ia_prot; + + /* Check if file can be migrated */ + ret = __is_file_migratable(this, loc, &stbuf, xattr_rsp, flag, defrag, conf, + fop_errno); + if (ret) { + if (ret == HARDLINK_MIG_INPROGRESS) + ret = 0; + goto out; + } + + /* Take care of the special files */ + if (!IA_ISREG(stbuf.ia_type)) { + /* Special files */ + ret = migrate_special_files(this, from, to, loc, &stbuf, fop_errno); + goto out; + } + + /* create the destination, with required modes/xattr */ + ret = __dht_rebalance_create_dst_file(this, to, from, loc, &stbuf, &dst_fd, + fop_errno); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, 0, + "Create dst failed" + " on - %s for file - %s", + to->name, loc->path); + goto out; + } + + clean_dst = _gf_true; + + ret = __dht_check_free_space(this, to, from, loc, &stbuf, flag, conf, + &target_changed, &new_target, fop_errno); + if (target_changed) { + /* Can't handle for hardlinks. Marking this as failure */ + if (flag == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS || stbuf.ia_nlink > 1) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_INSUFF_SPACE, + "Exiting migration for" + " file - %s. flag - %d, stbuf.ia_nlink - %d", + loc->path, flag, stbuf.ia_nlink); + ret = -1; + goto out; + } + + ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL); if (ret) { - *fop_errno = -ret; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, *fop_errno, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: lookup failed on %s", - loc->path, from->name); - goto out; + gf_log(this->name, GF_LOG_WARNING, + "%s: failed to perform truncate on %s (%s)", loc->path, + to->name, strerror(-ret)); } - /* preserve source mode, so set the same to the destination */ - src_ia_prot = stbuf.ia_prot; + syncop_close(dst_fd); + dst_fd = NULL; - /* Check if file can be migrated */ - ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag, defrag, conf, - fop_errno); - if (ret) { - if (ret == HARDLINK_MIG_INPROGRESS) - ret = 0; - goto out; - } + old_target = to; + to = new_target; - /* Take care of the special files */ - if (!IA_ISREG (stbuf.ia_type)) { - /* Special files */ - ret = migrate_special_files (this, from, to, loc, &stbuf, - fop_errno); - goto out; - } + clean_dst = _gf_false; - /* create the destination, with required modes/xattr */ - ret = __dht_rebalance_create_dst_file (this, to, from, loc, &stbuf, - &dst_fd, fop_errno); + /* if the file migration is successful to this new target, then + * update the xattr on the old destination to point the new + * destination. We need to do update this only post migration + * as in case of failure the linkto needs to point to the source + * subvol */ + ret = __dht_rebalance_create_dst_file(this, to, from, loc, &stbuf, + &dst_fd, fop_errno); if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Create dst failed" - " on - %s for file - %s", to->name, loc->path); - goto out; - } - - clean_dst = _gf_true; - - ret = __dht_check_free_space (this, to, from, loc, &stbuf, flag, conf, - &target_changed, &new_target, fop_errno); - if (target_changed) { - /* Can't handle for hardlinks. Marking this as failure */ - if (flag == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS || stbuf.ia_nlink > 1) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_SUBVOL_INSUFF_SPACE, "Exiting migration for" - " file - %s. flag - %d, stbuf.ia_nlink - %d", - loc->path, flag, stbuf.ia_nlink); - ret = -1; - goto out; - } + gf_log(this->name, GF_LOG_ERROR, + "Create dst failed" + " on - %s for file - %s", + to->name, loc->path); + goto out; + } else { + gf_msg(this->name, GF_LOG_INFO, 0, 0, + "destination for file " + "- %s is changed to - %s", + loc->path, to->name); + clean_dst = _gf_true; + } + } + + if (ret) { + goto out; + } + + /* Open the source, and also update mode/xattr */ + ret = __dht_rebalance_open_src_file(this, from, to, loc, &stbuf, &src_fd, + &clean_src, fop_errno); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed: failed to open %s on %s", loc->path, + from->name); + goto out; + } + + /* TODO: move all xattr related operations to fd based operations */ + ret = syncop_listxattr(from, loc, &xattr, NULL, NULL); + if (ret < 0) { + *fop_errno = -ret; + gf_msg(this->name, GF_LOG_WARNING, *fop_errno, + DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: failed to get xattr from %s", + loc->path, from->name); + ret = -1; + goto out; + } + + /* Copying posix acls to the linkto file messes up the permissions*/ + dht_strip_out_acls(xattr); + + /* Remove the linkto xattr as we don't want to overwrite the value + * set on the dst. + */ + dict_del(xattr, conf->link_xattr_name); + + /* We need to error out if this fails as having the wrong shard xattrs + * set on the dst could cause data corruption + */ + ret = syncop_fsetxattr(to, dst_fd, xattr, 0, NULL, NULL); + if (ret < 0) { + *fop_errno = -ret; + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to set xattr on %s", loc->path, to->name); + ret = -1; + goto out; + } + if (xattr_rsp) { + /* we no more require this key */ + dict_del(dict, conf->link_xattr_name); + dict_unref(xattr_rsp); + } + + ret = syncop_fstat(from, src_fd, &stbuf, dict, &xattr_rsp); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:failed to lookup %s on %s ", loc->path, + from->name); + *fop_errno = -ret; + ret = -1; + goto out; + } + + /* Check again if file has hardlink */ + ret = __check_file_has_hardlink(this, loc, &stbuf, xattr_rsp, flag, defrag, + conf, fop_errno); + if (ret) { + if (ret == HARDLINK_MIG_INPROGRESS) + ret = 0; + goto out; + } + /* Try to preserve 'holes' while migrating data */ + if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE)) + file_has_holes = 1; + + ret = __dht_rebalance_migrate_data(this, defrag, from, to, src_fd, dst_fd, + stbuf.ia_size, file_has_holes, + fop_errno); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed: %s: failed to migrate data", loc->path); - ret = syncop_ftruncate (to, dst_fd, 0, NULL, NULL, NULL, NULL); - if (ret) { - gf_log (this->name, GF_LOG_WARNING, - "%s: failed to perform truncate on %s (%s)", - loc->path, to->name, strerror (-ret)); - } + ret = -1; + goto out; + } + + /* TODO: Sync the locks */ + + ret = syncop_fsync(to, dst_fd, 0, NULL, NULL, NULL, NULL); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, "%s: failed to fsync on %s (%s)", + loc->path, to->name, strerror(-ret)); + *fop_errno = -ret; + } + + /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */ + + ret = syncop_fstat(from, src_fd, &new_stbuf, NULL, NULL); + if (ret < 0) { + /* Failed to get the stat info */ + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed: failed to fstat file %s on %s ", loc->path, + from->name); + *fop_errno = -ret; + ret = -1; + goto out; + } - syncop_close (dst_fd); - dst_fd = NULL; + /* Lock the entire source file to prevent clients from taking a + lock on it as dht_lk does not handle file migration. - old_target = to; - to = new_target; + This still leaves a small window where conflicting locks can + be granted to different clients. If client1 requests a blocking + lock on the src file, it will be granted after the migrating + process releases its lock. If client2 requests a lock on the dst + data file, it will also be granted, but all FOPs will be redirected + to the dst data file. + */ - clean_dst = _gf_false; + /* Take meta lock */ + if (conf->lock_migration_enabled) { + meta_dict = dict_new(); + if (!meta_dict) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "dict_new failed"); - /* if the file migration is successful to this new target, then - * update the xattr on the old destination to point the new - * destination. We need to do update this only post migration - * as in case of failure the linkto needs to point to the source - * subvol */ - ret = __dht_rebalance_create_dst_file (this, to, from, loc, &stbuf, - &dst_fd, fop_errno); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, "Create dst failed" - " on - %s for file - %s", to->name, loc->path); - goto out; - } else { - gf_msg (this->name, GF_LOG_INFO, 0, 0, "destination for file " - "- %s is changed to - %s", loc->path, to->name); - clean_dst = _gf_true; - } + *fop_errno = ENOMEM; + ret = -1; + goto out; } + ret = dict_set_str(meta_dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); if (ret) { - goto out; + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, + "Failed to set dictionary value: key = %s," + " path = %s", + GLUSTERFS_INTERNAL_FOP_KEY, loc->path); + *fop_errno = ENOMEM; + ret = -1; + goto out; } - /* Open the source, and also update mode/xattr */ - ret = __dht_rebalance_open_src_file (this, from, to, loc, &stbuf, &src_fd, - &clean_src, fop_errno); + ret = dict_set_int32(meta_dict, GF_META_LOCK_KEY, 1); if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed: failed to open %s on %s", - loc->path, from->name); - goto out; - } - - /* TODO: move all xattr related operations to fd based operations */ - ret = syncop_listxattr (from, loc, &xattr, NULL, NULL); - if (ret < 0) { - *fop_errno = -ret; - gf_msg (this->name, GF_LOG_WARNING, *fop_errno, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: failed to get xattr from %s", - loc->path, from->name); - ret = -1; - goto out; - } - - /* Copying posix acls to the linkto file messes up the permissions*/ - dht_strip_out_acls (xattr); - - /* Remove the linkto xattr as we don't want to overwrite the value - * set on the dst. - */ - dict_del (xattr, conf->link_xattr_name); - - /* We need to error out if this fails as having the wrong shard xattrs - * set on the dst could cause data corruption - */ - ret = syncop_fsetxattr (to, dst_fd, xattr, 0, NULL, NULL); - if (ret < 0) { - *fop_errno = -ret; - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to set xattr on %s", - loc->path, to->name); - ret = -1; - goto out; - } - - if (xattr_rsp) { - /* we no more require this key */ - dict_del (dict, conf->link_xattr_name); - dict_unref (xattr_rsp); + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Trace dict_set failed"); + *fop_errno = ENOMEM; + ret = -1; + goto out; } - ret = syncop_fstat (from, src_fd, &stbuf, dict, &xattr_rsp); + ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL); if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:failed to lookup %s on %s ", - loc->path, from->name); - *fop_errno = -ret; - ret = -1; - goto out; - } + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Trace syncop_setxattr metalock failed"); - /* Check again if file has hardlink */ - ret = __check_file_has_hardlink (this, loc, &stbuf, xattr_rsp, - flag, defrag, conf, fop_errno); - if (ret) { - if (ret == HARDLINK_MIG_INPROGRESS) - ret = 0; - goto out; + *fop_errno = -ret; + ret = -1; + goto out; + } else { + meta_locked = _gf_true; } - /* Try to preserve 'holes' while migrating data */ - if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE)) - file_has_holes = 1; + } + if (!conf->lock_migration_enabled) { + plock.l_type = F_WRLCK; + plock.l_start = 0; + plock.l_len = 0; + plock.l_whence = SEEK_SET; - ret = __dht_rebalance_migrate_data (this, defrag, from, to, - src_fd, dst_fd, stbuf.ia_size, - file_has_holes, fop_errno); + ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL); if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed: %s: failed to migrate data", - loc->path); - - ret = -1; - goto out; + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: Failed to lock on %s", + loc->path, from->name); + *fop_errno = -ret; + ret = -1; + goto out; } - /* TODO: Sync the locks */ + p_locked = _gf_true; - ret = syncop_fsync (to, dst_fd, 0, NULL, NULL, NULL, NULL); - if (ret) { - gf_log (this->name, GF_LOG_WARNING, - "%s: failed to fsync on %s (%s)", - loc->path, to->name, strerror (-ret)); - *fop_errno = -ret; - } + } else { + INIT_LIST_HEAD(&locklist.list); + ret = syncop_getactivelk(from, loc, &locklist, NULL, NULL); + if (ret == 0) { + gf_log(this->name, GF_LOG_INFO, "No active locks on:%s", loc->path); - /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */ + } else if (ret > 0) { + ret = syncop_setactivelk(to, loc, &locklist, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, + DHT_MSG_LOCK_MIGRATION_FAILED, "write lock failed on:%s", + loc->path); - ret = syncop_fstat (from, src_fd, &new_stbuf, NULL, NULL); - if (ret < 0) { - /* Failed to get the stat info */ - gf_msg ( this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed: failed to fstat file %s on %s ", - loc->path, from->name); *fop_errno = -ret; ret = -1; - goto out; - } - - /* Lock the entire source file to prevent clients from taking a - lock on it as dht_lk does not handle file migration. - - This still leaves a small window where conflicting locks can - be granted to different clients. If client1 requests a blocking - lock on the src file, it will be granted after the migrating - process releases its lock. If client2 requests a lock on the dst - data file, it will also be granted, but all FOPs will be redirected - to the dst data file. - */ - - /* Take meta lock */ - - if (conf->lock_migration_enabled) { - meta_dict = dict_new (); - if (!meta_dict) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "dict_new failed"); - - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - - ret = dict_set_str (meta_dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_DICT_SET_FAILED, - "Failed to set dictionary value: key = %s," - " path = %s", GLUSTERFS_INTERNAL_FOP_KEY, - loc->path); - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - - ret = dict_set_int32 (meta_dict, GF_META_LOCK_KEY, 1); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Trace dict_set failed"); - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - - ret = syncop_setxattr (from, loc, meta_dict, 0, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Trace syncop_setxattr metalock failed"); - - *fop_errno = -ret; - ret = -1; - goto out; - } else { - meta_locked = _gf_true; - } - } - - if (!conf->lock_migration_enabled) { - plock.l_type = F_WRLCK; - plock.l_start = 0; - plock.l_len = 0; - plock.l_whence = SEEK_SET; - - ret = syncop_lk (from, src_fd, F_SETLK, &plock, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: Failed to lock on %s", - loc->path, from->name); - *fop_errno = -ret; - ret = -1; - goto out; - } - - p_locked = _gf_true; - + goto metaunlock; + } } else { - - INIT_LIST_HEAD (&locklist.list); - - ret = syncop_getactivelk (from, loc, &locklist, NULL, NULL); - if (ret == 0) { - gf_log (this->name, GF_LOG_INFO, "No active locks on:%s" - , loc->path); - - } else if (ret > 0) { - - ret = syncop_setactivelk (to, loc, &locklist, NULL, - NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_LOCK_MIGRATION_FAILED, - "write lock failed on:%s", loc->path); - - *fop_errno = -ret; - ret = -1; - goto metaunlock; - } - } else { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_LOCK_MIGRATION_FAILED, - "getactivelk failed for file: %s", loc->path); - *fop_errno = -ret; - } - } - - - /* source would have both sticky bit and sgid bit set, reset it to 0, - and set the source permission on destination, if it was not set - prior to setting rebalance-modes in source */ - if (!src_ia_prot.sticky) - new_stbuf.ia_prot.sticky = 0; - - if (!src_ia_prot.sgid) - new_stbuf.ia_prot.sgid = 0; - - /* TODO: if the source actually had sticky bit, or sgid bit set, - we are not handling it */ - - ret = syncop_fsetattr (to, dst_fd, &new_stbuf, - (GF_SET_ATTR_UID | GF_SET_ATTR_GID | - GF_SET_ATTR_MODE), NULL, NULL, NULL, NULL); + gf_msg(this->name, GF_LOG_ERROR, -ret, + DHT_MSG_LOCK_MIGRATION_FAILED, + "getactivelk failed for file: %s", loc->path); + *fop_errno = -ret; + } + } + + /* source would have both sticky bit and sgid bit set, reset it to 0, + and set the source permission on destination, if it was not set + prior to setting rebalance-modes in source */ + if (!src_ia_prot.sticky) + new_stbuf.ia_prot.sticky = 0; + + if (!src_ia_prot.sgid) + new_stbuf.ia_prot.sgid = 0; + + /* TODO: if the source actually had sticky bit, or sgid bit set, + we are not handling it */ + + ret = syncop_fsetattr( + to, dst_fd, &new_stbuf, + (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL, + NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: failed to perform setattr on %s ", + loc->path, to->name); + *fop_errno = -ret; + ret = -1; + goto metaunlock; + } + + /* Because 'futimes' is not portable */ + ret = syncop_setattr(to, loc, &new_stbuf, + (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME), NULL, NULL, + NULL, NULL); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, + "%s: failed to perform setattr on %s ", loc->path, to->name); + *fop_errno = -ret; + } + + if (target_changed) { + dict_del(dict, GLUSTERFS_POSIXLK_COUNT); + ret = dict_set_str(dict, conf->link_xattr_name, to->name); if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: failed to perform setattr on %s ", - loc->path, to->name); - *fop_errno = -ret; + gf_log(this->name, GF_LOG_ERROR, + "failed to set xattr in dict for %s (linkto:%s)", loc->path, + to->name); + *fop_errno = ENOMEM; + ret = -1; + goto out; + } + + ret = syncop_setxattr(old_target, loc, dict, 0, NULL, NULL); + if (ret && -ret != ESTALE && -ret != ENOENT) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "failed to set xattr on %s in %s", loc->path, + old_target->name); + *fop_errno = -ret; + ret = -1; + goto out; + } else if (-ret == ESTALE || -ret == ENOENT) { + /* The failure ESTALE indicates that the linkto + * file on the hashed subvol might have been deleted. + * In this case will create a linkto file with new target + * as linkto xattr value*/ + linkto_fd = fd_create(loc->inode, DHT_REBALANCE_PID); + if (!linkto_fd) { + gf_msg(this->name, GF_LOG_ERROR, errno, + DHT_MSG_MIGRATE_FILE_FAILED, "%s: fd create failed", + loc->path); + *fop_errno = ENOMEM; ret = -1; - goto metaunlock; - } - - /* Because 'futimes' is not portable */ - ret = syncop_setattr (to, loc, &new_stbuf, - (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME), - NULL, NULL, NULL, NULL); - if (ret) { - gf_log (this->name, GF_LOG_WARNING, - "%s: failed to perform setattr on %s ", - loc->path, to->name); + goto out; + } + ret = syncop_create(old_target, loc, O_RDWR, DHT_LINKFILE_MODE, + linkto_fd, NULL, dict, NULL); + if (ret != 0 && -ret != EEXIST && -ret != ESTALE) { *fop_errno = -ret; - } - - if (target_changed) { - dict_del (dict, GLUSTERFS_POSIXLK_COUNT); - ret = dict_set_str (dict, conf->link_xattr_name, to->name); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "failed to set xattr in dict for %s (linkto:%s)", - loc->path, to->name); - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - - ret = syncop_setxattr (old_target, loc, dict, 0, NULL, NULL); - if (ret && -ret != ESTALE && -ret != ENOENT) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to set xattr on %s in %s", - loc->path, old_target->name); - *fop_errno = -ret; - ret = -1; - goto out; - } else if (-ret == ESTALE || -ret == ENOENT) { - /* The failure ESTALE indicates that the linkto - * file on the hashed subvol might have been deleted. - * In this case will create a linkto file with new target - * as linkto xattr value*/ - linkto_fd = fd_create (loc->inode, DHT_REBALANCE_PID); - if (!linkto_fd) { - gf_msg (this->name, GF_LOG_ERROR, errno, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: fd create failed", loc->path); - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - ret = syncop_create (old_target, loc, O_RDWR, - DHT_LINKFILE_MODE, linkto_fd, - NULL, dict, NULL); - if (ret != 0 && -ret != EEXIST && -ret != ESTALE) { - *fop_errno = -ret; - ret = -1; - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "failed to create linkto file on %s in %s", - loc->path, old_target->name); - goto out; - } else if (ret == 0) { - ret = syncop_fsetattr (old_target, linkto_fd, &stbuf, - (GF_SET_ATTR_UID | GF_SET_ATTR_GID), - NULL, NULL, NULL, NULL); - if (ret < 0) { - *fop_errno = -ret; - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "chown failed for %s on %s", - loc->path, old_target->name); - } - } + ret = -1; + gf_msg(this->name, GF_LOG_ERROR, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "failed to create linkto file on %s in %s", loc->path, + old_target->name); + goto out; + } else if (ret == 0) { + ret = syncop_fsetattr(old_target, linkto_fd, &stbuf, + (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL, + NULL, NULL, NULL); + if (ret < 0) { + *fop_errno = -ret; + gf_msg(this->name, GF_LOG_ERROR, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "chown failed for %s on %s", loc->path, + old_target->name); } - } - - clean_dst = _gf_false; - - /* Posix acls are not set on DHT linkto files as part of the initial - * initial xattrs set on the dst file, so these need - * to be set on the dst file after the linkto attrs are removed. - * TODO: Optimize this. - */ - if (xattr) { - dict_unref (xattr); - xattr = NULL; - } - - /* Set only the Posix ACLs this time */ - ret = syncop_getxattr (from, loc, &xattr, POSIX_ACL_ACCESS_XATTR, - NULL, NULL); + } + } + } + + clean_dst = _gf_false; + + /* Posix acls are not set on DHT linkto files as part of the initial + * initial xattrs set on the dst file, so these need + * to be set on the dst file after the linkto attrs are removed. + * TODO: Optimize this. + */ + if (xattr) { + dict_unref(xattr); + xattr = NULL; + } + + /* Set only the Posix ACLs this time */ + ret = syncop_getxattr(from, loc, &xattr, POSIX_ACL_ACCESS_XATTR, NULL, + NULL); + if (ret < 0) { + if ((-ret != ENODATA) && (-ret != ENOATTR)) { + gf_msg(this->name, GF_LOG_WARNING, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: failed to get xattr from %s", + loc->path, from->name); + *fop_errno = -ret; + } + } else { + ret = syncop_setxattr(to, loc, xattr, 0, NULL, NULL); if (ret < 0) { - if ((-ret != ENODATA) && (-ret != ENOATTR)) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: failed to get xattr from %s", - loc->path, from->name); - *fop_errno = -ret; - } + /* Potential problem here where Posix ACLs will + * not be set on the target file */ + + gf_msg(this->name, GF_LOG_WARNING, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: failed to set xattr on %s", + loc->path, to->name); + *fop_errno = -ret; + } + } + + /* store size of previous migrated file */ + if (defrag && defrag->tier_conf.is_tier) { + if (from != TIER_HASHED_SUBVOL) { + defrag->tier_conf.st_last_promoted_size = stbuf.ia_size; } else { - ret = syncop_setxattr (to, loc, xattr, 0, NULL, NULL); - if (ret < 0) { - /* Potential problem here where Posix ACLs will - * not be set on the target file */ - - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: failed to set xattr on %s", - loc->path, to->name); - *fop_errno = -ret; - } - } - - /* store size of previous migrated file */ - if (defrag && defrag->tier_conf.is_tier) { - if (from != TIER_HASHED_SUBVOL) { - defrag->tier_conf.st_last_promoted_size = stbuf.ia_size; - } else { - /* Don't delete the linkto file on the hashed subvol */ - delete_src_linkto = _gf_false; - defrag->tier_conf.st_last_demoted_size = stbuf.ia_size; - } - } - - /* The src file is being unlinked after this so we don't need - to clean it up */ - clean_src = _gf_false; - - /* Make the source as a linkfile first before deleting it */ - empty_iatt.ia_prot.sticky = 1; - ret = syncop_fsetattr (from, src_fd, &empty_iatt, - GF_SET_ATTR_MODE, NULL, NULL, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed:" - "%s: failed to perform setattr on %s ", - loc->path, from->name); - *fop_errno = -ret; - ret = -1; - goto metaunlock; - } - - /* Free up the data blocks on the source node, as the whole - file is migrated */ - ret = syncop_ftruncate (from, src_fd, 0, NULL, NULL, NULL, NULL); - if (ret) { - gf_log (this->name, GF_LOG_WARNING, - "%s: failed to perform truncate on %s (%s)", - loc->path, from->name, strerror (-ret)); - *fop_errno = -ret; - } - - /* remove the 'linkto' xattr from the destination */ - ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name, 0, NULL); + /* Don't delete the linkto file on the hashed subvol */ + delete_src_linkto = _gf_false; + defrag->tier_conf.st_last_demoted_size = stbuf.ia_size; + } + } + + /* The src file is being unlinked after this so we don't need + to clean it up */ + clean_src = _gf_false; + + /* Make the source as a linkfile first before deleting it */ + empty_iatt.ia_prot.sticky = 1; + ret = syncop_fsetattr(from, src_fd, &empty_iatt, GF_SET_ATTR_MODE, NULL, + NULL, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed:" + "%s: failed to perform setattr on %s ", + loc->path, from->name); + *fop_errno = -ret; + ret = -1; + goto metaunlock; + } + + /* Free up the data blocks on the source node, as the whole + file is migrated */ + ret = syncop_ftruncate(from, src_fd, 0, NULL, NULL, NULL, NULL); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, + "%s: failed to perform truncate on %s (%s)", loc->path, + from->name, strerror(-ret)); + *fop_errno = -ret; + } + + /* remove the 'linkto' xattr from the destination */ + ret = syncop_fremovexattr(to, dst_fd, conf->link_xattr_name, 0, NULL); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, + "%s: failed to perform removexattr on %s (%s)", loc->path, + to->name, strerror(-ret)); + *fop_errno = -ret; + } + + /* Do a stat and check the gfid before unlink */ + + /* + * Cached file changes its state from non-linkto to linkto file after + * migrating data. If lookup from any other mount-point is performed, + * converted-linkto-cached file will be treated as a stale and will be + * unlinked. But by this time, file is already migrated. So further + * failure because of ENOENT should not be treated as error + */ + + ret = syncop_stat(from, loc, &empty_iatt, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to do a stat on %s", loc->path, from->name); + + if (-ret != ENOENT) { + *fop_errno = -ret; + ret = -1; + goto metaunlock; + } + + rcvd_enoent_from_src = 1; + } + + if ((gf_uuid_compare(empty_iatt.ia_gfid, loc->gfid) == 0) && + (!rcvd_enoent_from_src) && delete_src_linkto) { + /* take out the source from namespace */ + ret = syncop_unlink(from, loc, NULL, NULL); if (ret) { - gf_log (this->name, GF_LOG_WARNING, - "%s: failed to perform removexattr on %s (%s)", - loc->path, to->name, strerror (-ret)); - *fop_errno = -ret; + gf_msg(this->name, GF_LOG_WARNING, -ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to perform unlink on %s", loc->path, from->name); + *fop_errno = -ret; + ret = -1; + goto metaunlock; } + } - /* Do a stat and check the gfid before unlink */ - - /* - * Cached file changes its state from non-linkto to linkto file after - * migrating data. If lookup from any other mount-point is performed, - * converted-linkto-cached file will be treated as a stale and will be - * unlinked. But by this time, file is already migrated. So further - * failure because of ENOENT should not be treated as error - */ + ret = syncop_lookup(this, loc, NULL, NULL, NULL, NULL); + if (ret) { + gf_msg_debug(this->name, -ret, + "%s: failed to lookup the file on subvolumes", loc->path); + *fop_errno = -ret; + } - ret = syncop_stat (from, loc, &empty_iatt, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to do a stat on %s", - loc->path, from->name); - - if (-ret != ENOENT) { - *fop_errno = -ret; - ret = -1; - goto metaunlock; - } - - rcvd_enoent_from_src = 1; - } + gf_msg(this->name, log_level, 0, DHT_MSG_MIGRATE_FILE_COMPLETE, + "completed migration of %s from subvolume %s to %s", loc->path, + from->name, to->name); - - if ((gf_uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0 ) && - (!rcvd_enoent_from_src) && delete_src_linkto) { - /* take out the source from namespace */ - ret = syncop_unlink (from, loc, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to perform unlink on %s", - loc->path, from->name); - *fop_errno = -ret; - ret = -1; - goto metaunlock; - } - } - - ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL); - if (ret) { - gf_msg_debug (this->name, -ret, - "%s: failed to lookup the file on subvolumes", - loc->path); - *fop_errno = -ret; - } - - gf_msg (this->name, log_level, 0, - DHT_MSG_MIGRATE_FILE_COMPLETE, - "completed migration of %s from subvolume %s to %s", - loc->path, from->name, to->name); - - ret = 0; + ret = 0; metaunlock: - if (conf->lock_migration_enabled && meta_locked) { + if (conf->lock_migration_enabled && meta_locked) { + dict_del(meta_dict, GF_META_LOCK_KEY); - dict_del (meta_dict, GF_META_LOCK_KEY); - - ret = dict_set_int32 (meta_dict, GF_META_UNLOCK_KEY, 1); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Trace dict_set failed"); - - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - - if (clean_dst == _gf_false) - ret = dict_set_int32 (meta_dict, "status", 1); - else - ret = dict_set_int32 (meta_dict, "status", 0); - - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "Trace dict_set failed"); - - *fop_errno = ENOMEM; - ret = -1; - goto out; - } - - ret = syncop_setxattr (from, loc, meta_dict, 0, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Trace syncop_setxattr meta unlock failed"); - - *fop_errno = -ret; - ret = -1; - goto out; - } - } - -out: - if (clean_src) { - /* Revert source mode and xattr changes*/ - lk_ret = __dht_migration_cleanup_src_file (this, loc, src_fd, - from, &src_ia_prot); - if (lk_ret) { - gf_msg (this->name, GF_LOG_WARNING, 0, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to cleanup source file on %s", - loc->path, from->name); - } - } + ret = dict_set_int32(meta_dict, GF_META_UNLOCK_KEY, 1); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Trace dict_set failed"); - /* reset the destination back to 0 */ - if (clean_dst) { - lk_ret = syncop_ftruncate (to, dst_fd, 0, NULL, NULL, - NULL, NULL); - if (lk_ret) { - gf_msg (this->name, GF_LOG_ERROR, -lk_ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed: " - "%s: failed to reset target size back to 0", - loc->path); - } + *fop_errno = ENOMEM; + ret = -1; + goto out; } - if (inodelk_locked) { - flock.l_type = F_UNLCK; + if (clean_dst == _gf_false) + ret = dict_set_int32(meta_dict, "status", 1); + else + ret = dict_set_int32(meta_dict, "status", 0); - lk_ret = syncop_inodelk (from, DHT_FILE_MIGRATE_DOMAIN, - &tmp_loc, F_SETLK, &flock, NULL, NULL); - if (lk_ret < 0) { - gf_msg (this->name, GF_LOG_WARNING, -lk_ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to unlock file on %s", - loc->path, from->name); - } - } + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "Trace dict_set failed"); - if (entrylk_locked) { - lk_ret = syncop_entrylk (hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, - &parent_loc, loc->name, ENTRYLK_UNLOCK, - ENTRYLK_UNLOCK, NULL, NULL); - if (lk_ret < 0) { - gf_msg (this->name, GF_LOG_WARNING, -lk_ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to unlock entrylk on %s", - loc->path, hashed_subvol->name); - } + *fop_errno = ENOMEM; + ret = -1; + goto out; } - if (p_locked) { - plock.l_type = F_UNLCK; - lk_ret = syncop_lk (from, src_fd, F_SETLK, &plock, NULL, NULL); + ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Trace syncop_setxattr meta unlock failed"); - if (lk_ret < 0) { - gf_msg (this->name, GF_LOG_WARNING, -lk_ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "%s: failed to unlock file on %s", - loc->path, from->name); - } + *fop_errno = -ret; + ret = -1; + goto out; } + } - if (!dht_is_tier_xlator (this)) { - lk_ret = syncop_removexattr (to, loc, - GF_PROTECT_FROM_EXTERNAL_WRITES, - NULL, NULL); - if (lk_ret && (lk_ret != -ENODATA) && (lk_ret != -ENOATTR)) { - gf_msg (this->name, GF_LOG_WARNING, -lk_ret, 0, - "%s: removexattr failed key %s", loc->path, - GF_PROTECT_FROM_EXTERNAL_WRITES); - } +out: + if (clean_src) { + /* Revert source mode and xattr changes*/ + lk_ret = __dht_migration_cleanup_src_file(this, loc, src_fd, from, + &src_ia_prot); + if (lk_ret) { + gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to cleanup source file on %s", loc->path, + from->name); + } + } + + /* reset the destination back to 0 */ + if (clean_dst) { + lk_ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL); + if (lk_ret) { + gf_msg(this->name, GF_LOG_ERROR, -lk_ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed: " + "%s: failed to reset target size back to 0", + loc->path); + } + } + + if (inodelk_locked) { + flock.l_type = F_UNLCK; + + lk_ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, + F_SETLK, &flock, NULL, NULL); + if (lk_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, -lk_ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to unlock file on %s", loc->path, from->name); + } + } + + if (entrylk_locked) { + lk_ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, + &parent_loc, loc->name, ENTRYLK_UNLOCK, + ENTRYLK_UNLOCK, NULL, NULL); + if (lk_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, -lk_ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to unlock entrylk on %s", loc->path, + hashed_subvol->name); + } + } + + if (p_locked) { + plock.l_type = F_UNLCK; + lk_ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL); + + if (lk_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, -lk_ret, + DHT_MSG_MIGRATE_FILE_FAILED, + "%s: failed to unlock file on %s", loc->path, from->name); + } + } + + if (!dht_is_tier_xlator(this)) { + lk_ret = syncop_removexattr(to, loc, GF_PROTECT_FROM_EXTERNAL_WRITES, + NULL, NULL); + if (lk_ret && (lk_ret != -ENODATA) && (lk_ret != -ENOATTR)) { + gf_msg(this->name, GF_LOG_WARNING, -lk_ret, 0, + "%s: removexattr failed key %s", loc->path, + GF_PROTECT_FROM_EXTERNAL_WRITES); } + } - if (dict) - dict_unref (dict); + if (dict) + dict_unref(dict); - if (xattr) - dict_unref (xattr); - if (xattr_rsp) - dict_unref (xattr_rsp); + if (xattr) + dict_unref(xattr); + if (xattr_rsp) + dict_unref(xattr_rsp); - if (dst_fd) - syncop_close (dst_fd); - if (src_fd) - syncop_close (src_fd); - if (linkto_fd) - syncop_close (linkto_fd); + if (dst_fd) + syncop_close(dst_fd); + if (src_fd) + syncop_close(src_fd); + if (linkto_fd) + syncop_close(linkto_fd); - loc_wipe (&tmp_loc); - loc_wipe (&parent_loc); + loc_wipe(&tmp_loc); + loc_wipe(&parent_loc); - return ret; + return ret; } static int -rebalance_task (void *data) +rebalance_task(void *data) { - int ret = -1; - dht_local_t *local = NULL; - call_frame_t *frame = NULL; - int fop_errno = 0; + int ret = -1; + dht_local_t *local = NULL; + call_frame_t *frame = NULL; + int fop_errno = 0; - frame = data; + frame = data; - local = frame->local; + local = frame->local; - /* This function is 'synchrounous', hence if it returns, - we are done with the task */ - ret = dht_migrate_file (THIS, &local->loc, local->rebalance.from_subvol, - local->rebalance.target_node, local->flags, - &fop_errno); + /* This function is 'synchrounous', hence if it returns, + we are done with the task */ + ret = dht_migrate_file(THIS, &local->loc, local->rebalance.from_subvol, + local->rebalance.target_node, local->flags, + &fop_errno); - return ret; + return ret; } static int -rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data) +rebalance_task_completion(int op_ret, call_frame_t *sync_frame, void *data) { - int32_t op_errno = EINVAL; - - if (op_ret == -1) { - /* Failure of migration process, mostly due to write process. - as we can't preserve the exact errno, lets say there was - no space to migrate-data - */ - op_errno = ENOSPC; - } else if (op_ret == 1) { - /* migration didn't happen, but is not a failure, let the user - understand that he doesn't have permission to migrate the - file. - */ - op_ret = -1; - op_errno = EPERM; - } else if (op_ret != 0) { - op_errno = -op_ret; - op_ret = -1; - } + int32_t op_errno = EINVAL; - DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL); - return 0; + if (op_ret == -1) { + /* Failure of migration process, mostly due to write process. + as we can't preserve the exact errno, lets say there was + no space to migrate-data + */ + op_errno = ENOSPC; + } else if (op_ret == 1) { + /* migration didn't happen, but is not a failure, let the user + understand that he doesn't have permission to migrate the + file. + */ + op_ret = -1; + op_errno = EPERM; + } else if (op_ret != 0) { + op_errno = -op_ret; + op_ret = -1; + } + + DHT_STACK_UNWIND(setxattr, sync_frame, op_ret, op_errno, NULL); + return 0; } int -dht_start_rebalance_task (xlator_t *this, call_frame_t *frame) +dht_start_rebalance_task(xlator_t *this, call_frame_t *frame) { - int ret = -1; + int ret = -1; - ret = synctask_new (this->ctx->env, rebalance_task, - rebalance_task_completion, - frame, frame); - return ret; + ret = synctask_new(this->ctx->env, rebalance_task, + rebalance_task_completion, frame, frame); + return ret; } - int -gf_listener_stop (xlator_t *this) +gf_listener_stop(xlator_t *this) { - glusterfs_ctx_t *ctx = NULL; - cmd_args_t *cmd_args = NULL; - int ret = 0; - - ctx = this->ctx; - GF_ASSERT (ctx); - cmd_args = &ctx->cmd_args; - if (cmd_args->sock_file) { - ret = sys_unlink (cmd_args->sock_file); - if (ret && (ENOENT == errno)) { - ret = 0; - } - } - - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, errno, - DHT_MSG_SOCKET_ERROR, - "Failed to unlink listener " - "socket %s", cmd_args->sock_file); - } - return ret; + glusterfs_ctx_t *ctx = NULL; + cmd_args_t *cmd_args = NULL; + int ret = 0; + + ctx = this->ctx; + GF_ASSERT(ctx); + cmd_args = &ctx->cmd_args; + if (cmd_args->sock_file) { + ret = sys_unlink(cmd_args->sock_file); + if (ret && (ENOENT == errno)) { + ret = 0; + } + } + + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, DHT_MSG_SOCKET_ERROR, + "Failed to unlink listener " + "socket %s", + cmd_args->sock_file); + } + return ret; } void -dht_build_root_inode (xlator_t *this, inode_t **inode) +dht_build_root_inode(xlator_t *this, inode_t **inode) { - inode_table_t *itable = NULL; - uuid_t root_gfid = {0, }; + inode_table_t *itable = NULL; + uuid_t root_gfid = { + 0, + }; - itable = inode_table_new (0, this); - if (!itable) - return; + itable = inode_table_new(0, this); + if (!itable) + return; - root_gfid[15] = 1; - *inode = inode_find (itable, root_gfid); + root_gfid[15] = 1; + *inode = inode_find(itable, root_gfid); } void -dht_build_root_loc (inode_t *inode, loc_t *loc) +dht_build_root_loc(inode_t *inode, loc_t *loc) { - loc->path = "/"; - loc->inode = inode; - loc->inode->ia_type = IA_IFDIR; - memset (loc->gfid, 0, 16); - loc->gfid[15] = 1; + loc->path = "/"; + loc->inode = inode; + loc->inode->ia_type = IA_IFDIR; + memset(loc->gfid, 0, 16); + loc->gfid[15] = 1; } /* return values: 1 -> error, bug ignore and continue 0 -> proceed -1 -> error, handle it */ int32_t -gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag) +gf_defrag_handle_migrate_error(int32_t op_errno, gf_defrag_info_t *defrag) { - int ret = 0; - /* if errno is not ENOTCONN, we can still continue - with rebalance process */ - if (op_errno != ENOTCONN) { - ret = 1; - goto out; - } - - if (op_errno == ENOTCONN) { - /* Most probably mount point went missing (mostly due - to a brick down), say rebalance failure to user, - let him restart it if everything is fine */ - defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; - ret = -1; - goto out; - } + int ret = 0; + /* if errno is not ENOTCONN, we can still continue + with rebalance process */ + if (op_errno != ENOTCONN) { + ret = 1; + goto out; + } + + if (op_errno == ENOTCONN) { + /* Most probably mount point went missing (mostly due + to a brick down), say rebalance failure to user, + let him restart it if everything is fine */ + defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; + ret = -1; + goto out; + } out: - return ret; + return ret; } static gf_boolean_t -gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size) +gf_defrag_pattern_match(gf_defrag_info_t *defrag, char *name, uint64_t size) { - gf_defrag_pattern_list_t *trav = NULL; - gf_boolean_t match = _gf_false; - gf_boolean_t ret = _gf_false; + gf_defrag_pattern_list_t *trav = NULL; + gf_boolean_t match = _gf_false; + gf_boolean_t ret = _gf_false; - GF_VALIDATE_OR_GOTO ("dht", defrag, out); + GF_VALIDATE_OR_GOTO("dht", defrag, out); - trav = defrag->defrag_pattern; - while (trav) { - if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) { - match = _gf_true; - break; - } - trav = trav->next; + trav = defrag->defrag_pattern; + while (trav) { + if (!fnmatch(trav->path_pattern, name, FNM_NOESCAPE)) { + match = _gf_true; + break; } + trav = trav->next; + } - if ((match == _gf_true) && (size >= trav->size)) - ret = _gf_true; + if ((match == _gf_true) && (size >= trav->size)) + ret = _gf_true; - out: - return ret; +out: + return ret; } -int dht_dfreaddirp_done (dht_dfoffset_ctx_t *offset_var, int cnt) { - - int i; - int result = 1; +int +dht_dfreaddirp_done(dht_dfoffset_ctx_t *offset_var, int cnt) +{ + int i; + int result = 1; - for (i = 0; i < cnt; i++) { - if (offset_var[i].readdir_done == 0) { - result = 0; - break; - } + for (i = 0; i < cnt; i++) { + if (offset_var[i].readdir_done == 0) { + result = 0; + break; } - return result; + } + return result; } -int static -gf_defrag_ctx_subvols_init (dht_dfoffset_ctx_t *offset_var, xlator_t *this) { - - int i; - dht_conf_t *conf = NULL; +int static gf_defrag_ctx_subvols_init(dht_dfoffset_ctx_t *offset_var, + xlator_t *this) +{ + int i; + dht_conf_t *conf = NULL; - conf = this->private; + conf = this->private; - if (!conf) - return -1; + if (!conf) + return -1; - for (i = 0; i < conf->local_subvols_cnt; i++) { - offset_var[i].this = conf->local_subvols[i]; - offset_var[i].offset = (off_t) 0; - offset_var[i].readdir_done = 0; - } + for (i = 0; i < conf->local_subvols_cnt; i++) { + offset_var[i].this = conf->local_subvols[i]; + offset_var[i].offset = (off_t)0; + offset_var[i].readdir_done = 0; + } - return 0; + return 0; } - static int -dht_get_first_non_null_index (subvol_nodeuuids_info_t *entry) +dht_get_first_non_null_index(subvol_nodeuuids_info_t *entry) { - int i = 0; - int index = 0; + int i = 0; + int index = 0; - for (i = 0; i < entry->count; i++) { - if (!gf_uuid_is_null (entry->elements[i].uuid)) { - index = i; - goto out; - } + for (i = 0; i < entry->count; i++) { + if (!gf_uuid_is_null(entry->elements[i].uuid)) { + index = i; + goto out; } + } - if (i == entry->count) { - index = -1; - } + if (i == entry->count) { + index = -1; + } out: - return index; + return index; } - /* Return value * 0 : this node does not migrate the file * 1 : this node migrates the file @@ -2688,1022 +2595,986 @@ out: */ int -gf_defrag_should_i_migrate (xlator_t *this, int local_subvol_index, uuid_t gfid) +gf_defrag_should_i_migrate(xlator_t *this, int local_subvol_index, uuid_t gfid) { - int ret = 0; - int i = local_subvol_index; - char *str = NULL; - uint32_t hashval = 0; - int32_t index = 0; - dht_conf_t *conf = NULL; - char buf[UUID_CANONICAL_FORM_LEN + 1] = {0, }; - subvol_nodeuuids_info_t *entry = NULL; - - - conf = this->private; - - /* Pure distribute. A subvol in this case - will be handled by only one node */ - - entry = &(conf->local_nodeuuids[i]); - if (entry->count == 1) { - return 1; - } - - str = uuid_utoa_r (gfid, buf); - ret = dht_hash_compute (this, 0, str, &hashval); - if (ret == 0) { - index = (hashval % entry->count); - if (entry->elements[index].info - == REBAL_NODEUUID_MINE) { - /* Index matches this node's nodeuuid.*/ - ret = 1; - goto out; - } - - /* Brick down - some other node has to migrate these files*/ - if (gf_uuid_is_null (entry->elements[index].uuid)) { - /* Fall back to the first non-null index */ - index = dht_get_first_non_null_index (entry); - - if (index == -1) { - /* None of the bricks in the subvol are up. - * CHILD_DOWN will kill the process soon */ + int ret = 0; + int i = local_subvol_index; + char *str = NULL; + uint32_t hashval = 0; + int32_t index = 0; + dht_conf_t *conf = NULL; + char buf[UUID_CANONICAL_FORM_LEN + 1] = { + 0, + }; + subvol_nodeuuids_info_t *entry = NULL; + + conf = this->private; + + /* Pure distribute. A subvol in this case + will be handled by only one node */ + + entry = &(conf->local_nodeuuids[i]); + if (entry->count == 1) { + return 1; + } + + str = uuid_utoa_r(gfid, buf); + ret = dht_hash_compute(this, 0, str, &hashval); + if (ret == 0) { + index = (hashval % entry->count); + if (entry->elements[index].info == REBAL_NODEUUID_MINE) { + /* Index matches this node's nodeuuid.*/ + ret = 1; + goto out; + } + + /* Brick down - some other node has to migrate these files*/ + if (gf_uuid_is_null(entry->elements[index].uuid)) { + /* Fall back to the first non-null index */ + index = dht_get_first_non_null_index(entry); + + if (index == -1) { + /* None of the bricks in the subvol are up. + * CHILD_DOWN will kill the process soon */ - return 0; - } + return 0; + } - if (entry->elements[index].info == REBAL_NODEUUID_MINE) { - /* Index matches this node's nodeuuid.*/ - ret = 1; - goto out; - } - } + if (entry->elements[index].info == REBAL_NODEUUID_MINE) { + /* Index matches this node's nodeuuid.*/ + ret = 1; + goto out; + } } + } out: - return ret; + return ret; } - int -gf_defrag_migrate_single_file (void *opaque) +gf_defrag_migrate_single_file(void *opaque) { - xlator_t *this = NULL; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; - int ret = 0; - gf_dirent_t *entry = NULL; - struct timeval start = {0,}; - loc_t entry_loc = {0,}; - loc_t *loc = NULL; - struct iatt iatt = {0,}; - dict_t *migrate_data = NULL; - struct timeval end = {0,}; - double elapsed = {0,}; - struct dht_container *rebal_entry = NULL; - inode_t *inode = NULL; - xlator_t *hashed_subvol = NULL; - xlator_t *cached_subvol = NULL; - call_frame_t *statfs_frame = NULL; - xlator_t *old_THIS = NULL; - data_t *tmp = NULL; - int fop_errno = 0; - gf_dht_migrate_data_type_t rebal_type = GF_DHT_MIGRATE_DATA; - char value[MAX_REBAL_TYPE_SIZE] = {0,}; - struct iatt *iatt_ptr = NULL; - gf_boolean_t update_skippedcount = _gf_true; - int i = 0; - - rebal_entry = (struct dht_container *)opaque; - if (!rebal_entry) { - gf_log ("DHT", GF_LOG_ERROR, "rebal_entry is NULL"); - ret = -1; - goto out; - } + xlator_t *this = NULL; + dht_conf_t *conf = NULL; + gf_defrag_info_t *defrag = NULL; + int ret = 0; + gf_dirent_t *entry = NULL; + struct timeval start = { + 0, + }; + loc_t entry_loc = { + 0, + }; + loc_t *loc = NULL; + struct iatt iatt = { + 0, + }; + dict_t *migrate_data = NULL; + struct timeval end = { + 0, + }; + double elapsed = { + 0, + }; + struct dht_container *rebal_entry = NULL; + inode_t *inode = NULL; + xlator_t *hashed_subvol = NULL; + xlator_t *cached_subvol = NULL; + call_frame_t *statfs_frame = NULL; + xlator_t *old_THIS = NULL; + data_t *tmp = NULL; + int fop_errno = 0; + gf_dht_migrate_data_type_t rebal_type = GF_DHT_MIGRATE_DATA; + char value[MAX_REBAL_TYPE_SIZE] = { + 0, + }; + struct iatt *iatt_ptr = NULL; + gf_boolean_t update_skippedcount = _gf_true; + int i = 0; + + rebal_entry = (struct dht_container *)opaque; + if (!rebal_entry) { + gf_log("DHT", GF_LOG_ERROR, "rebal_entry is NULL"); + ret = -1; + goto out; + } - this = rebal_entry->this; + this = rebal_entry->this; - conf = this->private; + conf = this->private; - defrag = conf->defrag; + defrag = conf->defrag; - loc = rebal_entry->parent_loc; + loc = rebal_entry->parent_loc; - migrate_data = rebal_entry->migrate_data; + migrate_data = rebal_entry->migrate_data; - entry = rebal_entry->df_entry; - iatt_ptr = &entry->d_stat; + entry = rebal_entry->df_entry; + iatt_ptr = &entry->d_stat; - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { - ret = -1; - goto out; - } - - if (defrag->stats == _gf_true) { - gettimeofday (&start, NULL); - } - - if (defrag->defrag_pattern && - (gf_defrag_pattern_match (defrag, entry->d_name, - entry->d_stat.ia_size) == _gf_false)) { - gf_log (this->name, GF_LOG_ERROR, "pattern_match failed"); - goto out; - } - - memset (&entry_loc, 0, sizeof (entry_loc)); - - ret = dht_build_child_loc (this, &entry_loc, loc, entry->d_name); - if (ret) { - LOCK (&defrag->lock); - { - defrag->total_failures += 1; - } - UNLOCK (&defrag->lock); + if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { + ret = -1; + goto out; + } - ret = 0; + if (defrag->stats == _gf_true) { + gettimeofday(&start, NULL); + } - gf_log (this->name, GF_LOG_ERROR, "Child loc build failed"); + if (defrag->defrag_pattern && + (gf_defrag_pattern_match(defrag, entry->d_name, + entry->d_stat.ia_size) == _gf_false)) { + gf_log(this->name, GF_LOG_ERROR, "pattern_match failed"); + goto out; + } - goto out; - } + memset(&entry_loc, 0, sizeof(entry_loc)); - if (!gf_defrag_should_i_migrate (this, rebal_entry->local_subvol_index, - entry->d_stat.ia_gfid)) { - gf_msg_debug (this->name, 0, "Don't migrate %s ", - entry_loc.path); - goto out; + ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name); + if (ret) { + LOCK(&defrag->lock); + { + defrag->total_failures += 1; } + UNLOCK(&defrag->lock); - gf_uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid); + ret = 0; - gf_uuid_copy (entry_loc.pargfid, loc->gfid); + gf_log(this->name, GF_LOG_ERROR, "Child loc build failed"); - ret = syncop_lookup (this, &entry_loc, &iatt, NULL, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_MIGRATE_FILE_FAILED, - "Migrate file failed: %s lookup failed", - entry_loc.path); - - /* Increase failure count only for remove-brick op, so that - * user is warned to check the removed-brick for any files left - * unmigrated - */ - if (conf->decommission_subvols_cnt) { - LOCK (&defrag->lock); - { - defrag->total_failures += 1; - } - UNLOCK (&defrag->lock); - } + goto out; + } - ret = 0; - goto out; - } + if (!gf_defrag_should_i_migrate(this, rebal_entry->local_subvol_index, + entry->d_stat.ia_gfid)) { + gf_msg_debug(this->name, 0, "Don't migrate %s ", entry_loc.path); + goto out; + } - iatt_ptr = &iatt; + gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid); - hashed_subvol = dht_subvol_get_hashed (this, &entry_loc); - if (!hashed_subvol) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_HASHED_SUBVOL_GET_FAILED, - "Failed to get hashed subvol for %s", - entry_loc.path); - ret = 0; - goto out; - } - - cached_subvol = dht_subvol_get_cached (this, entry_loc.inode); - if (!cached_subvol) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_CACHED_SUBVOL_GET_FAILED, - "Failed to get cached subvol for %s", - entry_loc.path); - - ret = 0; - goto out; - } + gf_uuid_copy(entry_loc.pargfid, loc->gfid); - if (hashed_subvol == cached_subvol) { - ret = 0; - goto out; - } + ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED, + "Migrate file failed: %s lookup failed", entry_loc.path); - inode = inode_link (entry_loc.inode, entry_loc.parent, entry->d_name, &iatt); - inode_unref (entry_loc.inode); - /* use the inode returned by inode_link */ - entry_loc.inode = inode; - - old_THIS = THIS; - THIS = this; - statfs_frame = create_frame (this, this->ctx->pool); - if (!statfs_frame) { - gf_msg (this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM, - "Insufficient memory. Frame creation failed"); - ret = -1; - goto out; + /* Increase failure count only for remove-brick op, so that + * user is warned to check the removed-brick for any files left + * unmigrated + */ + if (conf->decommission_subvols_cnt) { + LOCK(&defrag->lock); + { + defrag->total_failures += 1; + } + UNLOCK(&defrag->lock); } - /* async statfs information for honoring min-free-disk */ - dht_get_du_info (statfs_frame, this, loc); - THIS = old_THIS; + ret = 0; + goto out; + } - tmp = dict_get (migrate_data, GF_XATTR_FILE_MIGRATE_KEY); - if (tmp) { - memcpy (value, tmp->data, tmp->len); - if (strcmp (value, "force") == 0) - rebal_type = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS; + iatt_ptr = &iatt; - if (conf->decommission_in_progress) - rebal_type = GF_DHT_MIGRATE_HARDLINK; - } + hashed_subvol = dht_subvol_get_hashed(this, &entry_loc); + if (!hashed_subvol) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED, + "Failed to get hashed subvol for %s", entry_loc.path); + ret = 0; + goto out; + } - ret = dht_migrate_file (this, &entry_loc, cached_subvol, - hashed_subvol, rebal_type, &fop_errno); - if (ret == 1) { - if (fop_errno == ENOSPC) { - gf_msg_debug (this->name, 0, "migrate-data skipped for" - " %s due to space constraints", - entry_loc.path); - - /* For remove-brick case if the source is not one of the - * removed-brick, do not mark the error as failure */ - if (conf->decommission_subvols_cnt) { - for (i = 0; i < conf->subvolume_cnt; i++) { - if (conf->decommissioned_bricks[i] == cached_subvol) { - LOCK (&defrag->lock); - { - defrag->total_failures += 1; - update_skippedcount = _gf_false; - } - UNLOCK (&defrag->lock); - - break; - } - } - } + cached_subvol = dht_subvol_get_cached(this, entry_loc.inode); + if (!cached_subvol) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CACHED_SUBVOL_GET_FAILED, + "Failed to get cached subvol for %s", entry_loc.path); - if (update_skippedcount) { - LOCK (&defrag->lock); - { - defrag->skipped += 1; - } - UNLOCK (&defrag->lock); - - gf_msg (this->name, GF_LOG_INFO, 0, - DHT_MSG_MIGRATE_FILE_SKIPPED, - "File migration skipped for %s.", - entry_loc.path); - } + ret = 0; + goto out; + } - } else if (fop_errno == ENOTSUP) { - gf_msg_debug (this->name, 0, "migrate-data skipped for" - " hardlink %s ", entry_loc.path); - LOCK (&defrag->lock); + if (hashed_subvol == cached_subvol) { + ret = 0; + goto out; + } + + inode = inode_link(entry_loc.inode, entry_loc.parent, entry->d_name, &iatt); + inode_unref(entry_loc.inode); + /* use the inode returned by inode_link */ + entry_loc.inode = inode; + + old_THIS = THIS; + THIS = this; + statfs_frame = create_frame(this, this->ctx->pool); + if (!statfs_frame) { + gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM, + "Insufficient memory. Frame creation failed"); + ret = -1; + goto out; + } + + /* async statfs information for honoring min-free-disk */ + dht_get_du_info(statfs_frame, this, loc); + THIS = old_THIS; + + tmp = dict_get(migrate_data, GF_XATTR_FILE_MIGRATE_KEY); + if (tmp) { + memcpy(value, tmp->data, tmp->len); + if (strcmp(value, "force") == 0) + rebal_type = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS; + + if (conf->decommission_in_progress) + rebal_type = GF_DHT_MIGRATE_HARDLINK; + } + + ret = dht_migrate_file(this, &entry_loc, cached_subvol, hashed_subvol, + rebal_type, &fop_errno); + if (ret == 1) { + if (fop_errno == ENOSPC) { + gf_msg_debug(this->name, 0, + "migrate-data skipped for" + " %s due to space constraints", + entry_loc.path); + + /* For remove-brick case if the source is not one of the + * removed-brick, do not mark the error as failure */ + if (conf->decommission_subvols_cnt) { + for (i = 0; i < conf->subvolume_cnt; i++) { + if (conf->decommissioned_bricks[i] == cached_subvol) { + LOCK(&defrag->lock); { - defrag->skipped += 1; + defrag->total_failures += 1; + update_skippedcount = _gf_false; } - UNLOCK (&defrag->lock); + UNLOCK(&defrag->lock); - gf_msg (this->name, GF_LOG_INFO, 0, - DHT_MSG_MIGRATE_FILE_SKIPPED, - "File migration skipped for %s.", - entry_loc.path); + break; + } } + } - ret = 0; - goto out; - } else if (ret < 0) { - if (fop_errno != EEXIST) { - gf_msg (this->name, GF_LOG_ERROR, fop_errno, - DHT_MSG_MIGRATE_FILE_FAILED, - "migrate-data failed for %s", entry_loc.path); - - LOCK (&defrag->lock); - { - defrag->total_failures += 1; - } - UNLOCK (&defrag->lock); - + if (update_skippedcount) { + LOCK(&defrag->lock); + { + defrag->skipped += 1; } + UNLOCK(&defrag->lock); - ret = gf_defrag_handle_migrate_error (fop_errno, defrag); + gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED, + "File migration skipped for %s.", entry_loc.path); + } - if (!ret) { - gf_msg(this->name, GF_LOG_ERROR, fop_errno, - DHT_MSG_MIGRATE_FILE_FAILED, - "migrate-data on %s failed:", entry_loc.path); - } else if (ret == 1) { - ret = 0; - } + } else if (fop_errno == ENOTSUP) { + gf_msg_debug(this->name, 0, + "migrate-data skipped for" + " hardlink %s ", + entry_loc.path); + LOCK(&defrag->lock); + { + defrag->skipped += 1; + } + UNLOCK(&defrag->lock); - goto out; + gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED, + "File migration skipped for %s.", entry_loc.path); } - LOCK (&defrag->lock); - { - defrag->total_files += 1; - defrag->total_data += iatt.ia_size; - } - UNLOCK (&defrag->lock); - - if (defrag->stats == _gf_true) { - gettimeofday (&end, NULL); - elapsed = (end.tv_sec - start.tv_sec) * 1e6 + - (end.tv_usec - start.tv_usec); - gf_log (this->name, GF_LOG_INFO, "Migration of " - "file:%s size:%"PRIu64" bytes took %.2f" - "secs and ret: %d", entry_loc.name, - iatt.ia_size, elapsed/1e6, ret); + ret = 0; + goto out; + } else if (ret < 0) { + if (fop_errno != EEXIST) { + gf_msg(this->name, GF_LOG_ERROR, fop_errno, + DHT_MSG_MIGRATE_FILE_FAILED, "migrate-data failed for %s", + entry_loc.path); + + LOCK(&defrag->lock); + { + defrag->total_failures += 1; + } + UNLOCK(&defrag->lock); } + ret = gf_defrag_handle_migrate_error(fop_errno, defrag); + + if (!ret) { + gf_msg(this->name, GF_LOG_ERROR, fop_errno, + DHT_MSG_MIGRATE_FILE_FAILED, + "migrate-data on %s failed:", entry_loc.path); + } else if (ret == 1) { + ret = 0; + } + + goto out; + } + + LOCK(&defrag->lock); + { + defrag->total_files += 1; + defrag->total_data += iatt.ia_size; + } + UNLOCK(&defrag->lock); + + if (defrag->stats == _gf_true) { + gettimeofday(&end, NULL); + elapsed = (end.tv_sec - start.tv_sec) * 1e6 + + (end.tv_usec - start.tv_usec); + gf_log(this->name, GF_LOG_INFO, + "Migration of " + "file:%s size:%" PRIu64 + " bytes took %.2f" + "secs and ret: %d", + entry_loc.name, iatt.ia_size, elapsed / 1e6, ret); + } + out: - if (statfs_frame) { - STACK_DESTROY (statfs_frame->root); - } + if (statfs_frame) { + STACK_DESTROY(statfs_frame->root); + } - if (iatt_ptr) { - LOCK (&defrag->lock); - { - defrag->size_processed += iatt_ptr->ia_size; - } - UNLOCK (&defrag->lock); + if (iatt_ptr) { + LOCK(&defrag->lock); + { + defrag->size_processed += iatt_ptr->ia_size; } - loc_wipe (&entry_loc); - - return ret; + UNLOCK(&defrag->lock); + } + loc_wipe(&entry_loc); + return ret; } void * -gf_defrag_task (void *opaque) +gf_defrag_task(void *opaque) { - struct list_head *q_head = NULL; - struct dht_container *iterator = NULL; - gf_defrag_info_t *defrag = NULL; - int ret = 0; - pid_t pid = GF_CLIENT_PID_DEFRAG; - - defrag = (gf_defrag_info_t *)opaque; - if (!defrag) { - gf_msg ("dht", GF_LOG_ERROR, 0, 0, "defrag is NULL"); - goto out; + struct list_head *q_head = NULL; + struct dht_container *iterator = NULL; + gf_defrag_info_t *defrag = NULL; + int ret = 0; + pid_t pid = GF_CLIENT_PID_DEFRAG; + + defrag = (gf_defrag_info_t *)opaque; + if (!defrag) { + gf_msg("dht", GF_LOG_ERROR, 0, 0, "defrag is NULL"); + goto out; + } + + syncopctx_setfspid(&pid); + + q_head = &(defrag->queue[0].list); + + /* The following while loop will dequeue one entry from the defrag->queue + under lock. We will update the defrag->global_error only when there + is an error which is critical to stop the rebalance process. The stop + message will be intimated to other migrator threads by setting the + defrag->defrag_status to GF_DEFRAG_STATUS_FAILED. + + In defrag->queue, a low watermark (MIN_MIGRATE_QUEUE_COUNT) is + maintained so that crawler does not starve the file migration + workers and a high watermark (MAX_MIGRATE_QUEUE_COUNT) so that + crawler does not go far ahead in filling up the queue. + */ + + while (_gf_true) { + if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { + pthread_cond_broadcast(&defrag->rebalance_crawler_alarm); + pthread_cond_broadcast(&defrag->parallel_migration_cond); + goto out; } - syncopctx_setfspid (&pid); + pthread_mutex_lock(&defrag->dfq_mutex); + { + /*Throttle down: + If the reconfigured count is less than current thread + count, then the current thread will sleep */ + + /*TODO: Need to refactor the following block to work + *under defrag->lock. For now access + * defrag->current_thread_count and rthcount under + * dfq_mutex lock */ + while (!defrag->crawl_done && (defrag->recon_thread_count < + defrag->current_thread_count)) { + defrag->current_thread_count--; + gf_msg_debug("DHT", 0, + "Thread sleeping. " + "current thread count: %d", + defrag->current_thread_count); + + pthread_cond_wait(&defrag->df_wakeup_thread, + &defrag->dfq_mutex); + + defrag->current_thread_count++; + gf_msg_debug("DHT", 0, + "Thread wokeup. " + "current thread count: %d", + defrag->current_thread_count); + } + + if (defrag->q_entry_count) { + iterator = list_entry(q_head->next, typeof(*iterator), list); + + gf_msg_debug("DHT", 0, + "picking entry " + "%s", + iterator->df_entry->d_name); + + list_del_init(&(iterator->list)); + + defrag->q_entry_count--; + + if ((defrag->q_entry_count < MIN_MIGRATE_QUEUE_COUNT) && + defrag->wakeup_crawler) { + pthread_cond_broadcast(&defrag->rebalance_crawler_alarm); + } + pthread_mutex_unlock(&defrag->dfq_mutex); + ret = gf_defrag_migrate_single_file((void *)iterator); - q_head = &(defrag->queue[0].list); + /*Critical errors: ENOTCONN and ENOSPACE*/ + if (ret) { + dht_set_global_defrag_error(defrag, ret); - /* The following while loop will dequeue one entry from the defrag->queue - under lock. We will update the defrag->global_error only when there - is an error which is critical to stop the rebalance process. The stop - message will be intimated to other migrator threads by setting the - defrag->defrag_status to GF_DEFRAG_STATUS_FAILED. + defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; - In defrag->queue, a low watermark (MIN_MIGRATE_QUEUE_COUNT) is - maintained so that crawler does not starve the file migration - workers and a high watermark (MAX_MIGRATE_QUEUE_COUNT) so that - crawler does not go far ahead in filling up the queue. - */ + pthread_cond_broadcast(&defrag->rebalance_crawler_alarm); - while (_gf_true) { + pthread_cond_broadcast(&defrag->parallel_migration_cond); - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { - pthread_cond_broadcast ( - &defrag->rebalance_crawler_alarm); - pthread_cond_broadcast ( - &defrag->parallel_migration_cond); - goto out; + goto out; } - pthread_mutex_lock (&defrag->dfq_mutex); - { - - /*Throttle down: - If the reconfigured count is less than current thread - count, then the current thread will sleep */ - - /*TODO: Need to refactor the following block to work - *under defrag->lock. For now access - * defrag->current_thread_count and rthcount under - * dfq_mutex lock */ - while (!defrag->crawl_done && - (defrag->recon_thread_count < - defrag->current_thread_count)) { - defrag->current_thread_count--; - gf_msg_debug ("DHT", 0, "Thread sleeping. " - "current thread count: %d", - defrag->current_thread_count); - - pthread_cond_wait ( - &defrag->df_wakeup_thread, - &defrag->dfq_mutex); - - defrag->current_thread_count++; - gf_msg_debug ("DHT", 0, "Thread wokeup. " - "current thread count: %d", - defrag->current_thread_count); - - } + gf_defrag_free_container(iterator); + + continue; + } else { + /* defrag->crawl_done flag is set means crawling + file system is done and hence a list_empty when + the above flag is set indicates there are no more + entries to be added to the queue and rebalance is + finished */ + + if (!defrag->crawl_done) { + defrag->current_thread_count--; + gf_msg_debug("DHT", 0, + "Thread " + "sleeping while waiting " + "for migration entries. " + "current thread count:%d", + defrag->current_thread_count); + + pthread_cond_wait(&defrag->parallel_migration_cond, + &defrag->dfq_mutex); + } - if (defrag->q_entry_count) { - iterator = list_entry (q_head->next, - typeof(*iterator), list); - - gf_msg_debug ("DHT", 0, "picking entry " - "%s", iterator->df_entry->d_name); - - list_del_init (&(iterator->list)); - - defrag->q_entry_count--; - - if ((defrag->q_entry_count < - MIN_MIGRATE_QUEUE_COUNT) && - defrag->wakeup_crawler) { - pthread_cond_broadcast ( - &defrag->rebalance_crawler_alarm); - } - pthread_mutex_unlock (&defrag->dfq_mutex); - ret = gf_defrag_migrate_single_file - ((void *)iterator); - - /*Critical errors: ENOTCONN and ENOSPACE*/ - if (ret) { - dht_set_global_defrag_error - (defrag, ret); - - defrag->defrag_status = - GF_DEFRAG_STATUS_FAILED; - - pthread_cond_broadcast ( - &defrag->rebalance_crawler_alarm); - - pthread_cond_broadcast ( - &defrag->parallel_migration_cond); - - goto out; - } - - gf_defrag_free_container (iterator); - - continue; - } else { - - /* defrag->crawl_done flag is set means crawling - file system is done and hence a list_empty when - the above flag is set indicates there are no more - entries to be added to the queue and rebalance is - finished */ - - if (!defrag->crawl_done) { - - defrag->current_thread_count--; - gf_msg_debug ("DHT", 0, "Thread " - "sleeping while waiting " - "for migration entries. " - "current thread count:%d", - defrag->current_thread_count); - - pthread_cond_wait ( - &defrag->parallel_migration_cond, - &defrag->dfq_mutex); - } - - if (defrag->crawl_done && - !defrag->q_entry_count) { - defrag->current_thread_count++; - gf_msg_debug ("DHT", 0, "Exiting thread"); - - pthread_cond_broadcast ( - &defrag->parallel_migration_cond); - goto unlock; - } else { - defrag->current_thread_count++; - gf_msg_debug ("DHT", 0, "Thread woke up" - " as found migrating entries. " - "current thread count:%d", - defrag->current_thread_count); - - pthread_mutex_unlock - (&defrag->dfq_mutex); - continue; - } - } + if (defrag->crawl_done && !defrag->q_entry_count) { + defrag->current_thread_count++; + gf_msg_debug("DHT", 0, "Exiting thread"); + pthread_cond_broadcast(&defrag->parallel_migration_cond); + goto unlock; + } else { + defrag->current_thread_count++; + gf_msg_debug("DHT", 0, + "Thread woke up" + " as found migrating entries. " + "current thread count:%d", + defrag->current_thread_count); + + pthread_mutex_unlock(&defrag->dfq_mutex); + continue; } -unlock: - pthread_mutex_unlock (&defrag->dfq_mutex); - break; + } } + unlock: + pthread_mutex_unlock(&defrag->dfq_mutex); + break; + } out: - return NULL; + return NULL; } -int static -gf_defrag_get_entry (xlator_t *this, int i, struct dht_container **container, - loc_t *loc, dht_conf_t *conf, gf_defrag_info_t *defrag, - fd_t *fd, dict_t *migrate_data, - struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req, - int *should_commit_hash, int *perrno) +int static gf_defrag_get_entry(xlator_t *this, int i, + struct dht_container **container, loc_t *loc, + dht_conf_t *conf, gf_defrag_info_t *defrag, + fd_t *fd, dict_t *migrate_data, + struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req, + int *should_commit_hash, int *perrno) { - int ret = -1; - char is_linkfile = 0; - gf_dirent_t *df_entry = NULL; - struct dht_container *tmp_container = NULL; + int ret = -1; + char is_linkfile = 0; + gf_dirent_t *df_entry = NULL; + struct dht_container *tmp_container = NULL; - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { - ret = -1; - goto out; - } - - if (dir_dfmeta->offset_var[i].readdir_done == 1) { - ret = 0; - goto out; - } - - if (dir_dfmeta->fetch_entries[i] == 1) { - ret = syncop_readdirp (conf->local_subvols[i], fd, 131072, - dir_dfmeta->offset_var[i].offset, - &(dir_dfmeta->equeue[i]), - xattr_req, NULL); - if (ret == 0) { - dir_dfmeta->offset_var[i].readdir_done = 1; - ret = 0; - goto out; - } - - if (ret < 0) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_DATA_FAILED, - "Readdirp failed. Aborting data migration for " - "directory: %s", loc->path); - *perrno = -ret; - ret = -1; - goto out; - } + if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { + ret = -1; + goto out; + } - if (list_empty (&(dir_dfmeta->equeue[i].list))) { - dir_dfmeta->offset_var[i].readdir_done = 1; - ret = 0; - goto out; - } + if (dir_dfmeta->offset_var[i].readdir_done == 1) { + ret = 0; + goto out; + } - dir_dfmeta->fetch_entries[i] = 0; + if (dir_dfmeta->fetch_entries[i] == 1) { + ret = syncop_readdirp(conf->local_subvols[i], fd, 131072, + dir_dfmeta->offset_var[i].offset, + &(dir_dfmeta->equeue[i]), xattr_req, NULL); + if (ret == 0) { + dir_dfmeta->offset_var[i].readdir_done = 1; + ret = 0; + goto out; } - while (1) { - - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { - ret = -1; - goto out; - } - - df_entry = list_entry (dir_dfmeta->iterator[i]->next, - typeof (*df_entry), list); - - if (&df_entry->list == dir_dfmeta->head[i]) { - gf_dirent_free (&(dir_dfmeta->equeue[i])); - INIT_LIST_HEAD (&(dir_dfmeta->equeue[i].list)); - dir_dfmeta->fetch_entries[i] = 1; - dir_dfmeta->iterator[i] = dir_dfmeta->head[i]; - ret = 0; - goto out; - } - - dir_dfmeta->iterator[i] = dir_dfmeta->iterator[i]->next; - - dir_dfmeta->offset_var[i].offset = df_entry->d_off; - if (!strcmp (df_entry->d_name, ".") || - !strcmp (df_entry->d_name, "..")) - continue; - - if (IA_ISDIR (df_entry->d_stat.ia_type)) { - defrag->size_processed += df_entry->d_stat.ia_size; - continue; - } - - defrag->num_files_lookedup++; - - if (defrag->defrag_pattern && - (gf_defrag_pattern_match (defrag, df_entry->d_name, - df_entry->d_stat.ia_size) - == _gf_false)) { - defrag->size_processed += df_entry->d_stat.ia_size; - continue; - } - - is_linkfile = check_is_linkfile (NULL, &df_entry->d_stat, - df_entry->dict, - conf->link_xattr_name); - - if (is_linkfile) { - /* No need to add linkto file to the queue for - migration. Only the actual data file need to - be checked for migration criteria. - */ - - gf_msg_debug (this->name, 0, "Skipping linkfile" - " %s on subvol: %s", df_entry->d_name, - conf->local_subvols[i]->name); - continue; - } - - /*Build Container Structure */ + if (ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, -ret, + DHT_MSG_MIGRATE_DATA_FAILED, + "Readdirp failed. Aborting data migration for " + "directory: %s", + loc->path); + *perrno = -ret; + ret = -1; + goto out; + } - tmp_container = GF_CALLOC (1, sizeof(struct dht_container), - gf_dht_mt_container_t); - if (!tmp_container) { - gf_log (this->name, GF_LOG_ERROR, "Failed to allocate " - "memory for container"); - ret = -1; - goto out; - } - tmp_container->df_entry = gf_dirent_for_name (df_entry->d_name); - if (!tmp_container->df_entry) { - gf_log (this->name, GF_LOG_ERROR, "Failed to allocate " - "memory for df_entry"); - ret = -1; - goto out; - } + if (list_empty(&(dir_dfmeta->equeue[i].list))) { + dir_dfmeta->offset_var[i].readdir_done = 1; + ret = 0; + goto out; + } - tmp_container->local_subvol_index = i; + dir_dfmeta->fetch_entries[i] = 0; + } - tmp_container->df_entry->d_stat = df_entry->d_stat; + while (1) { + if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { + ret = -1; + goto out; + } - tmp_container->df_entry->d_ino = df_entry->d_ino; + df_entry = list_entry(dir_dfmeta->iterator[i]->next, typeof(*df_entry), + list); - tmp_container->df_entry->d_type = df_entry->d_type; + if (&df_entry->list == dir_dfmeta->head[i]) { + gf_dirent_free(&(dir_dfmeta->equeue[i])); + INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list)); + dir_dfmeta->fetch_entries[i] = 1; + dir_dfmeta->iterator[i] = dir_dfmeta->head[i]; + ret = 0; + goto out; + } - tmp_container->df_entry->d_len = df_entry->d_len; + dir_dfmeta->iterator[i] = dir_dfmeta->iterator[i]->next; - tmp_container->parent_loc = GF_CALLOC(1, sizeof(*loc), - gf_dht_mt_loc_t); - if (!tmp_container->parent_loc) { - gf_log (this->name, GF_LOG_ERROR, "Failed to allocate " - "memory for loc"); - ret = -1; - goto out; - } + dir_dfmeta->offset_var[i].offset = df_entry->d_off; + if (!strcmp(df_entry->d_name, ".") || !strcmp(df_entry->d_name, "..")) + continue; + if (IA_ISDIR(df_entry->d_stat.ia_type)) { + defrag->size_processed += df_entry->d_stat.ia_size; + continue; + } - ret = loc_copy (tmp_container->parent_loc, loc); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "loc_copy failed"); - ret = -1; - goto out; - } + defrag->num_files_lookedup++; - tmp_container->migrate_data = migrate_data; + if (defrag->defrag_pattern && + (gf_defrag_pattern_match(defrag, df_entry->d_name, + df_entry->d_stat.ia_size) == _gf_false)) { + defrag->size_processed += df_entry->d_stat.ia_size; + continue; + } - tmp_container->this = this; + is_linkfile = check_is_linkfile(NULL, &df_entry->d_stat, df_entry->dict, + conf->link_xattr_name); - if (df_entry->dict) - tmp_container->df_entry->dict = - dict_ref (df_entry->dict); + if (is_linkfile) { + /* No need to add linkto file to the queue for + migration. Only the actual data file need to + be checked for migration criteria. + */ - /*Build Container Structure >> END*/ + gf_msg_debug(this->name, 0, + "Skipping linkfile" + " %s on subvol: %s", + df_entry->d_name, conf->local_subvols[i]->name); + continue; + } - ret = 0; - goto out; + /*Build Container Structure */ + tmp_container = GF_CALLOC(1, sizeof(struct dht_container), + gf_dht_mt_container_t); + if (!tmp_container) { + gf_log(this->name, GF_LOG_ERROR, + "Failed to allocate " + "memory for container"); + ret = -1; + goto out; } - -out: - if (ret == 0) { - *container = tmp_container; - } else { - if (tmp_container) { - gf_defrag_free_container (tmp_container); - } + tmp_container->df_entry = gf_dirent_for_name(df_entry->d_name); + if (!tmp_container->df_entry) { + gf_log(this->name, GF_LOG_ERROR, + "Failed to allocate " + "memory for df_entry"); + ret = -1; + goto out; } - return ret; -} + tmp_container->local_subvol_index = i; -int -gf_defrag_process_dir (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, - dict_t *migrate_data, int *perrno) -{ - int ret = -1; - fd_t *fd = NULL; - dht_conf_t *conf = NULL; - gf_dirent_t entries; - dict_t *xattr_req = NULL; - struct timeval dir_start = {0,}; - struct timeval end = {0,}; - double elapsed = {0,}; - int local_subvols_cnt = 0; - int i = 0; - int j = 0; - struct dht_container *container = NULL; - int ldfq_count = 0; - int dfc_index = 0; - int throttle_up = 0; - struct dir_dfmeta *dir_dfmeta = NULL; - int should_commit_hash = 1; - - gf_log (this->name, GF_LOG_INFO, "migrate data called on %s", - loc->path); - gettimeofday (&dir_start, NULL); + tmp_container->df_entry->d_stat = df_entry->d_stat; - conf = this->private; - local_subvols_cnt = conf->local_subvols_cnt; + tmp_container->df_entry->d_ino = df_entry->d_ino; - if (!local_subvols_cnt) { - ret = 0; - goto out; - } + tmp_container->df_entry->d_type = df_entry->d_type; - fd = fd_create (loc->inode, defrag->pid); - if (!fd) { - gf_log (this->name, GF_LOG_ERROR, "Failed to create fd"); - ret = -1; - goto out; - } + tmp_container->df_entry->d_len = df_entry->d_len; - ret = syncop_opendir (this, loc, fd, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_MIGRATE_DATA_FAILED, - "Migrate data failed: Failed to open dir %s", - loc->path); - *perrno = -ret; - ret = -1; - goto out; + tmp_container->parent_loc = GF_CALLOC(1, sizeof(*loc), gf_dht_mt_loc_t); + if (!tmp_container->parent_loc) { + gf_log(this->name, GF_LOG_ERROR, + "Failed to allocate " + "memory for loc"); + ret = -1; + goto out; } - fd_bind (fd); - dir_dfmeta = GF_CALLOC (1, sizeof (*dir_dfmeta), - gf_common_mt_pointer); - if (!dir_dfmeta) { - gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta is NULL"); - ret = -1; - goto out; + ret = loc_copy(tmp_container->parent_loc, loc); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, "loc_copy failed"); + ret = -1; + goto out; } + tmp_container->migrate_data = migrate_data; - dir_dfmeta->head = GF_CALLOC (local_subvols_cnt, - sizeof (*(dir_dfmeta->head)), - gf_common_mt_pointer); - if (!dir_dfmeta->head) { - gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta->head is NULL"); - ret = -1; - goto out; - } + tmp_container->this = this; - dir_dfmeta->iterator = GF_CALLOC (local_subvols_cnt, - sizeof (*(dir_dfmeta->iterator)), - gf_common_mt_pointer); - if (!dir_dfmeta->iterator) { - gf_log (this->name, GF_LOG_ERROR, - "dir_dfmeta->iterator is NULL"); - ret = -1; - goto out; - } + if (df_entry->dict) + tmp_container->df_entry->dict = dict_ref(df_entry->dict); - dir_dfmeta->equeue = GF_CALLOC (local_subvols_cnt, sizeof (entries), - gf_dht_mt_dirent_t); - if (!dir_dfmeta->equeue) { - gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta->equeue is NULL"); - ret = -1; - goto out; - } + /*Build Container Structure >> END*/ - dir_dfmeta->offset_var = GF_CALLOC (local_subvols_cnt, - sizeof (dht_dfoffset_ctx_t), - gf_dht_mt_octx_t); - if (!dir_dfmeta->offset_var) { - gf_log (this->name, GF_LOG_ERROR, - "dir_dfmeta->offset_var is NULL"); - ret = -1; - goto out; - } - ret = gf_defrag_ctx_subvols_init (dir_dfmeta->offset_var, this); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, "dht_dfoffset_ctx_t" - "initialization failed"); - ret = -1; - goto out; - } - - dir_dfmeta->fetch_entries = GF_CALLOC (local_subvols_cnt, - sizeof (int), gf_common_mt_int); - if (!dir_dfmeta->fetch_entries) { - gf_log (this->name, GF_LOG_ERROR, - "dir_dfmeta->fetch_entries is NULL"); - ret = -1; - goto out; - } - - for (i = 0; i < local_subvols_cnt ; i++) { - INIT_LIST_HEAD (&(dir_dfmeta->equeue[i].list)); - dir_dfmeta->head[i] = &(dir_dfmeta->equeue[i].list); - dir_dfmeta->iterator[i] = dir_dfmeta->head[i]; - dir_dfmeta->fetch_entries[i] = 1; - } + ret = 0; + goto out; + } - xattr_req = dict_new (); - if (!xattr_req) { - gf_log (this->name, GF_LOG_ERROR, "dict_new failed"); - ret = -1; - goto out; +out: + if (ret == 0) { + *container = tmp_container; + } else { + if (tmp_container) { + gf_defrag_free_container(tmp_container); } + } - ret = dict_set_uint32 (xattr_req, - conf->link_xattr_name, 256); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, "failed to set dict for " - "key: %s", conf->link_xattr_name); - ret = -1; - goto out; - } + return ret; +} - /* - Job: Read entries from each local subvol and store the entries - in equeue array of linked list. Now pick one entry from the - equeue array in a round robin basis and add them to defrag Queue. - */ +int +gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, + dict_t *migrate_data, int *perrno) +{ + int ret = -1; + fd_t *fd = NULL; + dht_conf_t *conf = NULL; + gf_dirent_t entries; + dict_t *xattr_req = NULL; + struct timeval dir_start = { + 0, + }; + struct timeval end = { + 0, + }; + double elapsed = { + 0, + }; + int local_subvols_cnt = 0; + int i = 0; + int j = 0; + struct dht_container *container = NULL; + int ldfq_count = 0; + int dfc_index = 0; + int throttle_up = 0; + struct dir_dfmeta *dir_dfmeta = NULL; + int should_commit_hash = 1; + + gf_log(this->name, GF_LOG_INFO, "migrate data called on %s", loc->path); + gettimeofday(&dir_start, NULL); + + conf = this->private; + local_subvols_cnt = conf->local_subvols_cnt; + + if (!local_subvols_cnt) { + ret = 0; + goto out; + } - while (!dht_dfreaddirp_done(dir_dfmeta->offset_var, - local_subvols_cnt)) { + fd = fd_create(loc->inode, defrag->pid); + if (!fd) { + gf_log(this->name, GF_LOG_ERROR, "Failed to create fd"); + ret = -1; + goto out; + } + + ret = syncop_opendir(this, loc, fd, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_DATA_FAILED, + "Migrate data failed: Failed to open dir %s", loc->path); + *perrno = -ret; + ret = -1; + goto out; + } - pthread_mutex_lock (&defrag->dfq_mutex); - { + fd_bind(fd); + dir_dfmeta = GF_CALLOC(1, sizeof(*dir_dfmeta), gf_common_mt_pointer); + if (!dir_dfmeta) { + gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta is NULL"); + ret = -1; + goto out; + } - /*Throttle up: If reconfigured count is higher than - current thread count, wake up the sleeping threads - TODO: Need to refactor this. Instead of making the - thread sleep and wake, we should terminate and spawn - threads on-demand*/ - - if (defrag->recon_thread_count > - defrag->current_thread_count) { - throttle_up = - (defrag->recon_thread_count - - defrag->current_thread_count); - for (j = 0; j < throttle_up; j++) { - pthread_cond_signal ( - &defrag->df_wakeup_thread); - } + dir_dfmeta->head = GF_CALLOC(local_subvols_cnt, sizeof(*(dir_dfmeta->head)), + gf_common_mt_pointer); + if (!dir_dfmeta->head) { + gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->head is NULL"); + ret = -1; + goto out; + } + + dir_dfmeta->iterator = GF_CALLOC(local_subvols_cnt, + sizeof(*(dir_dfmeta->iterator)), + gf_common_mt_pointer); + if (!dir_dfmeta->iterator) { + gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->iterator is NULL"); + ret = -1; + goto out; + } - } + dir_dfmeta->equeue = GF_CALLOC(local_subvols_cnt, sizeof(entries), + gf_dht_mt_dirent_t); + if (!dir_dfmeta->equeue) { + gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->equeue is NULL"); + ret = -1; + goto out; + } - while (defrag->q_entry_count > - MAX_MIGRATE_QUEUE_COUNT) { - defrag->wakeup_crawler = 1; - pthread_cond_wait ( - &defrag->rebalance_crawler_alarm, - &defrag->dfq_mutex); - } + dir_dfmeta->offset_var = GF_CALLOC( + local_subvols_cnt, sizeof(dht_dfoffset_ctx_t), gf_dht_mt_octx_t); + if (!dir_dfmeta->offset_var) { + gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->offset_var is NULL"); + ret = -1; + goto out; + } + ret = gf_defrag_ctx_subvols_init(dir_dfmeta->offset_var, this); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "dht_dfoffset_ctx_t" + "initialization failed"); + ret = -1; + goto out; + } - ldfq_count = defrag->q_entry_count; + dir_dfmeta->fetch_entries = GF_CALLOC(local_subvols_cnt, sizeof(int), + gf_common_mt_int); + if (!dir_dfmeta->fetch_entries) { + gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->fetch_entries is NULL"); + ret = -1; + goto out; + } + + for (i = 0; i < local_subvols_cnt; i++) { + INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list)); + dir_dfmeta->head[i] = &(dir_dfmeta->equeue[i].list); + dir_dfmeta->iterator[i] = dir_dfmeta->head[i]; + dir_dfmeta->fetch_entries[i] = 1; + } + + xattr_req = dict_new(); + if (!xattr_req) { + gf_log(this->name, GF_LOG_ERROR, "dict_new failed"); + ret = -1; + goto out; + } + + ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "failed to set dict for " + "key: %s", + conf->link_xattr_name); + ret = -1; + goto out; + } - if (defrag->wakeup_crawler) { - defrag->wakeup_crawler = 0; - } + /* + Job: Read entries from each local subvol and store the entries + in equeue array of linked list. Now pick one entry from the + equeue array in a round robin basis and add them to defrag Queue. + */ + while (!dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) { + pthread_mutex_lock(&defrag->dfq_mutex); + { + /*Throttle up: If reconfigured count is higher than + current thread count, wake up the sleeping threads + TODO: Need to refactor this. Instead of making the + thread sleep and wake, we should terminate and spawn + threads on-demand*/ + + if (defrag->recon_thread_count > defrag->current_thread_count) { + throttle_up = (defrag->recon_thread_count - + defrag->current_thread_count); + for (j = 0; j < throttle_up; j++) { + pthread_cond_signal(&defrag->df_wakeup_thread); } - pthread_mutex_unlock (&defrag->dfq_mutex); - - while (ldfq_count <= MAX_MIGRATE_QUEUE_COUNT && - !dht_dfreaddirp_done(dir_dfmeta->offset_var, - local_subvols_cnt)) { + } - ret = gf_defrag_get_entry (this, dfc_index, &container, - loc, conf, defrag, fd, - migrate_data, dir_dfmeta, - xattr_req, - &should_commit_hash, perrno); + while (defrag->q_entry_count > MAX_MIGRATE_QUEUE_COUNT) { + defrag->wakeup_crawler = 1; + pthread_cond_wait(&defrag->rebalance_crawler_alarm, + &defrag->dfq_mutex); + } - if (ret) { - gf_log (this->name, GF_LOG_WARNING, "Found " - "error from gf_defrag_get_entry"); + ldfq_count = defrag->q_entry_count; - ret = -1; - goto out; - } - - /* Check if we got an entry, else we need to move the - index to the next subvol */ - if (!container) { - GF_CRAWL_INDEX_MOVE(dfc_index, - local_subvols_cnt); - continue; - } + if (defrag->wakeup_crawler) { + defrag->wakeup_crawler = 0; + } + } + pthread_mutex_unlock(&defrag->dfq_mutex); - /* Q this entry in the dfq */ - pthread_mutex_lock (&defrag->dfq_mutex); - { - list_add_tail (&container->list, - &(defrag->queue[0].list)); - defrag->q_entry_count++; - ldfq_count = defrag->q_entry_count; - - gf_msg_debug (this->name, 0, "added " - "file:%s parent:%s to the queue ", - container->df_entry->d_name, - container->parent_loc->path); - - pthread_cond_signal ( - &defrag->parallel_migration_cond); - } - pthread_mutex_unlock (&defrag->dfq_mutex); + while ( + ldfq_count <= MAX_MIGRATE_QUEUE_COUNT && + !dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) { + ret = gf_defrag_get_entry(this, dfc_index, &container, loc, conf, + defrag, fd, migrate_data, dir_dfmeta, + xattr_req, &should_commit_hash, perrno); - GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt); - } - } + if (ret) { + gf_log(this->name, GF_LOG_WARNING, + "Found " + "error from gf_defrag_get_entry"); - gettimeofday (&end, NULL); - elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 + - (end.tv_usec - dir_start.tv_usec); - gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took " - "%.2f secs", loc->path, elapsed/1e6); - ret = 0; + ret = -1; + goto out; + } + + /* Check if we got an entry, else we need to move the + index to the next subvol */ + if (!container) { + GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt); + continue; + } + + /* Q this entry in the dfq */ + pthread_mutex_lock(&defrag->dfq_mutex); + { + list_add_tail(&container->list, &(defrag->queue[0].list)); + defrag->q_entry_count++; + ldfq_count = defrag->q_entry_count; + + gf_msg_debug(this->name, 0, + "added " + "file:%s parent:%s to the queue ", + container->df_entry->d_name, + container->parent_loc->path); + + pthread_cond_signal(&defrag->parallel_migration_cond); + } + pthread_mutex_unlock(&defrag->dfq_mutex); + + GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt); + } + } + + gettimeofday(&end, NULL); + elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 + + (end.tv_usec - dir_start.tv_usec); + gf_log(this->name, GF_LOG_INFO, + "Migration operation on dir %s took " + "%.2f secs", + loc->path, elapsed / 1e6); + ret = 0; out: - gf_defrag_free_dir_dfmeta (dir_dfmeta, local_subvols_cnt); + gf_defrag_free_dir_dfmeta(dir_dfmeta, local_subvols_cnt); - if (xattr_req) - dict_unref(xattr_req); + if (xattr_req) + dict_unref(xattr_req); - if (fd) - fd_unref (fd); + if (fd) + fd_unref(fd); - if (ret == 0 && should_commit_hash == 0) { - ret = 2; - } + if (ret == 0 && should_commit_hash == 0) { + ret = 2; + } - /* It does not matter if it errored out - this number is - * used to calculate rebalance estimated time to complete. - * No locking required as dirs are processed by a single thread. - */ - defrag->num_dirs_processed++; - return ret; + /* It does not matter if it errored out - this number is + * used to calculate rebalance estimated time to complete. + * No locking required as dirs are processed by a single thread. + */ + defrag->num_dirs_processed++; + return ret; } int -gf_defrag_settle_hash (xlator_t *this, gf_defrag_info_t *defrag, - loc_t *loc, dict_t *fix_layout) +gf_defrag_settle_hash(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, + dict_t *fix_layout) { - int ret; - dht_conf_t *conf = NULL; - /* - * Now we're ready to update the directory commit hash for the volume - * root, so that hash miscompares and broadcast lookups can stop. - * However, we want to skip that if fix-layout is all we did. In - * that case, we want the miscompares etc. to continue until a real - * rebalance is complete. - */ - if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX - || defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || - defrag->cmd == GF_DEFRAG_CMD_DETACH_START) { - return 0; - } - - conf = this->private; - if (!conf) { - /*Uh oh - */ - return -1; - } + int ret; + dht_conf_t *conf = NULL; + /* + * Now we're ready to update the directory commit hash for the volume + * root, so that hash miscompares and broadcast lookups can stop. + * However, we want to skip that if fix-layout is all we did. In + * that case, we want the miscompares etc. to continue until a real + * rebalance is complete. + */ + if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX || + defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || + defrag->cmd == GF_DEFRAG_CMD_DETACH_START) { + return 0; + } - if (conf->local_subvols_cnt == 0 || !conf->lookup_optimize) { - /* Commit hash updates are only done on local subvolumes and - * only when lookup optimization is needed (for older client - * support) - */ - return 0; - } + conf = this->private; + if (!conf) { + /*Uh oh + */ + return -1; + } - ret = dict_set_uint32 (fix_layout, "new-commit-hash", - defrag->new_commit_hash); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "Failed to set new-commit-hash"); - return -1; - } + if (conf->local_subvols_cnt == 0 || !conf->lookup_optimize) { + /* Commit hash updates are only done on local subvolumes and + * only when lookup optimization is needed (for older client + * support) + */ + return 0; + } - ret = syncop_setxattr (this, loc, fix_layout, 0, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_LAYOUT_FIX_FAILED, - "fix layout on %s failed", loc->path); + ret = dict_set_uint32(fix_layout, "new-commit-hash", + defrag->new_commit_hash); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, "Failed to set new-commit-hash"); + return -1; + } - if (-ret == ENOENT || -ret == ESTALE) { - /* Dir most likely is deleted */ - return 0; - } + ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED, + "fix layout on %s failed", loc->path); - return -1; + if (-ret == ENOENT || -ret == ESTALE) { + /* Dir most likely is deleted */ + return 0; } - /* TBD: find more efficient solution than adding/deleting every time */ - dict_del(fix_layout, "new-commit-hash"); - - return 0; -} + return -1; + } + /* TBD: find more efficient solution than adding/deleting every time */ + dict_del(fix_layout, "new-commit-hash"); + return 0; +} /* Function for doing a named lookup on file inodes during an attach tier * So that a hardlink lookup heal i.e gfid to parent gfid lookup heal @@ -3717,1721 +3588,1672 @@ gf_defrag_settle_hash (xlator_t *this, gf_defrag_info_t *defrag, * eventual consistency * */ int -gf_fix_layout_tier_attach_lookup (xlator_t *this, - loc_t *parent_loc, +gf_fix_layout_tier_attach_lookup(xlator_t *this, loc_t *parent_loc, gf_dirent_t *file_dentry) { - int ret = -1; - dict_t *lookup_xdata = NULL; - dht_conf_t *conf = NULL; - loc_t file_loc = {0,}; - struct iatt iatt = {0,}; + int ret = -1; + dict_t *lookup_xdata = NULL; + dht_conf_t *conf = NULL; + loc_t file_loc = { + 0, + }; + struct iatt iatt = { + 0, + }; - GF_VALIDATE_OR_GOTO ("tier", this, out); + GF_VALIDATE_OR_GOTO("tier", this, out); - GF_VALIDATE_OR_GOTO (this->name, parent_loc, out); + GF_VALIDATE_OR_GOTO(this->name, parent_loc, out); - GF_VALIDATE_OR_GOTO (this->name, file_dentry, out); + GF_VALIDATE_OR_GOTO(this->name, file_dentry, out); - GF_VALIDATE_OR_GOTO (this->name, this->private, out); - - if (!parent_loc->inode) { - gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "%s/%s parent is NULL", parent_loc->path, - file_dentry->d_name); - goto out; - } + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + if (!parent_loc->inode) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, + "%s/%s parent is NULL", parent_loc->path, file_dentry->d_name); + goto out; + } - conf = this->private; + conf = this->private; - loc_wipe (&file_loc); + loc_wipe(&file_loc); - if (gf_uuid_is_null (file_dentry->d_stat.ia_gfid)) { - gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "%s/%s gfid not present", parent_loc->path, - file_dentry->d_name); - goto out; - } + if (gf_uuid_is_null(file_dentry->d_stat.ia_gfid)) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, + "%s/%s gfid not present", parent_loc->path, file_dentry->d_name); + goto out; + } - gf_uuid_copy (file_loc.gfid, file_dentry->d_stat.ia_gfid); - - if (gf_uuid_is_null (parent_loc->gfid)) { - gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "%s/%s" - " gfid not present", parent_loc->path, - file_dentry->d_name); - goto out; - } + gf_uuid_copy(file_loc.gfid, file_dentry->d_stat.ia_gfid); - gf_uuid_copy (file_loc.pargfid, parent_loc->gfid); + if (gf_uuid_is_null(parent_loc->gfid)) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, + "%s/%s" + " gfid not present", + parent_loc->path, file_dentry->d_name); + goto out; + } + gf_uuid_copy(file_loc.pargfid, parent_loc->gfid); - ret = dht_build_child_loc (this, &file_loc, parent_loc, - file_dentry->d_name); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "Child loc build failed"); - ret = -1; - goto out; - } - - lookup_xdata = dict_new (); - if (!lookup_xdata) { - gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "Failed creating lookup dict for %s", - file_dentry->d_name); - goto out; - } - - ret = dict_set_int32 (lookup_xdata, CTR_ATTACH_TIER_LOOKUP, 1); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, - "Failed to set lookup flag"); - goto out; - } - - gf_uuid_copy (file_loc.parent->gfid, parent_loc->gfid); - - /* Sending lookup to cold tier only */ - ret = syncop_lookup (conf->subvolumes[0], &file_loc, &iatt, - NULL, lookup_xdata, NULL); - if (ret) { - /* If the file does not exist on the cold tier than it must */ - /* have been discovered on the hot tier. This is not an error. */ - gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS, - "%s lookup to cold tier on attach heal failed", file_loc.path); - goto out; - } - - ret = 0; + ret = dht_build_child_loc(this, &file_loc, parent_loc, file_dentry->d_name); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, + "Child loc build failed"); + ret = -1; + goto out; + } + + lookup_xdata = dict_new(); + if (!lookup_xdata) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, + "Failed creating lookup dict for %s", file_dentry->d_name); + goto out; + } + + ret = dict_set_int32(lookup_xdata, CTR_ATTACH_TIER_LOOKUP, 1); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR, + "Failed to set lookup flag"); + goto out; + } + + gf_uuid_copy(file_loc.parent->gfid, parent_loc->gfid); + + /* Sending lookup to cold tier only */ + ret = syncop_lookup(conf->subvolumes[0], &file_loc, &iatt, NULL, + lookup_xdata, NULL); + if (ret) { + /* If the file does not exist on the cold tier than it must */ + /* have been discovered on the hot tier. This is not an error. */ + gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS, + "%s lookup to cold tier on attach heal failed", file_loc.path); + goto out; + } + + ret = 0; out: - loc_wipe (&file_loc); + loc_wipe(&file_loc); - if (lookup_xdata) - dict_unref (lookup_xdata); + if (lookup_xdata) + dict_unref(lookup_xdata); - return ret; + return ret; } int -gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, - dict_t *fix_layout, dict_t *migrate_data) +gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, + dict_t *fix_layout, dict_t *migrate_data) { - int ret = -1; - loc_t entry_loc = {0,}; - fd_t *fd = NULL; - gf_dirent_t entries; - gf_dirent_t *tmp = NULL; - gf_dirent_t *entry = NULL; - gf_boolean_t free_entries = _gf_false; - off_t offset = 0; - struct iatt iatt = {0,}; - inode_t *linked_inode = NULL, *inode = NULL; - dht_conf_t *conf = NULL; - int should_commit_hash = 1; - int perrno = 0; - - conf = this->private; - if (!conf) { - ret = -1; - goto out; - } - - ret = syncop_lookup (this, loc, &iatt, NULL, NULL, NULL); - if (ret) { - if (strcmp (loc->path, "/") == 0) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_DIR_LOOKUP_FAILED, - "lookup failed for:%s", loc->path); - - defrag->total_failures++; - ret = -1; - goto out; - } + int ret = -1; + loc_t entry_loc = { + 0, + }; + fd_t *fd = NULL; + gf_dirent_t entries; + gf_dirent_t *tmp = NULL; + gf_dirent_t *entry = NULL; + gf_boolean_t free_entries = _gf_false; + off_t offset = 0; + struct iatt iatt = { + 0, + }; + inode_t *linked_inode = NULL, *inode = NULL; + dht_conf_t *conf = NULL; + int should_commit_hash = 1; + int perrno = 0; + + conf = this->private; + if (!conf) { + ret = -1; + goto out; + } - if (-ret == ENOENT || -ret == ESTALE) { - gf_msg (this->name, GF_LOG_INFO, -ret, - DHT_MSG_DIR_LOOKUP_FAILED, - "Dir:%s renamed or removed. Skipping", - loc->path); - if (conf->decommission_subvols_cnt) { - defrag->total_failures++; - } - ret = 0; - goto out; - } else { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_DIR_LOOKUP_FAILED, - "lookup failed for:%s", loc->path); + ret = syncop_lookup(this, loc, &iatt, NULL, NULL, NULL); + if (ret) { + if (strcmp(loc->path, "/") == 0) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED, + "lookup failed for:%s", loc->path); - defrag->total_failures++; - goto out; - } + defrag->total_failures++; + ret = -1; + goto out; } - fd = fd_create (loc->inode, defrag->pid); - if (!fd) { - gf_log (this->name, GF_LOG_ERROR, "Failed to create fd"); - ret = -1; - goto out; - } + if (-ret == ENOENT || -ret == ESTALE) { + gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_LOOKUP_FAILED, + "Dir:%s renamed or removed. Skipping", loc->path); + if (conf->decommission_subvols_cnt) { + defrag->total_failures++; + } + ret = 0; + goto out; + } else { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED, + "lookup failed for:%s", loc->path); - ret = syncop_opendir (this, loc, fd, NULL, NULL); - if (ret) { - if (-ret == ENOENT || -ret == ESTALE) { - if (conf->decommission_subvols_cnt) { - defrag->total_failures++; - } - ret = 0; - goto out; - } + defrag->total_failures++; + goto out; + } + } - gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s, " - "err:%d", loc->path, -ret); + fd = fd_create(loc->inode, defrag->pid); + if (!fd) { + gf_log(this->name, GF_LOG_ERROR, "Failed to create fd"); + ret = -1; + goto out; + } - ret = -1; - goto out; + ret = syncop_opendir(this, loc, fd, NULL, NULL); + if (ret) { + if (-ret == ENOENT || -ret == ESTALE) { + if (conf->decommission_subvols_cnt) { + defrag->total_failures++; + } + ret = 0; + goto out; } - fd_bind (fd); - INIT_LIST_HEAD (&entries.list); + gf_log(this->name, GF_LOG_ERROR, + "Failed to open dir %s, " + "err:%d", + loc->path, -ret); - while ((ret = syncop_readdirp (this, fd, 131072, offset, &entries, - NULL, NULL)) != 0) - { - if (ret < 0) { - if (-ret == ENOENT || -ret == ESTALE) { - if (conf->decommission_subvols_cnt) { - defrag->total_failures++; - } - ret = 0; - goto out; - } + ret = -1; + goto out; + } - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_READDIR_ERROR, "readdirp failed for " - "path %s. Aborting fix-layout", loc->path); + fd_bind(fd); + INIT_LIST_HEAD(&entries.list); - ret = -1; - goto out; + while ((ret = syncop_readdirp(this, fd, 131072, offset, &entries, NULL, + NULL)) != 0) { + if (ret < 0) { + if (-ret == ENOENT || -ret == ESTALE) { + if (conf->decommission_subvols_cnt) { + defrag->total_failures++; } + ret = 0; + goto out; + } - if (list_empty (&entries.list)) - break; - - free_entries = _gf_true; - - list_for_each_entry_safe (entry, tmp, &entries.list, list) { - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { - ret = 1; - goto out; - } - - offset = entry->d_off; - - if (!strcmp (entry->d_name, ".") || - !strcmp (entry->d_name, "..")) - continue; - if (!IA_ISDIR (entry->d_stat.ia_type)) { - - /* If its a fix layout during the attach - * tier operation do lookups on files - * on cold subvolume so that there is a - * CTR DB Lookup Heal triggered on existing - * data. - * */ - if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { - gf_fix_layout_tier_attach_lookup - (this, loc, entry); - } - - continue; - } - loc_wipe (&entry_loc); - - ret = dht_build_child_loc (this, &entry_loc, loc, - entry->d_name); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, "Child loc" - " build failed for entry: %s", - entry->d_name); - - if (conf->decommission_in_progress) { - defrag->defrag_status = - GF_DEFRAG_STATUS_FAILED; - - goto out; - } else { - should_commit_hash = 0; + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_READDIR_ERROR, + "readdirp failed for " + "path %s. Aborting fix-layout", + loc->path); - continue; - } - } + ret = -1; + goto out; + } - if (gf_uuid_is_null (entry->d_stat.ia_gfid)) { - gf_log (this->name, GF_LOG_ERROR, "%s/%s" - " gfid not present", loc->path, - entry->d_name); - continue; - } + if (list_empty(&entries.list)) + break; + free_entries = _gf_true; - gf_uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid); + list_for_each_entry_safe(entry, tmp, &entries.list, list) + { + if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) { + ret = 1; + goto out; + } + + offset = entry->d_off; + + if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) + continue; + if (!IA_ISDIR(entry->d_stat.ia_type)) { + /* If its a fix layout during the attach + * tier operation do lookups on files + * on cold subvolume so that there is a + * CTR DB Lookup Heal triggered on existing + * data. + * */ + if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { + gf_fix_layout_tier_attach_lookup(this, loc, entry); + } - /*In case the gfid stored in the inode by inode_link - * and the gfid obtained in the lookup differs, then - * client3_3_lookup_cbk will return ESTALE and proper - * error will be captured - */ + continue; + } + loc_wipe(&entry_loc); - linked_inode = inode_link (entry_loc.inode, loc->inode, - entry->d_name, - &entry->d_stat); + ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "Child loc" + " build failed for entry: %s", + entry->d_name); - inode = entry_loc.inode; - entry_loc.inode = linked_inode; - inode_unref (inode); + if (conf->decommission_in_progress) { + defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; - if (gf_uuid_is_null (loc->gfid)) { - gf_log (this->name, GF_LOG_ERROR, "%s/%s" - " gfid not present", loc->path, - entry->d_name); - continue; - } + goto out; + } else { + should_commit_hash = 0; - gf_uuid_copy (entry_loc.pargfid, loc->gfid); - - ret = syncop_lookup (this, &entry_loc, &iatt, NULL, - NULL, NULL); - if (ret) { - if (-ret == ENOENT || -ret == ESTALE) { - gf_msg (this->name, GF_LOG_INFO, -ret, - DHT_MSG_DIR_LOOKUP_FAILED, - "Dir:%s renamed or removed. " - "Skipping", loc->path); - ret = 0; - if (conf->decommission_subvols_cnt) { - defrag->total_failures++; - } - continue; - } else { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_DIR_LOOKUP_FAILED, - "lookup failed for:%s", - entry_loc.path); - - defrag->total_failures++; - - if (conf->decommission_in_progress) { - defrag->defrag_status = - GF_DEFRAG_STATUS_FAILED; - ret = -1; - goto out; - } else { - should_commit_hash = 0; - continue; - } - } - } + continue; + } + } + + if (gf_uuid_is_null(entry->d_stat.ia_gfid)) { + gf_log(this->name, GF_LOG_ERROR, + "%s/%s" + " gfid not present", + loc->path, entry->d_name); + continue; + } + + gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid); + + /*In case the gfid stored in the inode by inode_link + * and the gfid obtained in the lookup differs, then + * client3_3_lookup_cbk will return ESTALE and proper + * error will be captured + */ + + linked_inode = inode_link(entry_loc.inode, loc->inode, + entry->d_name, &entry->d_stat); + + inode = entry_loc.inode; + entry_loc.inode = linked_inode; + inode_unref(inode); + + if (gf_uuid_is_null(loc->gfid)) { + gf_log(this->name, GF_LOG_ERROR, + "%s/%s" + " gfid not present", + loc->path, entry->d_name); + continue; + } + + gf_uuid_copy(entry_loc.pargfid, loc->gfid); + + ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL); + if (ret) { + if (-ret == ENOENT || -ret == ESTALE) { + gf_msg(this->name, GF_LOG_INFO, -ret, + DHT_MSG_DIR_LOOKUP_FAILED, + "Dir:%s renamed or removed. " + "Skipping", + loc->path); + ret = 0; + if (conf->decommission_subvols_cnt) { + defrag->total_failures++; + } + continue; + } else { + gf_msg(this->name, GF_LOG_ERROR, -ret, + DHT_MSG_DIR_LOOKUP_FAILED, "lookup failed for:%s", + entry_loc.path); - /* A return value of 2 means, either process_dir or - * lookup of a dir failed. Hence, don't commit hash - * for the current directory*/ + defrag->total_failures++; - ret = gf_defrag_fix_layout (this, defrag, &entry_loc, - fix_layout, migrate_data); + if (conf->decommission_in_progress) { + defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; + ret = -1; + goto out; + } else { + should_commit_hash = 0; + continue; + } + } + } - if (ret && ret != 2) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_LAYOUT_FIX_FAILED, - "Fix layout failed for %s", - entry_loc.path); + /* A return value of 2 means, either process_dir or + * lookup of a dir failed. Hence, don't commit hash + * for the current directory*/ - defrag->total_failures++; + ret = gf_defrag_fix_layout(this, defrag, &entry_loc, fix_layout, + migrate_data); - if (conf->decommission_in_progress) { - defrag->defrag_status = - GF_DEFRAG_STATUS_FAILED; + if (ret && ret != 2) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_FIX_FAILED, + "Fix layout failed for %s", entry_loc.path); - goto out; - } else { - /* Let's not commit-hash if - * gf_defrag_fix_layout failed*/ - continue; - } - } - } + defrag->total_failures++; - gf_dirent_free (&entries); - free_entries = _gf_false; - INIT_LIST_HEAD (&entries.list); - } + if (conf->decommission_in_progress) { + defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; - ret = syncop_setxattr (this, loc, fix_layout, 0, NULL, NULL); - if (ret) { - if (-ret == ENOENT || -ret == ESTALE) { - gf_msg (this->name, GF_LOG_INFO, -ret, - DHT_MSG_LAYOUT_FIX_FAILED, - "Setxattr failed. Dir %s " - "renamed or removed", - loc->path); - if (conf->decommission_subvols_cnt) { - defrag->total_failures++; - } - ret = 0; + goto out; } else { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_LAYOUT_FIX_FAILED, - "Setxattr failed for %s", - loc->path); + /* Let's not commit-hash if + * gf_defrag_fix_layout failed*/ + continue; + } + } + } + + gf_dirent_free(&entries); + free_entries = _gf_false; + INIT_LIST_HEAD(&entries.list); + } + + ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL); + if (ret) { + if (-ret == ENOENT || -ret == ESTALE) { + gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_LAYOUT_FIX_FAILED, + "Setxattr failed. Dir %s " + "renamed or removed", + loc->path); + if (conf->decommission_subvols_cnt) { + defrag->total_failures++; + } + ret = 0; + } else { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED, + "Setxattr failed for %s", loc->path); - defrag->total_failures++; + defrag->total_failures++; - if (conf->decommission_in_progress) { - defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; - ret = -1; - goto out; - } - } + if (conf->decommission_in_progress) { + defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; + ret = -1; + goto out; + } } + } - if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) && - (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX)) { - ret = gf_defrag_process_dir (this, defrag, loc, migrate_data, - &perrno); - - if (ret && (ret != 2)) { - if (perrno == ENOENT || perrno == ESTALE) { - ret = 0; - goto out; - } else { - - defrag->total_failures++; + if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) && + (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX)) { + ret = gf_defrag_process_dir(this, defrag, loc, migrate_data, &perrno); - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_DEFRAG_PROCESS_DIR_FAILED, - "gf_defrag_process_dir failed for " - "directory: %s", loc->path); + if (ret && (ret != 2)) { + if (perrno == ENOENT || perrno == ESTALE) { + ret = 0; + goto out; + } else { + defrag->total_failures++; - if (conf->decommission_in_progress) { - goto out; - } + gf_msg(this->name, GF_LOG_ERROR, 0, + DHT_MSG_DEFRAG_PROCESS_DIR_FAILED, + "gf_defrag_process_dir failed for " + "directory: %s", + loc->path); - should_commit_hash = 0; - } - } else if (ret == 2) { - should_commit_hash = 0; + if (conf->decommission_in_progress) { + goto out; } - } - gf_msg_trace (this->name, 0, "fix layout called on %s", loc->path); + should_commit_hash = 0; + } + } else if (ret == 2) { + should_commit_hash = 0; + } + } - if (should_commit_hash && - gf_defrag_settle_hash (this, defrag, loc, fix_layout) != 0) { + gf_msg_trace(this->name, 0, "fix layout called on %s", loc->path); - defrag->total_failures++; + if (should_commit_hash && + gf_defrag_settle_hash(this, defrag, loc, fix_layout) != 0) { + defrag->total_failures++; - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_SETTLE_HASH_FAILED, - "Settle hash failed for %s", - loc->path); + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SETTLE_HASH_FAILED, + "Settle hash failed for %s", loc->path); - ret = -1; + ret = -1; - if (conf->decommission_in_progress) { - defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; - goto out; - } + if (conf->decommission_in_progress) { + defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; + goto out; } + } - ret = 0; + ret = 0; out: - if (free_entries) - gf_dirent_free (&entries); + if (free_entries) + gf_dirent_free(&entries); - loc_wipe (&entry_loc); + loc_wipe(&entry_loc); - if (fd) - fd_unref (fd); + if (fd) + fd_unref(fd); - if (ret == 0 && should_commit_hash == 0) { - ret = 2; - } - - return ret; + if (ret == 0 && should_commit_hash == 0) { + ret = 2; + } + return ret; } - - /****************************************************************************** * Tier background Fix layout functions ******************************************************************************/ /* This is the background tier fixlayout thread */ void * -gf_tier_do_fix_layout (void *args) +gf_tier_do_fix_layout(void *args) { - gf_tier_fix_layout_arg_t *tier_fix_layout_arg = args; - int ret = -1; - xlator_t *this = NULL; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; - dict_t *dict = NULL; - loc_t loc = {0,}; - struct iatt iatt = {0,}; - struct iatt parent = {0,}; - - GF_VALIDATE_OR_GOTO ("tier", tier_fix_layout_arg, out); - GF_VALIDATE_OR_GOTO ("tier", tier_fix_layout_arg->this, out); - this = tier_fix_layout_arg->this; - - conf = this->private; - GF_VALIDATE_OR_GOTO (this->name, conf, out); - - defrag = conf->defrag; - GF_VALIDATE_OR_GOTO (this->name, defrag, out); - GF_VALIDATE_OR_GOTO (this->name, defrag->root_inode, out); - - GF_VALIDATE_OR_GOTO (this->name, tier_fix_layout_arg->fix_layout, out); - - - /* Get Root loc_t */ - dht_build_root_loc (defrag->root_inode, &loc); - ret = syncop_lookup (this, &loc, &iatt, &parent, NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_REBALANCE_START_FAILED, - "Lookup on root failed."); - ret = -1; - goto out; - } - - - /* Start the crawl */ - gf_msg (this->name, GF_LOG_INFO, 0, - DHT_MSG_LOG_TIER_STATUS, "Tiering Fixlayout started"); - - ret = gf_defrag_fix_layout (this, defrag, &loc, - tier_fix_layout_arg->fix_layout, NULL); - if (ret && ret != 2) { - gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_FAILED, - "Tiering fixlayout failed."); - ret = -1; - goto out; - } - - if (ret != 2 && gf_defrag_settle_hash - (this, defrag, &loc, - tier_fix_layout_arg->fix_layout) != 0) { - defrag->total_failures++; - ret = -1; - goto out; - } - - dict = dict_new (); - if (!dict) { - ret = -1; - goto out; - } + gf_tier_fix_layout_arg_t *tier_fix_layout_arg = args; + int ret = -1; + xlator_t *this = NULL; + dht_conf_t *conf = NULL; + gf_defrag_info_t *defrag = NULL; + dict_t *dict = NULL; + loc_t loc = { + 0, + }; + struct iatt iatt = { + 0, + }; + struct iatt parent = { + 0, + }; + + GF_VALIDATE_OR_GOTO("tier", tier_fix_layout_arg, out); + GF_VALIDATE_OR_GOTO("tier", tier_fix_layout_arg->this, out); + this = tier_fix_layout_arg->this; + + conf = this->private; + GF_VALIDATE_OR_GOTO(this->name, conf, out); + + defrag = conf->defrag; + GF_VALIDATE_OR_GOTO(this->name, defrag, out); + GF_VALIDATE_OR_GOTO(this->name, defrag->root_inode, out); + + GF_VALIDATE_OR_GOTO(this->name, tier_fix_layout_arg->fix_layout, out); + + /* Get Root loc_t */ + dht_build_root_loc(defrag->root_inode, &loc); + ret = syncop_lookup(this, &loc, &iatt, &parent, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_START_FAILED, + "Lookup on root failed."); + ret = -1; + goto out; + } + + /* Start the crawl */ + gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS, + "Tiering Fixlayout started"); + + ret = gf_defrag_fix_layout(this, defrag, &loc, + tier_fix_layout_arg->fix_layout, NULL); + if (ret && ret != 2) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_FAILED, + "Tiering fixlayout failed."); + ret = -1; + goto out; + } - ret = dict_set_str (dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, "yes"); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_REBALANCE_FAILED, - "Failed to set dictionary value: key = %s", - GF_XATTR_TIER_LAYOUT_FIXED_KEY); - ret = -1; - goto out; - } + if (ret != 2 && + gf_defrag_settle_hash(this, defrag, &loc, + tier_fix_layout_arg->fix_layout) != 0) { + defrag->total_failures++; + ret = -1; + goto out; + } - /* Marking the completion of tiering fix layout via a xattr on root */ - ret = syncop_setxattr (this, &loc, dict, 0, NULL, NULL); - if (ret) { - gf_log (this->name, GF_LOG_ERROR, "Failed to set tiering fix " - "layout completed xattr on %s", loc.path); - ret = -1; - goto out; - } + dict = dict_new(); + if (!dict) { + ret = -1; + goto out; + } + + ret = dict_set_str(dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, "yes"); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_FAILED, + "Failed to set dictionary value: key = %s", + GF_XATTR_TIER_LAYOUT_FIXED_KEY); + ret = -1; + goto out; + } + + /* Marking the completion of tiering fix layout via a xattr on root */ + ret = syncop_setxattr(this, &loc, dict, 0, NULL, NULL); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "Failed to set tiering fix " + "layout completed xattr on %s", + loc.path); + ret = -1; + goto out; + } - ret = 0; + ret = 0; out: - if (ret && defrag) - defrag->total_failures++; + if (ret && defrag) + defrag->total_failures++; - if (dict) - dict_unref (dict); + if (dict) + dict_unref(dict); - return NULL; + return NULL; } int -gf_tier_start_fix_layout (xlator_t *this, - loc_t *loc, - gf_defrag_info_t *defrag, +gf_tier_start_fix_layout(xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag, dict_t *fix_layout) { - int ret = -1; - dict_t *tier_dict = NULL; - gf_tier_fix_layout_arg_t *tier_fix_layout_arg = NULL; - - tier_dict = dict_new (); - if (!tier_dict) { - gf_log ("tier", GF_LOG_ERROR, "Tier fix layout failed :" - "Creation of tier_dict failed"); - ret = -1; - goto out; - } - - /* Check if layout is fixed already */ - ret = syncop_getxattr (this, loc, &tier_dict, - GF_XATTR_TIER_LAYOUT_FIXED_KEY, - NULL, NULL); - if (ret != 0) { - - tier_fix_layout_arg = &defrag->tier_conf.tier_fix_layout_arg; - - /*Fill crawl arguments */ - tier_fix_layout_arg->this = this; - tier_fix_layout_arg->fix_layout = fix_layout; - - /* Spawn the fix layout thread so that its done in the - * background */ - ret = gf_thread_create (&tier_fix_layout_arg->thread_id, NULL, - gf_tier_do_fix_layout, - tier_fix_layout_arg, "tierfixl"); - if (ret) { - gf_log ("tier", GF_LOG_ERROR, "Thread creation failed. " - "Background fix layout for tiering will not " - "work."); - defrag->total_failures++; - goto out; - } - } - ret = 0; + int ret = -1; + dict_t *tier_dict = NULL; + gf_tier_fix_layout_arg_t *tier_fix_layout_arg = NULL; + + tier_dict = dict_new(); + if (!tier_dict) { + gf_log("tier", GF_LOG_ERROR, + "Tier fix layout failed :" + "Creation of tier_dict failed"); + ret = -1; + goto out; + } + + /* Check if layout is fixed already */ + ret = syncop_getxattr(this, loc, &tier_dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, + NULL, NULL); + if (ret != 0) { + tier_fix_layout_arg = &defrag->tier_conf.tier_fix_layout_arg; + + /*Fill crawl arguments */ + tier_fix_layout_arg->this = this; + tier_fix_layout_arg->fix_layout = fix_layout; + + /* Spawn the fix layout thread so that its done in the + * background */ + ret = gf_thread_create(&tier_fix_layout_arg->thread_id, NULL, + gf_tier_do_fix_layout, tier_fix_layout_arg, + "tierfixl"); + if (ret) { + gf_log("tier", GF_LOG_ERROR, + "Thread creation failed. " + "Background fix layout for tiering will not " + "work."); + defrag->total_failures++; + goto out; + } + } + ret = 0; out: - if (tier_dict) - dict_unref (tier_dict); + if (tier_dict) + dict_unref(tier_dict); - return ret; + return ret; } void -gf_tier_clear_fix_layout (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag) +gf_tier_clear_fix_layout(xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag) { - int ret = -1; - dict_t *dict = NULL; - - GF_VALIDATE_OR_GOTO ("tier", this, out); - GF_VALIDATE_OR_GOTO (this->name, loc, out); - GF_VALIDATE_OR_GOTO (this->name, defrag, out); - - /* Check if background fixlayout is completed. This is not - * multi-process safe i.e there is a possibility that by the time - * we move to remove the xattr there it might have been cleared by some - * other detach process from other node. We ignore the error if such - * a thing happens */ - ret = syncop_getxattr (this, loc, &dict, - GF_XATTR_TIER_LAYOUT_FIXED_KEY, NULL, NULL); - if (ret) { - /* Background fixlayout not complete - nothing to clear*/ - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_LOG_TIER_STATUS, - "Unable to retrieve fixlayout xattr." - "Assume background fix layout not complete"); - goto out; - } - - ret = syncop_removexattr (this, loc, GF_XATTR_TIER_LAYOUT_FIXED_KEY, - NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_WARNING, -ret, - DHT_MSG_LOG_TIER_STATUS, - "Failed removing tier fix layout " - "xattr from %s", loc->path); - goto out; - } - ret = 0; + int ret = -1; + dict_t *dict = NULL; + + GF_VALIDATE_OR_GOTO("tier", this, out); + GF_VALIDATE_OR_GOTO(this->name, loc, out); + GF_VALIDATE_OR_GOTO(this->name, defrag, out); + + /* Check if background fixlayout is completed. This is not + * multi-process safe i.e there is a possibility that by the time + * we move to remove the xattr there it might have been cleared by some + * other detach process from other node. We ignore the error if such + * a thing happens */ + ret = syncop_getxattr(this, loc, &dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, + NULL, NULL); + if (ret) { + /* Background fixlayout not complete - nothing to clear*/ + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_LOG_TIER_STATUS, + "Unable to retrieve fixlayout xattr." + "Assume background fix layout not complete"); + goto out; + } + + ret = syncop_removexattr(this, loc, GF_XATTR_TIER_LAYOUT_FIXED_KEY, NULL, + NULL); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_LOG_TIER_STATUS, + "Failed removing tier fix layout " + "xattr from %s", + loc->path); + goto out; + } + ret = 0; out: - if (dict) - dict_unref (dict); + if (dict) + dict_unref(dict); } void -gf_tier_wait_fix_lookup (gf_defrag_info_t *defrag) { - if (defrag->tier_conf.tier_fix_layout_arg.thread_id) { - pthread_join (defrag->tier_conf.tier_fix_layout_arg.thread_id, - NULL); - } +gf_tier_wait_fix_lookup(gf_defrag_info_t *defrag) +{ + if (defrag->tier_conf.tier_fix_layout_arg.thread_id) { + pthread_join(defrag->tier_conf.tier_fix_layout_arg.thread_id, NULL); + } } /******************Tier background Fix layout functions END********************/ int -dht_init_local_subvols_and_nodeuuids (xlator_t *this, dht_conf_t *conf, - loc_t *loc) +dht_init_local_subvols_and_nodeuuids(xlator_t *this, dht_conf_t *conf, + loc_t *loc) { - - dict_t *dict = NULL; - gf_defrag_info_t *defrag = NULL; - uuid_t *uuid_ptr = NULL; - int ret = -1; - int i = 0; - int j = 0; - - defrag = conf->defrag; - - if (defrag->cmd != GF_DEFRAG_CMD_START_TIER) { - /* Find local subvolumes */ - ret = syncop_getxattr (this, loc, &dict, - GF_REBAL_FIND_LOCAL_SUBVOL, - NULL, NULL); - if (ret && (ret != -ENODATA)) { - gf_msg (this->name, GF_LOG_ERROR, -ret, 0, "local " - "subvolume determination failed with error: %d", - -ret); - ret = -1; - goto out; - } - - if (!ret) - goto out; + dict_t *dict = NULL; + gf_defrag_info_t *defrag = NULL; + uuid_t *uuid_ptr = NULL; + int ret = -1; + int i = 0; + int j = 0; + + defrag = conf->defrag; + + if (defrag->cmd != GF_DEFRAG_CMD_START_TIER) { + /* Find local subvolumes */ + ret = syncop_getxattr(this, loc, &dict, GF_REBAL_FIND_LOCAL_SUBVOL, + NULL, NULL); + if (ret && (ret != -ENODATA)) { + gf_msg(this->name, GF_LOG_ERROR, -ret, 0, + "local " + "subvolume determination failed with error: %d", + -ret); + ret = -1; + goto out; } - ret = syncop_getxattr (this, loc, &dict, - GF_REBAL_OLD_FIND_LOCAL_SUBVOL, - NULL, NULL); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, 0, "local " - "subvolume determination failed with error: %d", - -ret); - ret = -1; - goto out; - } - ret = 0; + if (!ret) + goto out; + } + + ret = syncop_getxattr(this, loc, &dict, GF_REBAL_OLD_FIND_LOCAL_SUBVOL, + NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, 0, + "local " + "subvolume determination failed with error: %d", + -ret); + ret = -1; + goto out; + } + ret = 0; out: - if (ret) { - return ret; - } + if (ret) { + return ret; + } - for (i = 0 ; i < conf->local_subvols_cnt; i++) { - gf_msg (this->name, GF_LOG_INFO, 0, 0, "local subvol: " - "%s", conf->local_subvols[i]->name); + for (i = 0; i < conf->local_subvols_cnt; i++) { + gf_msg(this->name, GF_LOG_INFO, 0, 0, + "local subvol: " + "%s", + conf->local_subvols[i]->name); - for (j = 0; j < conf->local_nodeuuids[i].count; j++) { - uuid_ptr = &(conf->local_nodeuuids[i].elements[j].uuid); - gf_msg (this->name, GF_LOG_INFO, 0, 0, - "node uuid : %s", - uuid_utoa(*uuid_ptr)); - } + for (j = 0; j < conf->local_nodeuuids[i].count; j++) { + uuid_ptr = &(conf->local_nodeuuids[i].elements[j].uuid); + gf_msg(this->name, GF_LOG_INFO, 0, 0, "node uuid : %s", + uuid_utoa(*uuid_ptr)); } + } - return ret; + return ret; } - /* Functions for the rebalance estimates feature */ uint64_t -gf_defrag_subvol_file_size (xlator_t *this, loc_t *root_loc) +gf_defrag_subvol_file_size(xlator_t *this, loc_t *root_loc) { - int ret = -1; - struct statvfs buf = {0,}; + int ret = -1; + struct statvfs buf = { + 0, + }; - if (!this) - return 0; + if (!this) + return 0; - ret = syncop_statfs (this, root_loc, &buf, NULL, NULL); - if (ret) { - /* Aargh! */ - return 0; - } - return ((buf.f_blocks - buf.f_bfree) * buf.f_frsize); + ret = syncop_statfs(this, root_loc, &buf, NULL, NULL); + if (ret) { + /* Aargh! */ + return 0; + } + return ((buf.f_blocks - buf.f_bfree) * buf.f_frsize); } uint64_t -gf_defrag_total_file_size (xlator_t *this, loc_t *root_loc) +gf_defrag_total_file_size(xlator_t *this, loc_t *root_loc) { - dht_conf_t *conf = NULL; - int i = 0; - uint64_t size_files = 0; - uint64_t total_size = 0; - - conf = this->private; - if (!conf) { - return 0; - } - - for (i = 0 ; i < conf->local_subvols_cnt; i++) { - size_files = gf_defrag_subvol_file_size (conf->local_subvols[i], - root_loc); - total_size += size_files; - gf_msg (this->name, GF_LOG_INFO, 0, 0, "local subvol: %s," - "cnt = %"PRIu64, conf->local_subvols[i]->name, - size_files); - } - - gf_msg (this->name, GF_LOG_INFO, 0, 0, - "Total size files = %"PRIu64, total_size); + dht_conf_t *conf = NULL; + int i = 0; + uint64_t size_files = 0; + uint64_t total_size = 0; - return total_size; + conf = this->private; + if (!conf) { + return 0; + } + + for (i = 0; i < conf->local_subvols_cnt; i++) { + size_files = gf_defrag_subvol_file_size(conf->local_subvols[i], + root_loc); + total_size += size_files; + gf_msg(this->name, GF_LOG_INFO, 0, 0, + "local subvol: %s," + "cnt = %" PRIu64, + conf->local_subvols[i]->name, size_files); + } + + gf_msg(this->name, GF_LOG_INFO, 0, 0, "Total size files = %" PRIu64, + total_size); + + return total_size; } - -static void* -dht_file_counter_thread (void *args) +static void * +dht_file_counter_thread(void *args) { - gf_defrag_info_t *defrag = NULL; - loc_t root_loc = {0,}; - struct timespec time_to_wait = {0,}; - struct timeval now = {0,}; - uint64_t tmp_size = 0; - - - if (!args) - return NULL; - - defrag = (gf_defrag_info_t *) args; - dht_build_root_loc (defrag->root_inode, &root_loc); - - while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) { - - gettimeofday (&now, NULL); - time_to_wait.tv_sec = now.tv_sec + 600; - time_to_wait.tv_nsec = 0; + gf_defrag_info_t *defrag = NULL; + loc_t root_loc = { + 0, + }; + struct timespec time_to_wait = { + 0, + }; + struct timeval now = { + 0, + }; + uint64_t tmp_size = 0; + + if (!args) + return NULL; + defrag = (gf_defrag_info_t *)args; + dht_build_root_loc(defrag->root_inode, &root_loc); - pthread_mutex_lock (&defrag->fc_mutex); - pthread_cond_timedwait (&defrag->fc_wakeup_cond, - &defrag->fc_mutex, - &time_to_wait); + while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) { + gettimeofday(&now, NULL); + time_to_wait.tv_sec = now.tv_sec + 600; + time_to_wait.tv_nsec = 0; - pthread_mutex_unlock (&defrag->fc_mutex); + pthread_mutex_lock(&defrag->fc_mutex); + pthread_cond_timedwait(&defrag->fc_wakeup_cond, &defrag->fc_mutex, + &time_to_wait); + pthread_mutex_unlock(&defrag->fc_mutex); - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) - break; + if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) + break; - tmp_size = gf_defrag_total_file_size (defrag->this, - &root_loc); + tmp_size = gf_defrag_total_file_size(defrag->this, &root_loc); - gf_log ("dht", GF_LOG_INFO, - "tmp data size =%"PRIu64, - tmp_size); + gf_log("dht", GF_LOG_INFO, "tmp data size =%" PRIu64, tmp_size); - if (!tmp_size) { - gf_msg ("dht", GF_LOG_ERROR, 0, 0, "Failed to get " - "the total data size. Unable to estimate " - "time to complete rebalance."); - } else { - g_totalsize = tmp_size; - gf_msg_debug ("dht", 0, - "total data size =%"PRIu64, - g_totalsize); - } + if (!tmp_size) { + gf_msg("dht", GF_LOG_ERROR, 0, 0, + "Failed to get " + "the total data size. Unable to estimate " + "time to complete rebalance."); + } else { + g_totalsize = tmp_size; + gf_msg_debug("dht", 0, "total data size =%" PRIu64, g_totalsize); } + } - return NULL; + return NULL; } int -gf_defrag_estimates_cleanup (xlator_t *this, gf_defrag_info_t *defrag, - pthread_t filecnt_thread) +gf_defrag_estimates_cleanup(xlator_t *this, gf_defrag_info_t *defrag, + pthread_t filecnt_thread) { - int ret = -1; - - /* Wake up the filecounter thread. - * By now the defrag status will no longer be - * GF_DEFRAG_STATUS_STARTED so the thread will exit the loop. - */ - pthread_mutex_lock (&defrag->fc_mutex); - { - pthread_cond_broadcast (&defrag->fc_wakeup_cond); - } - pthread_mutex_unlock (&defrag->fc_mutex); - - ret = pthread_join (filecnt_thread, NULL); - if (ret) { - gf_msg ("dht", GF_LOG_ERROR, ret, 0, - "file_counter_thread: pthread_join failed."); - ret = -1; - } - return ret; + int ret = -1; + + /* Wake up the filecounter thread. + * By now the defrag status will no longer be + * GF_DEFRAG_STATUS_STARTED so the thread will exit the loop. + */ + pthread_mutex_lock(&defrag->fc_mutex); + { + pthread_cond_broadcast(&defrag->fc_wakeup_cond); + } + pthread_mutex_unlock(&defrag->fc_mutex); + + ret = pthread_join(filecnt_thread, NULL); + if (ret) { + gf_msg("dht", GF_LOG_ERROR, ret, 0, + "file_counter_thread: pthread_join failed."); + ret = -1; + } + return ret; } - int -gf_defrag_estimates_init (xlator_t *this, loc_t *loc, - pthread_t *filecnt_thread) +gf_defrag_estimates_init(xlator_t *this, loc_t *loc, pthread_t *filecnt_thread) { - int ret = -1; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; - - conf = this->private; - defrag = conf->defrag; - - g_totalsize = gf_defrag_total_file_size (this, loc); - if (!g_totalsize) { - gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Failed to get " - "the total data size. Unable to estimate " - "time to complete rebalance."); - goto out; - } - - ret = gf_thread_create (filecnt_thread, NULL, - &dht_file_counter_thread, - (void *)defrag, "dhtfcnt"); - - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, ret, 0, "Failed to " - "create the file counter thread "); - ret = -1; - goto out; - } - ret = 0; + int ret = -1; + dht_conf_t *conf = NULL; + gf_defrag_info_t *defrag = NULL; + + conf = this->private; + defrag = conf->defrag; + + g_totalsize = gf_defrag_total_file_size(this, loc); + if (!g_totalsize) { + gf_msg(this->name, GF_LOG_ERROR, 0, 0, + "Failed to get " + "the total data size. Unable to estimate " + "time to complete rebalance."); + goto out; + } + + ret = gf_thread_create(filecnt_thread, NULL, &dht_file_counter_thread, + (void *)defrag, "dhtfcnt"); + + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, ret, 0, + "Failed to " + "create the file counter thread "); + ret = -1; + goto out; + } + ret = 0; out: - return ret; + return ret; } - /* Init and cleanup functions for parallel file migration*/ int -gf_defrag_parallel_migration_init (xlator_t *this, gf_defrag_info_t *defrag, - pthread_t **tid_array, int *thread_index) +gf_defrag_parallel_migration_init(xlator_t *this, gf_defrag_info_t *defrag, + pthread_t **tid_array, int *thread_index) { - int ret = -1; - int thread_spawn_count = 0; - int index = 0; - pthread_t *tid = NULL; - char thread_name[GF_THREAD_NAMEMAX] = {0,}; - - if (!defrag) - goto out; - - /* Initialize global entry queue */ - defrag->queue = GF_CALLOC (1, sizeof (struct dht_container), - gf_dht_mt_container_t); - - if (!defrag->queue) { - gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0, - "Failed to initialise migration queue"); - ret = -1; - goto out; - } + int ret = -1; + int thread_spawn_count = 0; + int index = 0; + pthread_t *tid = NULL; + char thread_name[GF_THREAD_NAMEMAX] = { + 0, + }; + + if (!defrag) + goto out; + + /* Initialize global entry queue */ + defrag->queue = GF_CALLOC(1, sizeof(struct dht_container), + gf_dht_mt_container_t); + + if (!defrag->queue) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0, + "Failed to initialise migration queue"); + ret = -1; + goto out; + } - INIT_LIST_HEAD (&(defrag->queue[0].list)); + INIT_LIST_HEAD(&(defrag->queue[0].list)); - thread_spawn_count = MAX (MAX_REBAL_THREADS, 4); + thread_spawn_count = MAX(MAX_REBAL_THREADS, 4); - gf_msg_debug (this->name, 0, "thread_spawn_count: %d", - thread_spawn_count); + gf_msg_debug(this->name, 0, "thread_spawn_count: %d", thread_spawn_count); - tid = GF_CALLOC (thread_spawn_count, sizeof (pthread_t), - gf_common_mt_pthread_t); - if (!tid) { - gf_msg (this->name, GF_LOG_ERROR, ENOMEM, 0, - "Failed to create migration threads"); - ret = -1; - goto out; - } - defrag->current_thread_count = thread_spawn_count; - - /*Spawn Threads Here*/ - while (index < thread_spawn_count) { - snprintf (thread_name, sizeof(thread_name), - "dhtmig%d", ((index + 1) & 0x3ff)); - ret = gf_thread_create (&(tid[index]), NULL, - &gf_defrag_task, (void *)defrag, - thread_name); - if (ret != 0) { - gf_msg ("DHT", GF_LOG_ERROR, ret, 0, - "Thread[%d] creation failed. ", - index); - ret = -1; - goto out; - } else { - gf_log ("DHT", GF_LOG_INFO, "Thread[%d] " - "creation successful", index); - } - index++; + tid = GF_CALLOC(thread_spawn_count, sizeof(pthread_t), + gf_common_mt_pthread_t); + if (!tid) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0, + "Failed to create migration threads"); + ret = -1; + goto out; + } + defrag->current_thread_count = thread_spawn_count; + + /*Spawn Threads Here*/ + while (index < thread_spawn_count) { + snprintf(thread_name, sizeof(thread_name), "dhtmig%d", + ((index + 1) & 0x3ff)); + ret = gf_thread_create(&(tid[index]), NULL, &gf_defrag_task, + (void *)defrag, thread_name); + if (ret != 0) { + gf_msg("DHT", GF_LOG_ERROR, ret, 0, "Thread[%d] creation failed. ", + index); + ret = -1; + goto out; + } else { + gf_log("DHT", GF_LOG_INFO, + "Thread[%d] " + "creation successful", + index); } + index++; + } - ret = 0; + ret = 0; out: - *thread_index = index; - *tid_array = tid; + *thread_index = index; + *tid_array = tid; - return ret; + return ret; } int -gf_defrag_parallel_migration_cleanup (gf_defrag_info_t *defrag, - pthread_t *tid_array, int thread_index) +gf_defrag_parallel_migration_cleanup(gf_defrag_info_t *defrag, + pthread_t *tid_array, int thread_index) { - int ret = -1; - int i = 0; + int ret = -1; + int i = 0; - if (!defrag) - goto out; + if (!defrag) + goto out; - /* Wake up all migration threads */ - pthread_mutex_lock (&defrag->dfq_mutex); - { - defrag->crawl_done = 1; + /* Wake up all migration threads */ + pthread_mutex_lock(&defrag->dfq_mutex); + { + defrag->crawl_done = 1; - pthread_cond_broadcast (&defrag->parallel_migration_cond); - pthread_cond_broadcast (&defrag->df_wakeup_thread); - } - pthread_mutex_unlock (&defrag->dfq_mutex); + pthread_cond_broadcast(&defrag->parallel_migration_cond); + pthread_cond_broadcast(&defrag->df_wakeup_thread); + } + pthread_mutex_unlock(&defrag->dfq_mutex); - /*Wait for all the threads to complete their task*/ - for (i = 0; i < thread_index; i++) { - pthread_join (tid_array[i], NULL); - } + /*Wait for all the threads to complete their task*/ + for (i = 0; i < thread_index; i++) { + pthread_join(tid_array[i], NULL); + } - GF_FREE (tid_array); + GF_FREE(tid_array); - /* Cleanup the migration queue */ - if (defrag->queue) { - gf_dirent_free (defrag->queue[0].df_entry); - INIT_LIST_HEAD (&(defrag->queue[0].list)); - } + /* Cleanup the migration queue */ + if (defrag->queue) { + gf_dirent_free(defrag->queue[0].df_entry); + INIT_LIST_HEAD(&(defrag->queue[0].list)); + } - GF_FREE (defrag->queue); + GF_FREE(defrag->queue); - ret = 0; + ret = 0; out: - return ret; + return ret; } - - int -gf_defrag_start_crawl (void *data) +gf_defrag_start_crawl(void *data) { - xlator_t *this = NULL; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; - dict_t *fix_layout = NULL; - dict_t *migrate_data = NULL; - dict_t *status = NULL; - glusterfs_ctx_t *ctx = NULL; - dht_methods_t *methods = NULL; - call_frame_t *statfs_frame = NULL; - xlator_t *old_THIS = NULL; - int ret = -1; - loc_t loc = {0,}; - struct iatt iatt = {0,}; - struct iatt parent = {0,}; - int thread_index = 0; - pthread_t *tid = NULL; - pthread_t filecnt_thread; - gf_boolean_t is_tier_detach = _gf_false; - gf_boolean_t fc_thread_started = _gf_false; - - this = data; - if (!this) - goto exit; - - ctx = this->ctx; - if (!ctx) - goto exit; - - conf = this->private; - if (!conf) - goto exit; - - defrag = conf->defrag; - if (!defrag) - goto exit; - - gettimeofday (&defrag->start_time, NULL); - dht_build_root_inode (this, &defrag->root_inode); - if (!defrag->root_inode) - goto out; + xlator_t *this = NULL; + dht_conf_t *conf = NULL; + gf_defrag_info_t *defrag = NULL; + dict_t *fix_layout = NULL; + dict_t *migrate_data = NULL; + dict_t *status = NULL; + glusterfs_ctx_t *ctx = NULL; + dht_methods_t *methods = NULL; + call_frame_t *statfs_frame = NULL; + xlator_t *old_THIS = NULL; + int ret = -1; + loc_t loc = { + 0, + }; + struct iatt iatt = { + 0, + }; + struct iatt parent = { + 0, + }; + int thread_index = 0; + pthread_t *tid = NULL; + pthread_t filecnt_thread; + gf_boolean_t is_tier_detach = _gf_false; + gf_boolean_t fc_thread_started = _gf_false; + + this = data; + if (!this) + goto exit; + + ctx = this->ctx; + if (!ctx) + goto exit; + + conf = this->private; + if (!conf) + goto exit; + + defrag = conf->defrag; + if (!defrag) + goto exit; + + gettimeofday(&defrag->start_time, NULL); + dht_build_root_inode(this, &defrag->root_inode); + if (!defrag->root_inode) + goto out; + + dht_build_root_loc(defrag->root_inode, &loc); + + /* fix-layout on '/' first */ + + ret = syncop_lookup(this, &loc, &iatt, &parent, NULL, NULL); + + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_START_FAILED, + "Failed to start rebalance: look up on / failed"); + ret = -1; + goto out; + } - dht_build_root_loc (defrag->root_inode, &loc); + old_THIS = THIS; + THIS = this; - /* fix-layout on '/' first */ + statfs_frame = create_frame(this, this->ctx->pool); + if (!statfs_frame) { + gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM, + "Insufficient memory. Frame creation failed"); + ret = -1; + goto out; + } - ret = syncop_lookup (this, &loc, &iatt, &parent, NULL, NULL); + /* async statfs update for honoring min-free-disk */ + dht_get_du_info(statfs_frame, this, &loc); + THIS = old_THIS; - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_REBALANCE_START_FAILED, - "Failed to start rebalance: look up on / failed"); - ret = -1; - goto out; - } + fix_layout = dict_new(); + if (!fix_layout) { + ret = -1; + goto out; + } + + /* + * Unfortunately, we can't do special xattrs (like fix.layout) and + * real ones in the same call currently, and changing it seems + * riskier than just doing two calls. + */ + + gf_log(this->name, GF_LOG_INFO, "%s using commit hash %u", __func__, + conf->vol_commit_hash); + + ret = dict_set_uint32(fix_layout, conf->commithash_xattr_name, + conf->vol_commit_hash); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, "Failed to set %s", + conf->commithash_xattr_name); + defrag->total_failures++; + ret = -1; + goto out; + } + + ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "Failed to set commit hash on %s. " + "Rebalance cannot proceed.", + loc.path); + defrag->total_failures++; + ret = -1; + goto out; + } + + /* We now return to our regularly scheduled program. */ + + ret = dict_set_str(fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes"); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED, + "Failed to start rebalance:" + "Failed to set dictionary value: key = %s", + GF_XATTR_FIX_LAYOUT_KEY); + defrag->total_failures++; + ret = -1; + goto out; + } - old_THIS = THIS; - THIS = this; + defrag->new_commit_hash = conf->vol_commit_hash; - statfs_frame = create_frame (this, this->ctx->pool); - if (!statfs_frame) { - gf_msg (this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM, - "Insufficient memory. Frame creation failed"); - ret = -1; - goto out; - } + ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_FAILED, + "fix layout on %s failed", loc.path); + defrag->total_failures++; + ret = -1; + goto out; + } - /* async statfs update for honoring min-free-disk */ - dht_get_du_info (statfs_frame, this, &loc); - THIS = old_THIS; + if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) { + /* We need to migrate files */ - fix_layout = dict_new (); - if (!fix_layout) { - ret = -1; - goto out; + migrate_data = dict_new(); + if (!migrate_data) { + defrag->total_failures++; + ret = -1; + goto out; } - - /* - * Unfortunately, we can't do special xattrs (like fix.layout) and - * real ones in the same call currently, and changing it seems - * riskier than just doing two calls. - */ - - gf_log (this->name, GF_LOG_INFO, "%s using commit hash %u", - __func__, conf->vol_commit_hash); - - ret = dict_set_uint32 (fix_layout, conf->commithash_xattr_name, - conf->vol_commit_hash); + ret = dict_set_str( + migrate_data, GF_XATTR_FILE_MIGRATE_KEY, + (defrag->cmd == GF_DEFRAG_CMD_START_FORCE) ? "force" : "non-force"); if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "Failed to set %s", conf->commithash_xattr_name); - defrag->total_failures++; - ret = -1; - goto out; + defrag->total_failures++; + ret = -1; + goto out; } - ret = syncop_setxattr (this, &loc, fix_layout, 0, NULL, NULL); + ret = dht_init_local_subvols_and_nodeuuids(this, conf, &loc); if (ret) { - gf_log (this->name, GF_LOG_ERROR, - "Failed to set commit hash on %s. " - "Rebalance cannot proceed.", - loc.path); - defrag->total_failures++; - ret = -1; - goto out; + ret = -1; + goto out; } - /* We now return to our regularly scheduled program. */ - - ret = dict_set_str (fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes"); + /* Initialise the structures required for parallel migration */ + ret = gf_defrag_parallel_migration_init(this, defrag, &tid, + &thread_index); if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_REBALANCE_START_FAILED, - "Failed to start rebalance:" - "Failed to set dictionary value: key = %s", - GF_XATTR_FIX_LAYOUT_KEY); - defrag->total_failures++; - ret = -1; - goto out; + gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Aborting rebalance."); + goto out; } - defrag->new_commit_hash = conf->vol_commit_hash; - - ret = syncop_setxattr (this, &loc, fix_layout, 0, NULL, NULL); + ret = gf_defrag_estimates_init(this, &loc, &filecnt_thread); if (ret) { - gf_msg (this->name, GF_LOG_ERROR, -ret, - DHT_MSG_REBALANCE_FAILED, - "fix layout on %s failed", - loc.path); - defrag->total_failures++; - ret = -1; - goto out; - } - - if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) { - /* We need to migrate files */ - - migrate_data = dict_new (); - if (!migrate_data) { - defrag->total_failures++; - ret = -1; - goto out; - } - ret = dict_set_str (migrate_data, GF_XATTR_FILE_MIGRATE_KEY, - (defrag->cmd == GF_DEFRAG_CMD_START_FORCE) - ? "force" : "non-force"); - if (ret) { - defrag->total_failures++; - ret = -1; - goto out; - } - - ret = dht_init_local_subvols_and_nodeuuids (this, conf, &loc); - if (ret) { - ret = -1; - goto out; - } - - /* Initialise the structures required for parallel migration */ - ret = gf_defrag_parallel_migration_init (this, defrag, &tid, - &thread_index); - if (ret) { - gf_msg (this->name, GF_LOG_ERROR, 0, 0, - "Aborting rebalance."); - goto out; - } - - ret = gf_defrag_estimates_init (this, &loc, &filecnt_thread); - if (ret) { - /* Not a fatal error. Allow the rebalance to proceed*/ - ret = 0; - } else { - fc_thread_started = _gf_true; - } - } - - - if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { - /* Fix layout for attach tier */ - ret = gf_tier_start_fix_layout (this, &loc, defrag, fix_layout); - if (ret) { - goto out; - } - - methods = &(conf->methods); - - /* Calling tier_start of tier.c */ - methods->migration_other(this, defrag); - if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || - defrag->cmd == GF_DEFRAG_CMD_DETACH_START) { - - ret = dict_set_str (migrate_data, - GF_XATTR_FILE_MIGRATE_KEY, - "force"); - if (ret) - goto out; - - } + /* Not a fatal error. Allow the rebalance to proceed*/ + ret = 0; } else { - ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout, - migrate_data); - if (ret && ret != 2) { - defrag->total_failures++; - ret = -1; - goto out; - } - - if (ret != 2 && gf_defrag_settle_hash - (this, defrag, &loc, fix_layout) != 0) { - defrag->total_failures++; - ret = -1; - goto out; - } - - if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || - defrag->cmd == GF_DEFRAG_CMD_DETACH_START) - is_tier_detach = _gf_true; - + fc_thread_started = _gf_true; } + } - gf_log ("DHT", GF_LOG_INFO, "crawling file-system completed"); -out: - - /* We are here means crawling the entire file system is done - or something failed. Set defrag->crawl_done flag to intimate - the migrator threads to exhaust the defrag->queue and terminate*/ - + if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { + /* Fix layout for attach tier */ + ret = gf_tier_start_fix_layout(this, &loc, defrag, fix_layout); if (ret) { - defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; + goto out; } + methods = &(conf->methods); - if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { - /* Wait for the tier fixlayout to - * complete if its was started.*/ - gf_tier_wait_fix_lookup (defrag); - } - - if (is_tier_detach && ret == 0) { - /* If it was a detach remove the tier fix-layout - * xattr on root. Ignoring the failure, as nothing has to be - * done, logging is done in gf_tier_clear_fix_layout */ - gf_tier_clear_fix_layout (this, &loc, defrag); - } - - gf_defrag_parallel_migration_cleanup (defrag, tid, thread_index); - - if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) && - (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) { - defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE; + /* Calling tier_start of tier.c */ + methods->migration_other(this, defrag); + if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || + defrag->cmd == GF_DEFRAG_CMD_DETACH_START) { + ret = dict_set_str(migrate_data, GF_XATTR_FILE_MIGRATE_KEY, + "force"); + if (ret) + goto out; } - - if (fc_thread_started) { - gf_defrag_estimates_cleanup (this, defrag, filecnt_thread); + } else { + ret = gf_defrag_fix_layout(this, defrag, &loc, fix_layout, + migrate_data); + if (ret && ret != 2) { + defrag->total_failures++; + ret = -1; + goto out; } - dht_send_rebalance_event (this, defrag->cmd, defrag->defrag_status); - - LOCK (&defrag->lock); - { - status = dict_new (); - gf_defrag_status_get (conf, status); - if (ctx && ctx->notify) - ctx->notify (GF_EN_DEFRAG_STATUS, status); - if (status) - dict_unref (status); - defrag->is_exiting = 1; + if (ret != 2 && + gf_defrag_settle_hash(this, defrag, &loc, fix_layout) != 0) { + defrag->total_failures++; + ret = -1; + goto out; } - UNLOCK (&defrag->lock); + if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER || + defrag->cmd == GF_DEFRAG_CMD_DETACH_START) + is_tier_detach = _gf_true; + } - GF_FREE (defrag); - conf->defrag = NULL; - - if (migrate_data) - dict_unref (migrate_data); + gf_log("DHT", GF_LOG_INFO, "crawling file-system completed"); +out: - if (statfs_frame) { - STACK_DESTROY (statfs_frame->root); - } + /* We are here means crawling the entire file system is done + or something failed. Set defrag->crawl_done flag to intimate + the migrator threads to exhaust the defrag->queue and terminate*/ + + if (ret) { + defrag->defrag_status = GF_DEFRAG_STATUS_FAILED; + } + + if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { + /* Wait for the tier fixlayout to + * complete if its was started.*/ + gf_tier_wait_fix_lookup(defrag); + } + + if (is_tier_detach && ret == 0) { + /* If it was a detach remove the tier fix-layout + * xattr on root. Ignoring the failure, as nothing has to be + * done, logging is done in gf_tier_clear_fix_layout */ + gf_tier_clear_fix_layout(this, &loc, defrag); + } + + gf_defrag_parallel_migration_cleanup(defrag, tid, thread_index); + + if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) && + (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) { + defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE; + } + + if (fc_thread_started) { + gf_defrag_estimates_cleanup(this, defrag, filecnt_thread); + } + + dht_send_rebalance_event(this, defrag->cmd, defrag->defrag_status); + + LOCK(&defrag->lock); + { + status = dict_new(); + gf_defrag_status_get(conf, status); + if (ctx && ctx->notify) + ctx->notify(GF_EN_DEFRAG_STATUS, status); + if (status) + dict_unref(status); + defrag->is_exiting = 1; + } + UNLOCK(&defrag->lock); + + GF_FREE(defrag); + conf->defrag = NULL; + + if (migrate_data) + dict_unref(migrate_data); + + if (statfs_frame) { + STACK_DESTROY(statfs_frame->root); + } exit: - return ret; + return ret; } - - static int -gf_defrag_done (int ret, call_frame_t *sync_frame, void *data) +gf_defrag_done(int ret, call_frame_t *sync_frame, void *data) { - gf_listener_stop (sync_frame->this); + gf_listener_stop(sync_frame->this); - STACK_DESTROY (sync_frame->root); - kill (getpid(), SIGTERM); - return 0; + STACK_DESTROY(sync_frame->root); + kill(getpid(), SIGTERM); + return 0; } void * -gf_defrag_start (void *data) +gf_defrag_start(void *data) { - int ret = -1; - call_frame_t *frame = NULL; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; - xlator_t *this = NULL; - xlator_t *old_THIS = NULL; - - this = data; - conf = this->private; - if (!conf) - goto out; + int ret = -1; + call_frame_t *frame = NULL; + dht_conf_t *conf = NULL; + gf_defrag_info_t *defrag = NULL; + xlator_t *this = NULL; + xlator_t *old_THIS = NULL; - defrag = conf->defrag; - if (!defrag) - goto out; + this = data; + conf = this->private; + if (!conf) + goto out; - frame = create_frame (this, this->ctx->pool); - if (!frame) - goto out; + defrag = conf->defrag; + if (!defrag) + goto out; + + frame = create_frame(this, this->ctx->pool); + if (!frame) + goto out; - frame->root->pid = GF_CLIENT_PID_DEFRAG; + frame->root->pid = GF_CLIENT_PID_DEFRAG; - defrag->pid = frame->root->pid; + defrag->pid = frame->root->pid; - defrag->defrag_status = GF_DEFRAG_STATUS_STARTED; + defrag->defrag_status = GF_DEFRAG_STATUS_STARTED; - old_THIS = THIS; - THIS = this; - ret = synctask_new (this->ctx->env, gf_defrag_start_crawl, - gf_defrag_done, frame, this); + old_THIS = THIS; + THIS = this; + ret = synctask_new(this->ctx->env, gf_defrag_start_crawl, gf_defrag_done, + frame, this); - if (ret) - gf_msg (this->name, GF_LOG_ERROR, 0, - DHT_MSG_REBALANCE_START_FAILED, - "Could not create task for rebalance"); - THIS = old_THIS; + if (ret) + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED, + "Could not create task for rebalance"); + THIS = old_THIS; out: - return NULL; + return NULL; } - uint64_t -gf_defrag_get_estimates_based_on_size (dht_conf_t *conf) +gf_defrag_get_estimates_based_on_size(dht_conf_t *conf) { - gf_defrag_info_t *defrag = NULL; - double rate_processed = 0; - uint64_t total_processed = 0; - uint64_t tmp_count = 0; - uint64_t time_to_complete = 0; - struct timeval now = {0,}; - double elapsed = 0; + gf_defrag_info_t *defrag = NULL; + double rate_processed = 0; + uint64_t total_processed = 0; + uint64_t tmp_count = 0; + uint64_t time_to_complete = 0; + struct timeval now = { + 0, + }; + double elapsed = 0; - defrag = conf->defrag; + defrag = conf->defrag; - if (!g_totalsize) - goto out; + if (!g_totalsize) + goto out; - gettimeofday (&now, NULL); - elapsed = now.tv_sec - defrag->start_time.tv_sec; + gettimeofday(&now, NULL); + elapsed = now.tv_sec - defrag->start_time.tv_sec; - /* Don't calculate the estimates for the first 10 minutes. - * It is unlikely to be accurate and estimates are not required - * if the process finishes in less than 10 mins. - */ + /* Don't calculate the estimates for the first 10 minutes. + * It is unlikely to be accurate and estimates are not required + * if the process finishes in less than 10 mins. + */ - if (elapsed < ESTIMATE_START_INTERVAL) { - gf_msg (THIS->name, GF_LOG_INFO, 0, 0, - "Rebalance estimates will not be available for the " - "first %d seconds.", ESTIMATE_START_INTERVAL); + if (elapsed < ESTIMATE_START_INTERVAL) { + gf_msg(THIS->name, GF_LOG_INFO, 0, 0, + "Rebalance estimates will not be available for the " + "first %d seconds.", + ESTIMATE_START_INTERVAL); - goto out; - } + goto out; + } - total_processed = defrag->size_processed; + total_processed = defrag->size_processed; - /* rate at which files processed */ - rate_processed = (total_processed)/elapsed; + /* rate at which files processed */ + rate_processed = (total_processed) / elapsed; - tmp_count = g_totalsize; + tmp_count = g_totalsize; - if (rate_processed) { - time_to_complete = (tmp_count)/rate_processed; + if (rate_processed) { + time_to_complete = (tmp_count) / rate_processed; - } else { - gf_msg (THIS->name, GF_LOG_ERROR, 0, 0, - "Unable to calculate estimated time for rebalance"); - } + } else { + gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, + "Unable to calculate estimated time for rebalance"); + } - gf_log (THIS->name, GF_LOG_INFO, - "TIME: (size) total_processed=%"PRIu64" tmp_cnt = %"PRIu64"," - "rate_processed=%f, elapsed = %f", total_processed, tmp_count, - rate_processed, elapsed); + gf_log(THIS->name, GF_LOG_INFO, + "TIME: (size) total_processed=%" PRIu64 " tmp_cnt = %" PRIu64 + "," + "rate_processed=%f, elapsed = %f", + total_processed, tmp_count, rate_processed, elapsed); out: - return time_to_complete; + return time_to_complete; } - int -gf_defrag_status_get (dht_conf_t *conf, dict_t *dict) +gf_defrag_status_get(dht_conf_t *conf, dict_t *dict) { - int ret = 0; - uint64_t files = 0; - uint64_t size = 0; - uint64_t lookup = 0; - uint64_t failures = 0; - uint64_t skipped = 0; - uint64_t promoted = 0; - uint64_t demoted = 0; - char *status = ""; - double elapsed = 0; - struct timeval end = {0,}; - uint64_t time_to_complete = 0; - uint64_t time_left = 0; - gf_defrag_info_t *defrag = conf->defrag; - - if (!defrag) - goto out; - - ret = 0; - if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) - goto out; - - files = defrag->total_files; - size = defrag->total_data; - lookup = defrag->num_files_lookedup; - failures = defrag->total_failures; - skipped = defrag->skipped; - promoted = defrag->total_files_promoted; - demoted = defrag->total_files_demoted; - - gettimeofday (&end, NULL); - - elapsed = end.tv_sec - defrag->start_time.tv_sec; - - - /* The rebalance is still in progress */ - - if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) - && (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED)) { - - time_to_complete = gf_defrag_get_estimates_based_on_size (conf); - - if (time_to_complete && (time_to_complete > elapsed)) - time_left = time_to_complete - elapsed; - - gf_log (THIS->name, GF_LOG_INFO, - "TIME: Estimated total time to complete (size)= %"PRIu64 - " seconds, seconds left = %"PRIu64"", - time_to_complete, time_left); - } - - if (!dict) - goto log; - - ret = dict_set_uint64 (dict, "promoted", promoted); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set promoted count"); - - ret = dict_set_uint64 (dict, "demoted", demoted); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set demoted count"); - - ret = dict_set_uint64 (dict, "files", files); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set file count"); - - ret = dict_set_uint64 (dict, "size", size); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set size of xfer"); - - ret = dict_set_uint64 (dict, "lookups", lookup); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set lookedup file count"); - - - ret = dict_set_int32 (dict, "status", defrag->defrag_status); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set status"); - - ret = dict_set_double (dict, "run-time", elapsed); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set run-time"); - - ret = dict_set_uint64 (dict, "failures", failures); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set failure count"); - - ret = dict_set_uint64 (dict, "skipped", skipped); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set skipped file count"); - - ret = dict_set_uint64 (dict, "time-left", time_left); - if (ret) - gf_log (THIS->name, GF_LOG_WARNING, - "failed to set time-left"); + int ret = 0; + uint64_t files = 0; + uint64_t size = 0; + uint64_t lookup = 0; + uint64_t failures = 0; + uint64_t skipped = 0; + uint64_t promoted = 0; + uint64_t demoted = 0; + char *status = ""; + double elapsed = 0; + struct timeval end = { + 0, + }; + uint64_t time_to_complete = 0; + uint64_t time_left = 0; + gf_defrag_info_t *defrag = conf->defrag; + + if (!defrag) + goto out; + + ret = 0; + if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) + goto out; + + files = defrag->total_files; + size = defrag->total_data; + lookup = defrag->num_files_lookedup; + failures = defrag->total_failures; + skipped = defrag->skipped; + promoted = defrag->total_files_promoted; + demoted = defrag->total_files_demoted; + + gettimeofday(&end, NULL); + + elapsed = end.tv_sec - defrag->start_time.tv_sec; + + /* The rebalance is still in progress */ + + if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) && + (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED)) { + time_to_complete = gf_defrag_get_estimates_based_on_size(conf); + + if (time_to_complete && (time_to_complete > elapsed)) + time_left = time_to_complete - elapsed; + + gf_log(THIS->name, GF_LOG_INFO, + "TIME: Estimated total time to complete (size)= %" PRIu64 + " seconds, seconds left = %" PRIu64 "", + time_to_complete, time_left); + } + + if (!dict) + goto log; + + ret = dict_set_uint64(dict, "promoted", promoted); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set promoted count"); + + ret = dict_set_uint64(dict, "demoted", demoted); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set demoted count"); + + ret = dict_set_uint64(dict, "files", files); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set file count"); + + ret = dict_set_uint64(dict, "size", size); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set size of xfer"); + + ret = dict_set_uint64(dict, "lookups", lookup); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set lookedup file count"); + + ret = dict_set_int32(dict, "status", defrag->defrag_status); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set status"); + + ret = dict_set_double(dict, "run-time", elapsed); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set run-time"); + + ret = dict_set_uint64(dict, "failures", failures); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set failure count"); + + ret = dict_set_uint64(dict, "skipped", skipped); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set skipped file count"); + + ret = dict_set_uint64(dict, "time-left", time_left); + if (ret) + gf_log(THIS->name, GF_LOG_WARNING, "failed to set time-left"); log: - switch (defrag->defrag_status) { + switch (defrag->defrag_status) { case GF_DEFRAG_STATUS_NOT_STARTED: - status = "not started"; - break; + status = "not started"; + break; case GF_DEFRAG_STATUS_STARTED: - status = "in progress"; - break; + status = "in progress"; + break; case GF_DEFRAG_STATUS_STOPPED: - status = "stopped"; - break; + status = "stopped"; + break; case GF_DEFRAG_STATUS_COMPLETE: - status = "completed"; - break; + status = "completed"; + break; case GF_DEFRAG_STATUS_FAILED: - status = "failed"; - break; + status = "failed"; + break; default: - break; - } - - gf_msg (THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS, - "Rebalance is %s. Time taken is %.2f secs", - status, elapsed); - gf_msg (THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS, - "Files migrated: %"PRIu64", size: %" - PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: " - "%"PRIu64, files, size, lookup, failures, skipped); + break; + } + + gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS, + "Rebalance is %s. Time taken is %.2f secs", status, elapsed); + gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS, + "Files migrated: %" PRIu64 ", size: %" PRIu64 ", lookups: %" PRIu64 + ", failures: %" PRIu64 + ", skipped: " + "%" PRIu64, + files, size, lookup, failures, skipped); out: - return 0; + return 0; } void -gf_defrag_set_pause_state (gf_tier_conf_t *tier_conf, tier_pause_state_t state) +gf_defrag_set_pause_state(gf_tier_conf_t *tier_conf, tier_pause_state_t state) { - pthread_mutex_lock (&tier_conf->pause_mutex); - tier_conf->pause_state = state; - pthread_mutex_unlock (&tier_conf->pause_mutex); + pthread_mutex_lock(&tier_conf->pause_mutex); + tier_conf->pause_state = state; + pthread_mutex_unlock(&tier_conf->pause_mutex); } - tier_pause_state_t -gf_defrag_get_pause_state (gf_tier_conf_t *tier_conf) +gf_defrag_get_pause_state(gf_tier_conf_t *tier_conf) { - int state; + int state; - pthread_mutex_lock (&tier_conf->pause_mutex); - state = tier_conf->pause_state; - pthread_mutex_unlock (&tier_conf->pause_mutex); + pthread_mutex_lock(&tier_conf->pause_mutex); + state = tier_conf->pause_state; + pthread_mutex_unlock(&tier_conf->pause_mutex); - return state; + return state; } tier_pause_state_t -gf_defrag_check_pause_tier (gf_tier_conf_t *tier_conf) +gf_defrag_check_pause_tier(gf_tier_conf_t *tier_conf) { - int woke = 0; - int state = -1; + int woke = 0; + int state = -1; - pthread_mutex_lock (&tier_conf->pause_mutex); + pthread_mutex_lock(&tier_conf->pause_mutex); - if (tier_conf->pause_state == TIER_RUNNING) - goto out; + if (tier_conf->pause_state == TIER_RUNNING) + goto out; - if (tier_conf->pause_state == TIER_PAUSED) - goto out; + if (tier_conf->pause_state == TIER_PAUSED) + goto out; - if (tier_conf->promote_in_progress || - tier_conf->demote_in_progress) - goto out; + if (tier_conf->promote_in_progress || tier_conf->demote_in_progress) + goto out; - tier_conf->pause_state = TIER_PAUSED; + tier_conf->pause_state = TIER_PAUSED; - if (tier_conf->pause_synctask) { - synctask_wake (tier_conf->pause_synctask); - tier_conf->pause_synctask = 0; - woke = 1; - } + if (tier_conf->pause_synctask) { + synctask_wake(tier_conf->pause_synctask); + tier_conf->pause_synctask = 0; + woke = 1; + } - gf_msg ("tier", GF_LOG_DEBUG, 0, - DHT_MSG_TIER_PAUSED, - "woken %d", woke); + gf_msg("tier", GF_LOG_DEBUG, 0, DHT_MSG_TIER_PAUSED, "woken %d", woke); - gf_event (EVENT_TIER_PAUSE, "vol=%s", tier_conf->volname); + gf_event(EVENT_TIER_PAUSE, "vol=%s", tier_conf->volname); out: - state = tier_conf->pause_state; + state = tier_conf->pause_state; - pthread_mutex_unlock (&tier_conf->pause_mutex); + pthread_mutex_unlock(&tier_conf->pause_mutex); - return state; + return state; } void -gf_defrag_pause_tier_timeout (void *data) +gf_defrag_pause_tier_timeout(void *data) { - xlator_t *this = NULL; - dht_conf_t *conf = NULL; - gf_defrag_info_t *defrag = NULL; + xlator_t *this = NULL; + dht_conf_t *conf = NULL; + gf_defrag_info_t *defrag = NULL; - this = (xlator_t *) data; - GF_VALIDATE_OR_GOTO ("tier", this, out); + this = (xlator_t *)data; + GF_VALIDATE_OR_GOTO("tier", this, out); - conf = this->private; - GF_VALIDATE_OR_GOTO (this->name, conf, out); + conf = this->private; + GF_VALIDATE_OR_GOTO(this->name, conf, out); - defrag = conf->defrag; - GF_VALIDATE_OR_GOTO (this->name, defrag, out); + defrag = conf->defrag; + GF_VALIDATE_OR_GOTO(this->name, defrag, out); - gf_msg (this->name, GF_LOG_DEBUG, 0, - DHT_MSG_TIER_PAUSED, - "Request pause timer timeout"); + gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_TIER_PAUSED, + "Request pause timer timeout"); - gf_defrag_check_pause_tier (&defrag->tier_conf); + gf_defrag_check_pause_tier(&defrag->tier_conf); out: - return; + return; } int -gf_defrag_pause_tier (xlator_t *this, gf_defrag_info_t *defrag) +gf_defrag_pause_tier(xlator_t *this, gf_defrag_info_t *defrag) { - int ret = 0; - struct timespec delta = {0,}; - int delay = 2; - - if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) - goto out; - - /* - * Set flag requesting to pause tiering. Wait 'delay' seconds for - * tiering to actually stop as indicated by the pause state - * before returning success or failure. - */ - gf_defrag_set_pause_state (&defrag->tier_conf, TIER_REQUEST_PAUSE); - - /* - * If migration is not underway, can pause immediately. - */ - gf_defrag_check_pause_tier (&defrag->tier_conf); - if (gf_defrag_get_pause_state (&defrag->tier_conf) == TIER_PAUSED) - goto out; - - gf_msg (this->name, GF_LOG_DEBUG, 0, - DHT_MSG_TIER_PAUSED, - "Request pause tier"); - - defrag->tier_conf.pause_synctask = synctask_get (); - delta.tv_sec = delay; - delta.tv_nsec = 0; - defrag->tier_conf.pause_timer = - gf_timer_call_after (this->ctx, delta, - gf_defrag_pause_tier_timeout, - this); - - synctask_yield (defrag->tier_conf.pause_synctask); - - if (gf_defrag_get_pause_state (&defrag->tier_conf) == TIER_PAUSED) - goto out; - - gf_defrag_set_pause_state (&defrag->tier_conf, TIER_RUNNING); - - ret = -1; + int ret = 0; + struct timespec delta = { + 0, + }; + int delay = 2; + + if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) + goto out; + + /* + * Set flag requesting to pause tiering. Wait 'delay' seconds for + * tiering to actually stop as indicated by the pause state + * before returning success or failure. + */ + gf_defrag_set_pause_state(&defrag->tier_conf, TIER_REQUEST_PAUSE); + + /* + * If migration is not underway, can pause immediately. + */ + gf_defrag_check_pause_tier(&defrag->tier_conf); + if (gf_defrag_get_pause_state(&defrag->tier_conf) == TIER_PAUSED) + goto out; + + gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_TIER_PAUSED, + "Request pause tier"); + + defrag->tier_conf.pause_synctask = synctask_get(); + delta.tv_sec = delay; + delta.tv_nsec = 0; + defrag->tier_conf.pause_timer = gf_timer_call_after( + this->ctx, delta, gf_defrag_pause_tier_timeout, this); + + synctask_yield(defrag->tier_conf.pause_synctask); + + if (gf_defrag_get_pause_state(&defrag->tier_conf) == TIER_PAUSED) + goto out; + + gf_defrag_set_pause_state(&defrag->tier_conf, TIER_RUNNING); + + ret = -1; out: - gf_msg (this->name, GF_LOG_DEBUG, 0, - DHT_MSG_TIER_PAUSED, - "Pause tiering ret=%d", ret); + gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_TIER_PAUSED, + "Pause tiering ret=%d", ret); - return ret; + return ret; } int -gf_defrag_resume_tier (xlator_t *this, gf_defrag_info_t *defrag) +gf_defrag_resume_tier(xlator_t *this, gf_defrag_info_t *defrag) { - gf_msg (this->name, GF_LOG_DEBUG, 0, - DHT_MSG_TIER_RESUME, - "Pause end. Resume tiering"); + gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_TIER_RESUME, + "Pause end. Resume tiering"); - gf_defrag_set_pause_state (&defrag->tier_conf, TIER_RUNNING); + gf_defrag_set_pause_state(&defrag->tier_conf, TIER_RUNNING); - gf_event (EVENT_TIER_RESUME, "vol=%s", defrag->tier_conf.volname); + gf_event(EVENT_TIER_RESUME, "vol=%s", defrag->tier_conf.volname); - return 0; + return 0; } int -gf_defrag_start_detach_tier (gf_defrag_info_t *defrag) +gf_defrag_start_detach_tier(gf_defrag_info_t *defrag) { - defrag->cmd = GF_DEFRAG_CMD_START_DETACH_TIER; + defrag->cmd = GF_DEFRAG_CMD_START_DETACH_TIER; - return 0; + return 0; } int -gf_defrag_stop (dht_conf_t *conf, gf_defrag_status_t status, - dict_t *output) +gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output) { - /* TODO: set a variable 'stop_defrag' here, it should be checked - in defrag loop */ - int ret = -1; - gf_defrag_info_t *defrag = conf->defrag; + /* TODO: set a variable 'stop_defrag' here, it should be checked + in defrag loop */ + int ret = -1; + gf_defrag_info_t *defrag = conf->defrag; - GF_ASSERT (defrag); + GF_ASSERT(defrag); - if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) { - goto out; - } + if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) { + goto out; + } - gf_msg ("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED, - "Received stop command on rebalance"); - defrag->defrag_status = status; + gf_msg("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED, + "Received stop command on rebalance"); + defrag->defrag_status = status; - if (output) - gf_defrag_status_get (conf, output); - ret = 0; + if (output) + gf_defrag_status_get(conf, output); + ret = 0; out: - gf_msg_debug ("", 0, "Returning %d", ret); - return ret; + gf_msg_debug("", 0, "Returning %d", ret); + return ret; } |