summaryrefslogtreecommitdiffstats
path: root/xlators/mgmt/glusterd/src/glusterd-rebalance.c
diff options
context:
space:
mode:
Diffstat (limited to 'xlators/mgmt/glusterd/src/glusterd-rebalance.c')
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-rebalance.c1052
1 files changed, 457 insertions, 595 deletions
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index e5a907eaa..b7b974c68 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -1,22 +1,12 @@
/*
- Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
- This file is part of GlusterFS.
-
- GlusterFS is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published
- by the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- GlusterFS is distributed in the hope that it will be useful, but
- WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see
- <http://www.gnu.org/licenses/>.
-*/
+ Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
#ifndef _CONFIG_H
#define _CONFIG_H
#include "config.h"
@@ -41,581 +31,336 @@
#include "glusterd-utils.h"
#include "glusterd-store.h"
#include "run.h"
+#include "glusterd-volgen.h"
#include "syscall.h"
#include "cli1-xdr.h"
#include "xdr-generic.h"
-/* return values - 0: success, +ve: stopped, -ve: failure */
+int32_t
+glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe);
int
-gf_glusterd_rebalance_move_data (glusterd_volinfo_t *volinfo, const char *dir)
+glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
+ size_t len, glusterd_op_t op)
{
- int ret = -1;
- DIR *fd = NULL;
- glusterd_defrag_info_t *defrag = NULL;
- struct dirent *entry = NULL;
- struct stat stbuf = {0,};
- char full_path[PATH_MAX] = {0,};
- char linkinfo[PATH_MAX] = {0,};
- char force_string[64] = {0,};
-
- if (!volinfo->defrag)
- goto out;
-
- defrag = volinfo->defrag;
-
- fd = opendir (dir);
- if (!fd)
+ int ret = -1;
+ xlator_t *this = NULL;
+
+ this = THIS;
+ GF_ASSERT (this);
+
+ /* Check only if operation is not remove-brick */
+ if ((GD_OP_REMOVE_BRICK != op) &&
+ !gd_is_remove_brick_committed (volinfo)) {
+ gf_log (this->name, GF_LOG_DEBUG, "A remove-brick task on "
+ "volume %s is not yet committed", volinfo->volname);
+ snprintf (op_errstr, len, "A remove-brick task on volume %s is"
+ " not yet committed. Either commit or stop the "
+ "remove-brick task.", volinfo->volname);
goto out;
-
- if ((defrag->cmd == GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE) ||
- (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)) {
- strcpy (force_string, "force");
- } else {
- strcpy (force_string, "not-force");
}
- while ((entry = readdir (fd))) {
- if (!entry)
- break;
-
- /* We have to honor 'stop' (or 'pause'|'commit') as early
- as possible */
- if (volinfo->defrag_status !=
- GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED) {
- /* It can be one of 'stopped|paused|commit' etc */
- closedir (fd);
- ret = 1;
- goto out;
- }
-
- if (!strcmp (entry->d_name, ".") || !strcmp (entry->d_name, ".."))
- continue;
-
- snprintf (full_path, PATH_MAX, "%s/%s", dir, entry->d_name);
-
- ret = lstat (full_path, &stbuf);
- if (ret == -1)
- continue;
-
- if (S_ISDIR (stbuf.st_mode))
- continue;
-
- defrag->num_files_lookedup += 1;
-
- /* TODO: bring in feature to support hardlink rebalance */
- if (stbuf.st_nlink > 1)
- continue;
-
- /* if distribute is present, it will honor this key.
- -1 is returned if distribute is not present or file doesn't
- have a link-file. If file has link-file, the path of
- link-file will be the value, and also that guarantees
- that file has to be mostly migrated */
- ret = sys_lgetxattr (full_path, GF_XATTR_LINKINFO_KEY,
- &linkinfo, PATH_MAX);
- if (ret <= 0)
- continue;
-
- ret = sys_lsetxattr (full_path, "distribute.migrate-data",
- force_string, strlen (force_string), 0);
-
- /* if errno is not ENOSPC or ENOTCONN, we can still continue
- with rebalance process */
- if ((ret == -1) && ((errno != ENOSPC) ||
- (errno != ENOTCONN)))
- continue;
-
- if ((ret == -1) && (errno == ENOTCONN)) {
- /* Most probably mount point went missing (mostly due
- to a brick down), say rebalance failure to user,
- let him restart it if everything is fine */
- volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
- break;
- }
-
- if ((ret == -1) && (errno == ENOSPC)) {
- /* rebalance process itself failed, may be
- remote brick went down, or write failed due to
- disk full etc etc.. */
- volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
- break;
- }
-
- LOCK (&defrag->lock);
- {
- defrag->total_files += 1;
- defrag->total_data += stbuf.st_size;
- }
- UNLOCK (&defrag->lock);
+ if (glusterd_is_defrag_on (volinfo)) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "rebalance on volume %s already started",
+ volinfo->volname);
+ snprintf (op_errstr, len, "Rebalance on %s is already started",
+ volinfo->volname);
+ goto out;
}
- closedir (fd);
- fd = opendir (dir);
- if (!fd)
+ if (glusterd_is_rb_started (volinfo) ||
+ glusterd_is_rb_paused (volinfo)) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "Rebalance failed as replace brick is in progress on volume %s",
+ volinfo->volname);
+ snprintf (op_errstr, len, "Rebalance failed as replace brick is in progress on "
+ "volume %s", volinfo->volname);
goto out;
- while ((entry = readdir (fd))) {
- if (!entry)
- break;
-
- /* We have to honor 'stop' (or 'pause'|'commit') as early
- as possible */
- if (volinfo->defrag_status !=
- GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED) {
- /* It can be one of 'stopped|paused|commit' etc */
- closedir (fd);
- ret = 1;
- goto out;
- }
-
- if (!strcmp (entry->d_name, ".") || !strcmp (entry->d_name, ".."))
- continue;
-
- snprintf (full_path, 1024, "%s/%s", dir, entry->d_name);
-
- ret = lstat (full_path, &stbuf);
- if (ret == -1)
- continue;
-
- if (!S_ISDIR (stbuf.st_mode))
- continue;
-
- ret = gf_glusterd_rebalance_move_data (volinfo, full_path);
- if (ret)
- break;
}
- closedir (fd);
-
- if (!entry)
- ret = 0;
+ ret = 0;
out:
+ gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
return ret;
}
-/* return values - 0: success, +ve: stopped, -ve: failure */
-int
-gf_glusterd_rebalance_fix_layout (glusterd_volinfo_t *volinfo, const char *dir)
-{
- int ret = -1;
- char full_path[1024] = {0,};
- struct stat stbuf = {0,};
- DIR *fd = NULL;
- struct dirent *entry = NULL;
-
- if (!volinfo->defrag)
- goto out;
-
- fd = opendir (dir);
- if (!fd)
- goto out;
-
- while ((entry = readdir (fd))) {
- if (!entry)
- break;
-
- /* We have to honor 'stop' (or 'pause'|'commit') as early
- as possible */
- if (volinfo->defrag_status !=
- GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED) {
- /* It can be one of 'stopped|paused|commit' etc */
- closedir (fd);
- ret = 1;
- goto out;
- }
-
- if (!strcmp (entry->d_name, ".") || !strcmp (entry->d_name, ".."))
- continue;
-
- snprintf (full_path, 1024, "%s/%s", dir, entry->d_name);
-
- ret = lstat (full_path, &stbuf);
- if (ret == -1)
- continue;
- if (S_ISDIR (stbuf.st_mode)) {
- /* Fix the layout of the directory */
- /* TODO: isn't error code not important ? */
- sys_lsetxattr (full_path, "distribute.fix.layout",
- "yes", 3, 0);
-
- volinfo->defrag->total_files += 1;
-
- /* Traverse into subdirectory */
- ret = gf_glusterd_rebalance_fix_layout (volinfo,
- full_path);
- if (ret)
- break;
- }
- }
- closedir (fd);
-
- if (!entry)
- ret = 0;
+int32_t
+__glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
+{
+ glusterd_volinfo_t *volinfo = NULL;
+ glusterd_defrag_info_t *defrag = NULL;
+ int ret = 0;
+ char pidfile[PATH_MAX];
+ glusterd_conf_t *priv = NULL;
-out:
- return ret;
-}
+ priv = THIS->private;
+ if (!priv)
+ return 0;
-void *
-glusterd_defrag_start (void *data)
-{
- glusterd_volinfo_t *volinfo = data;
- glusterd_defrag_info_t *defrag = NULL;
- int ret = -1;
- struct stat stbuf = {0,};
+ volinfo = mydata;
+ if (!volinfo)
+ return 0;
- THIS = volinfo->xl;
- defrag = volinfo->defrag;
+ defrag = volinfo->rebal.defrag;
if (!defrag)
- goto out;
+ return 0;
- sleep (1);
- ret = lstat (defrag->mount, &stbuf);
- if ((ret == -1) && (errno == ENOTCONN)) {
- /* Wait for some more time before starting rebalance */
- sleep (2);
- ret = lstat (defrag->mount, &stbuf);
- if (ret == -1) {
- volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
- volinfo->rebalance_files = 0;
- volinfo->rebalance_data = 0;
- volinfo->lookedup_files = 0;
- goto out;
- }
- }
+ if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
+ volinfo->rebal.defrag = NULL;
- /* Fix the root ('/') first */
- sys_lsetxattr (defrag->mount, "distribute.fix.layout",
- "yes", 3, 0);
+ GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
- if ((defrag->cmd == GF_DEFRAG_CMD_START) ||
- (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX)) {
- /* root's layout got fixed */
- defrag->total_files = 1;
+ switch (event) {
+ case RPC_CLNT_CONNECT:
+ {
+ if (defrag->connected)
+ return 0;
- /* Step 1: Fix layout of all the directories */
- ret = gf_glusterd_rebalance_fix_layout (volinfo, defrag->mount);
- if (ret < 0)
- volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
- /* in both 'stopped' or 'failure' cases goto out */
- if (ret) {
- goto out;
+ LOCK (&defrag->lock);
+ {
+ defrag->connected = 1;
}
+ UNLOCK (&defrag->lock);
- /* Completed first step */
- volinfo->defrag_status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
+ gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_CONNECT",
+ rpc->conn.trans->name);
+ break;
}
- if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
- /* It was used by number of layout fixes on directories */
- defrag->total_files = 0;
-
- volinfo->defrag_status = GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED;
+ case RPC_CLNT_DISCONNECT:
+ {
+ if (!defrag->connected)
+ return 0;
- /* Step 2: Iterate over directories to move data */
- ret = gf_glusterd_rebalance_move_data (volinfo, defrag->mount);
- if (ret < 0)
- volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
- /* in both 'stopped' or 'failure' cases goto out */
- if (ret) {
- goto out;
+ LOCK (&defrag->lock);
+ {
+ defrag->connected = 0;
}
+ UNLOCK (&defrag->lock);
- /* Completed second step */
- volinfo->defrag_status = GF_DEFRAG_STATUS_MIGRATE_DATA_COMPLETE;
- }
-
- /* Completed whole process */
- if ((defrag->cmd == GF_DEFRAG_CMD_START) ||
- (defrag->cmd == GF_DEFRAG_CMD_START_FORCE))
- volinfo->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
-
- volinfo->rebalance_files = defrag->total_files;
- volinfo->rebalance_data = defrag->total_data;
- volinfo->lookedup_files = defrag->num_files_lookedup;
-out:
- volinfo->defrag = NULL;
- if (defrag) {
- gf_log ("rebalance", GF_LOG_INFO, "rebalance on %s complete",
- defrag->mount);
+ if (!glusterd_is_service_running (pidfile, NULL)) {
+ if (volinfo->rebal.defrag_status ==
+ GF_DEFRAG_STATUS_STARTED) {
+ volinfo->rebal.defrag_status =
+ GF_DEFRAG_STATUS_FAILED;
+ }
+ }
- ret = runcmd ("umount", "-l", defrag->mount, NULL);
- LOCK_DESTROY (&defrag->lock);
+ glusterd_store_perform_node_state_store (volinfo);
- if (defrag->cbk_fn) {
- defrag->cbk_fn (volinfo, volinfo->defrag_status);
+ if (defrag->rpc) {
+ rpc_clnt_unref (defrag->rpc);
+ defrag->rpc = NULL;
}
+ if (defrag->cbk_fn)
+ defrag->cbk_fn (volinfo,
+ volinfo->rebal.defrag_status);
GF_FREE (defrag);
+ gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_DISCONNECT",
+ rpc->conn.trans->name);
+ break;
}
- return NULL;
+ default:
+ gf_log ("", GF_LOG_TRACE,
+ "got some other RPC event %d", event);
+ ret = 0;
+ break;
+ }
+
+ return ret;
}
-int
-glusterd_defrag_stop_validate (glusterd_volinfo_t *volinfo,
- char *op_errstr, size_t len)
+int32_t
+glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
{
- int ret = -1;
- if (glusterd_is_defrag_on (volinfo) == 0) {
- snprintf (op_errstr, len, "Rebalance on %s is either Completed "
- "or not yet started", volinfo->volname);
- goto out;
- }
- ret = 0;
-out:
- gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
- return ret;
+ return glusterd_big_locked_notify (rpc, mydata, event,
+ data, __glusterd_defrag_notify);
}
int
-glusterd_defrag_stop (glusterd_volinfo_t *volinfo, u_quad_t *files,
- u_quad_t *size, char *op_errstr, size_t len)
+glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
+ size_t len, int cmd, defrag_cbk_fn_t cbk,
+ glusterd_op_t op)
{
- /* TODO: set a variable 'stop_defrag' here, it should be checked
- in defrag loop */
- int ret = -1;
+ int ret = -1;
+ glusterd_defrag_info_t *defrag = NULL;
+ runner_t runner = {0,};
+ glusterd_conf_t *priv = NULL;
+ char defrag_path[PATH_MAX];
+ char sockfile[PATH_MAX] = {0,};
+ char pidfile[PATH_MAX] = {0,};
+ char logfile[PATH_MAX] = {0,};
+ dict_t *options = NULL;
+ char valgrind_logfile[PATH_MAX] = {0,};
+
+ priv = THIS->private;
+
GF_ASSERT (volinfo);
- GF_ASSERT (files);
- GF_ASSERT (size);
GF_ASSERT (op_errstr);
- if (!volinfo) {
- ret = -1;
+ ret = glusterd_defrag_start_validate (volinfo, op_errstr, len, op);
+ if (ret)
goto out;
- }
-
- ret = glusterd_defrag_stop_validate (volinfo, op_errstr, len);
- if (ret) {
- /* rebalance may be happening on other nodes */
- ret = 0;
+ if (!volinfo->rebal.defrag)
+ volinfo->rebal.defrag =
+ GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+ gf_gld_mt_defrag_info);
+ if (!volinfo->rebal.defrag)
goto out;
- }
- ret = 0;
- if (volinfo->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
- goto out;
- }
+ defrag = volinfo->rebal.defrag;
- LOCK (&volinfo->defrag->lock);
- {
- volinfo->defrag_status = GF_DEFRAG_STATUS_STOPPED;
- *files = volinfo->defrag->total_files;
- *size = volinfo->defrag->total_data;
- }
- UNLOCK (&volinfo->defrag->lock);
+ defrag->cmd = cmd;
- ret = 0;
-out:
- gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
- return ret;
-}
+ volinfo->rebal.defrag_cmd = cmd;
+ volinfo->rebal.op = op;
-int
-glusterd_defrag_status_get (glusterd_volinfo_t *volinfo,
- dict_t *dict)
-{
- int ret = 0;
- uint64_t files = 0;
- uint64_t size = 0;
- uint64_t lookup = 0;
+ LOCK_INIT (&defrag->lock);
- if (!volinfo || !dict)
- goto out;
+ volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
- ret = 0;
- if (volinfo->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
- goto out;
+ glusterd_volinfo_reset_defrag_stats (volinfo);
+ glusterd_store_perform_node_state_store (volinfo);
- if (volinfo->defrag) {
- LOCK (&volinfo->defrag->lock);
- {
- files = volinfo->defrag->total_files;
- size = volinfo->defrag->total_data;
- lookup = volinfo->defrag->num_files_lookedup;
- }
- UNLOCK (&volinfo->defrag->lock);
- } else {
- files = volinfo->rebalance_files;
- size = volinfo->rebalance_data;
- lookup = volinfo->lookedup_files;
+ GLUSTERD_GET_DEFRAG_DIR (defrag_path, volinfo, priv);
+ ret = mkdir_p (defrag_path, 0777, _gf_true);
+ if (ret) {
+ gf_log (THIS->name, GF_LOG_ERROR, "Failed to create "
+ "directory %s", defrag_path);
+ goto out;
}
- ret = dict_set_uint64 (dict, "files", files);
- if (ret)
- gf_log (THIS->name, GF_LOG_WARNING,
- "failed to set file count");
-
- ret = dict_set_uint64 (dict, "size", size);
- if (ret)
- gf_log (THIS->name, GF_LOG_WARNING,
- "failed to set size of xfer");
-
- ret = dict_set_uint64 (dict, "lookups", lookup);
- if (ret)
- gf_log (THIS->name, GF_LOG_WARNING,
- "failed to set lookedup file count");
-
- ret = dict_set_int32 (dict, "status", volinfo->defrag_status);
- if (ret)
- gf_log (THIS->name, GF_LOG_WARNING,
- "failed to set status");
+ GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo, priv);
+ GLUSTERD_GET_DEFRAG_PID_FILE (pidfile, volinfo, priv);
+ snprintf (logfile, PATH_MAX, "%s/%s-rebalance.log",
+ DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname);
+ runinit (&runner);
-out:
- return 0;
-}
+ if (priv->valgrind) {
+ snprintf (valgrind_logfile, PATH_MAX,
+ "%s/valgrind-%s-rebalance.log",
+ DEFAULT_LOG_FILE_DIRECTORY,
+ volinfo->volname);
-void
-glusterd_rebalance_cmd_attempted_log (int cmd, char *volname)
-{
- switch (cmd) {
- case GF_DEFRAG_CMD_START_LAYOUT_FIX:
- gf_cmd_log ("Volume rebalance"," on volname: %s "
- "cmd: start fix layout , attempted",
- volname);
- gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
- "volume start layout fix on %s", volname);
- break;
- case GF_DEFRAG_CMD_START_MIGRATE_DATA:
- case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
- gf_cmd_log ("Volume rebalance"," on volname: %s "
- "cmd: start data migrate attempted",
- volname);
- gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
- "volume start migrate data on %s", volname);
- break;
- case GF_DEFRAG_CMD_START:
- gf_cmd_log ("Volume rebalance"," on volname: %s "
- "cmd: start, attempted", volname);
- gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
- "volume start on %s", volname);
- break;
- case GF_DEFRAG_CMD_STOP:
- gf_cmd_log ("Volume rebalance"," on volname: %s "
- "cmd: stop, attempted", volname);
- gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
- "volume stop on %s", volname);
- break;
- default:
- break;
+ runner_add_args (&runner, "valgrind", "--leak-check=full",
+ "--trace-children=yes", "--track-origins=yes",
+ NULL);
+ runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
}
-}
-void
-glusterd_rebalance_cmd_log (int cmd, char *volname, int status)
-{
- if (cmd != GF_DEFRAG_CMD_STATUS) {
- gf_cmd_log ("volume rebalance"," on volname: %s %d %s",
- volname, cmd, ((status)?"FAILED":"SUCCESS"));
+ runner_add_args (&runner, SBIN_DIR"/glusterfs",
+ "-s", "localhost", "--volfile-id", volinfo->volname,
+ "--xlator-option", "*dht.use-readdirp=yes",
+ "--xlator-option", "*dht.lookup-unhashed=yes",
+ "--xlator-option", "*dht.assert-no-child-down=yes",
+ "--xlator-option", "*replicate*.data-self-heal=off",
+ "--xlator-option",
+ "*replicate*.metadata-self-heal=off",
+ "--xlator-option", "*replicate*.entry-self-heal=off",
+ "--xlator-option", "*replicate*.readdir-failover=off",
+ "--xlator-option", "*dht.readdir-optimize=on",
+ NULL);
+ runner_add_arg (&runner, "--xlator-option");
+ runner_argprintf ( &runner, "*dht.rebalance-cmd=%d",cmd);
+ runner_add_arg (&runner, "--xlator-option");
+ runner_argprintf (&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
+ runner_add_arg (&runner, "--socket-file");
+ runner_argprintf (&runner, "%s",sockfile);
+ runner_add_arg (&runner, "--pid-file");
+ runner_argprintf (&runner, "%s",pidfile);
+ runner_add_arg (&runner, "-l");
+ runner_argprintf (&runner, logfile);
+ if (volinfo->memory_accounting)
+ runner_add_arg (&runner, "--mem-accounting");
+
+ ret = runner_run_nowait (&runner);
+ if (ret) {
+ gf_log ("glusterd", GF_LOG_DEBUG, "rebalance command failed");
+ goto out;
}
-}
-int
-glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
- size_t len)
-{
- int ret = -1;
+ sleep (5);
- if (glusterd_is_defrag_on (volinfo)) {
- gf_log ("glusterd", GF_LOG_DEBUG,
- "rebalance on volume %s already started",
- volinfo->volname);
- snprintf (op_errstr, len, "Rebalance on %s is already started",
- volinfo->volname);
+ /* Setting frame-timeout to 10mins (600seconds).
+ * Unix domain sockets ensures that the connection is reliable. The
+ * default timeout of 30mins used for unreliable network connections is
+ * too long for unix domain socket connections.
+ */
+ ret = rpc_transport_unix_options_build (&options, sockfile, 600);
+ if (ret) {
+ gf_log (THIS->name, GF_LOG_ERROR, "Unix options build failed");
goto out;
}
- if (glusterd_is_rb_started (volinfo) ||
- glusterd_is_rb_paused (volinfo)) {
- gf_log ("glusterd", GF_LOG_DEBUG,
- "Rebalance failed as replace brick is in progress on volume %s",
- volinfo->volname);
- snprintf (op_errstr, len, "Rebalance failed as replace brick is in progress on "
- "volume %s", volinfo->volname);
+ synclock_unlock (&priv->big_lock);
+ ret = glusterd_rpc_create (&defrag->rpc, options,
+ glusterd_defrag_notify, volinfo);
+ synclock_lock (&priv->big_lock);
+ if (ret) {
+ gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
goto out;
}
- ret = 0;
+
+ if (cbk)
+ defrag->cbk_fn = cbk;
+
out:
- gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+ gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
return ret;
}
+
int
-glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
- size_t len, int cmd, defrag_cbk_fn_t cbk)
+glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
+ glusterd_conf_t *priv, int cmd)
{
- int ret = -1;
- glusterd_defrag_info_t *defrag = NULL;
- runner_t runner = {0,};
- glusterd_conf_t *priv = NULL;
+ dict_t *options = NULL;
+ char sockfile[PATH_MAX] = {0,};
+ int ret = -1;
+ glusterd_defrag_info_t *defrag = NULL;
- priv = THIS->private;
+ if (!volinfo->rebal.defrag)
+ volinfo->rebal.defrag =
+ GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+ gf_gld_mt_defrag_info);
- GF_ASSERT (volinfo);
- GF_ASSERT (op_errstr);
-
- ret = glusterd_defrag_start_validate (volinfo, op_errstr, len);
- if (ret)
- goto out;
- if (!volinfo->defrag)
- volinfo->defrag = GF_CALLOC (1, sizeof (glusterd_defrag_info_t),
- gf_gld_mt_defrag_info);
- if (!volinfo->defrag)
+ if (!volinfo->rebal.defrag)
goto out;
- defrag = volinfo->defrag;
+ defrag = volinfo->rebal.defrag;
defrag->cmd = cmd;
LOCK_INIT (&defrag->lock);
- snprintf (defrag->mount, 1024, "%s/mount/%s",
- priv->workdir, volinfo->volname);
- /* Create a directory, mount glusterfs over it, start glusterfs-defrag */
- runinit (&runner);
- runner_add_args (&runner, "mkdir", "-p", defrag->mount, NULL);
- ret = runner_run_reuse (&runner);
- if (ret) {
- runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
- runner_end (&runner);
- goto out;
- }
- runner_end (&runner);
- runinit (&runner);
- runner_add_args (&runner, SBIN_DIR"/glusterfs",
- "-s", "localhost", "--volfile-id", volinfo->volname,
- "--xlator-option", "*dht.use-readdirp=yes",
- "--xlator-option", "*dht.lookup-unhashed=yes",
- "--xlator-option", "*dht.assert-no-child-down=yes",
- defrag->mount, NULL);
- ret = runner_run_reuse (&runner);
+ GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo, priv);
+
+ /* Setting frame-timeout to 10mins (600seconds).
+ * Unix domain sockets ensures that the connection is reliable. The
+ * default timeout of 30mins used for unreliable network connections is
+ * too long for unix domain socket connections.
+ */
+ ret = rpc_transport_unix_options_build (&options, sockfile, 600);
if (ret) {
- runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
- runner_end (&runner);
+ gf_log (THIS->name, GF_LOG_ERROR, "Unix options build failed");
goto out;
}
- runner_end (&runner);
-
- volinfo->defrag_status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
- if ((cmd == GF_DEFRAG_CMD_START_MIGRATE_DATA) ||
- (cmd == GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE)) {
- volinfo->defrag_status = GF_DEFRAG_STATUS_MIGRATE_DATA_STARTED;
- }
- if (cbk)
- defrag->cbk_fn = cbk;
-
- ret = pthread_create (&defrag->th, NULL, glusterd_defrag_start,
- volinfo);
+ synclock_unlock (&priv->big_lock);
+ ret = glusterd_rpc_create (&defrag->rpc, options,
+ glusterd_defrag_notify, volinfo);
+ synclock_lock (&priv->big_lock);
if (ret) {
- runinit (&runner);
- runner_add_args (&runner, "umount", "-l", defrag->mount, NULL);
- ret = runner_run_reuse (&runner);
- if (ret)
- runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
- runner_end (&runner);
+ gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
+ goto out;
}
+ ret = 0;
out:
- gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
return ret;
}
@@ -658,26 +403,32 @@ out:
}
int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+__glusterd_handle_defrag_volume (rpcsvc_request_t *req)
{
- int32_t ret = -1;
- gf_cli_req cli_req = {{0,}};
- glusterd_conf_t *priv = NULL;
- dict_t *dict = NULL;
- char *volname = NULL;
- gf_cli_defrag_type cmd = 0;
+ int32_t ret = -1;
+ gf_cli_req cli_req = {{0,}};
+ glusterd_conf_t *priv = NULL;
+ dict_t *dict = NULL;
+ char *volname = NULL;
+ gf_cli_defrag_type cmd = 0;
+ char msg[2048] = {0,};
+ xlator_t *this = NULL;
GF_ASSERT (req);
+ this = THIS;
+ GF_ASSERT (this);
- priv = THIS->private;
+ priv = this->private;
+ GF_ASSERT (priv);
- if (!xdr_to_generic (req->msg[0], &cli_req,
- (xdrproc_t)xdr_gf_cli_req)) {
+ ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+ if (ret < 0) {
//failed to decode msg;
req->rpc_err = GARBAGE_ARGS;
goto out;
}
- if (cli_req.dict.dict_len) {
+
+ if (cli_req.dict.dict_len) {
/* Unserialize the dictionary */
dict = dict_new ();
@@ -685,34 +436,39 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req)
cli_req.dict.dict_len,
&dict);
if (ret < 0) {
- gf_log ("glusterd", GF_LOG_ERROR,
- "failed to "
+ gf_log (this->name, GF_LOG_ERROR, "failed to "
"unserialize req-buffer to dictionary");
+ snprintf (msg, sizeof (msg), "Unable to decode the "
+ "command");
goto out;
}
}
ret = dict_get_str (dict, "volname", &volname);
if (ret) {
- gf_log (THIS->name, GF_LOG_ERROR,
- "Failed to get volname");
+ snprintf (msg, sizeof (msg), "Failed to get volume name");
+ gf_log (this->name, GF_LOG_ERROR, "%s", msg);
goto out;
}
ret = dict_get_int32 (dict, "rebalance-command", (int32_t*)&cmd);
if (ret) {
- gf_log (THIS->name, GF_LOG_ERROR,
- "Failed to get command");
+ snprintf (msg, sizeof (msg), "Failed to get command");
+ gf_log (this->name, GF_LOG_ERROR, "%s", msg);
goto out;
}
- glusterd_rebalance_cmd_attempted_log (cmd, volname);
-
- ret = dict_set_static_bin (dict, "node-uuid", priv->uuid, 16);
+ ret = dict_set_static_bin (dict, "node-uuid", MY_UUID, 16);
if (ret)
goto out;
- ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict);
+ if ((cmd == GF_DEFRAG_CMD_STATUS) ||
+ (cmd == GF_DEFRAG_CMD_STOP)) {
+ ret = glusterd_op_begin (req, GD_OP_DEFRAG_BRICK_VOLUME,
+ dict, msg, sizeof (msg));
+ } else
+ ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict,
+ msg, sizeof (msg));
out:
@@ -720,57 +476,100 @@ out:
glusterd_op_sm ();
if (ret) {
- if (dict)
- dict_unref (dict);
- ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0, req,
- NULL, "operation failed");
+ if (msg[0] == '\0')
+ snprintf (msg, sizeof (msg), "Operation failed");
+ ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0,
+ req, dict, msg);
+
}
- if (cli_req.dict.dict_val)
- free (cli_req.dict.dict_val);//malloced by xdr
+ free (cli_req.dict.dict_val);//malloced by xdr
return 0;
}
+int
+glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_defrag_volume);
+}
+
int
glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
{
- char *volname = NULL;
- int ret = 0;
- int32_t cmd = 0;
- char msg[2048] = {0};
- glusterd_volinfo_t *volinfo = NULL;
+ char *volname = NULL;
+ int ret = 0;
+ int32_t cmd = 0;
+ char msg[2048] = {0};
+ glusterd_volinfo_t *volinfo = NULL;
+ char *task_id_str = NULL;
+ dict_t *op_ctx = NULL;
+ xlator_t *this = 0;
+
+ this = THIS;
+ GF_ASSERT (this);
ret = dict_get_str (dict, "volname", &volname);
if (ret) {
- gf_log (THIS->name, GF_LOG_DEBUG, "volname not found");
+ gf_log (this->name, GF_LOG_DEBUG, "volname not found");
goto out;
}
+
ret = dict_get_int32 (dict, "rebalance-command", &cmd);
if (ret) {
- gf_log (THIS->name, GF_LOG_DEBUG, "cmd not found");
+ gf_log (this->name, GF_LOG_DEBUG, "cmd not found");
goto out;
}
ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
msg, sizeof (msg));
if (ret) {
- gf_log (THIS->name, GF_LOG_DEBUG, "failed to validate");
+ gf_log (this->name, GF_LOG_DEBUG, "failed to validate");
goto out;
}
switch (cmd) {
case GF_DEFRAG_CMD_START:
case GF_DEFRAG_CMD_START_LAYOUT_FIX:
- case GF_DEFRAG_CMD_START_MIGRATE_DATA:
- case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
- ret = glusterd_defrag_start_validate (volinfo,
- msg, sizeof (msg));
+ case GF_DEFRAG_CMD_START_FORCE:
+ if (is_origin_glusterd (dict)) {
+ op_ctx = glusterd_op_get_ctx ();
+ if (!op_ctx) {
+ ret = -1;
+ gf_log (this->name, GF_LOG_ERROR,
+ "Failed to get op_ctx");
+ goto out;
+ }
+
+ ret = glusterd_generate_and_set_task_id
+ (op_ctx, GF_REBALANCE_TID_KEY);
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "Failed to generate task-id");
+ goto out;
+ }
+ } else {
+ ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
+ &task_id_str);
+ if (ret) {
+ snprintf (msg, sizeof (msg),
+ "Missing rebalance-id");
+ gf_log (this->name, GF_LOG_WARNING, "%s", msg);
+ ret = 0;
+ }
+ }
+ ret = glusterd_defrag_start_validate (volinfo, msg,
+ sizeof (msg),
+ GD_OP_REBALANCE);
if (ret) {
- gf_log (THIS->name, GF_LOG_DEBUG,
- "start validate failed");
+ gf_log (this->name, GF_LOG_DEBUG,
+ "start validate failed");
goto out;
}
+ break;
+ case GF_DEFRAG_CMD_STATUS:
+ case GF_DEFRAG_CMD_STOP:
+ break;
default:
break;
}
@@ -787,101 +586,164 @@ out:
int
glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
{
- char *volname = NULL;
- int ret = 0;
- int32_t cmd = 0;
- char msg[2048] = {0};
- glusterd_volinfo_t *volinfo = NULL;
- uint64_t files = 0;
- uint64_t size = 0;
- void *node_uuid = NULL;
- glusterd_conf_t *priv = NULL;
- dict_t *tmp_dict = NULL;
-
- priv = THIS->private;
+ char *volname = NULL;
+ int ret = 0;
+ int32_t cmd = 0;
+ char msg[2048] = {0};
+ glusterd_volinfo_t *volinfo = NULL;
+ glusterd_conf_t *priv = NULL;
+ glusterd_brickinfo_t *brickinfo = NULL;
+ glusterd_brickinfo_t *tmp = NULL;
+ gf_boolean_t volfile_update = _gf_false;
+ char *task_id_str = NULL;
+ dict_t *ctx = NULL;
+ xlator_t *this = NULL;
+
+ this = THIS;
+ GF_ASSERT (this);
+ priv = this->private;
ret = dict_get_str (dict, "volname", &volname);
if (ret) {
- gf_log (THIS->name, GF_LOG_DEBUG, "volname not given");
+ gf_log (this->name, GF_LOG_DEBUG, "volname not given");
goto out;
}
ret = dict_get_int32 (dict, "rebalance-command", &cmd);
if (ret) {
- gf_log (THIS->name, GF_LOG_DEBUG, "command not given");
+ gf_log (this->name, GF_LOG_DEBUG, "command not given");
goto out;
}
+
ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
msg, sizeof (msg));
if (ret) {
- gf_log (THIS->name, GF_LOG_DEBUG, "cmd validate failed");
+ gf_log (this->name, GF_LOG_DEBUG, "cmd validate failed");
goto out;
}
- if ((cmd != GF_DEFRAG_CMD_STATUS) &&
- (cmd != GF_DEFRAG_CMD_STOP)) {
- ret = dict_get_ptr (dict, "node-uuid", &node_uuid);
- if (ret) {
- gf_log (THIS->name, GF_LOG_DEBUG, "node-uuid not found");
- goto out;
- }
-
- /* perform this on only the node which has
- issued the command */
- if (uuid_compare (node_uuid, priv->uuid)) {
- gf_log (THIS->name, GF_LOG_DEBUG,
- "not the source node %s", uuid_utoa (priv->uuid));
- goto out;
+ /* Set task-id, if available, in op_ctx dict for operations other than
+ * start
+ */
+ if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
+ if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+ ctx = glusterd_op_get_ctx ();
+ if (!ctx) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "Failed to get op_ctx");
+ ret = -1;
+ goto out;
+ }
+
+ if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
+ ret = glusterd_copy_uuid_to_dict
+ (volinfo->rebal.rebalance_id, ctx,
+ GF_REMOVE_BRICK_TID_KEY);
+ else
+ ret = glusterd_copy_uuid_to_dict
+ (volinfo->rebal.rebalance_id, ctx,
+ GF_REBALANCE_TID_KEY);
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "Failed to set task-id");
+ goto out;
+ }
}
}
switch (cmd) {
case GF_DEFRAG_CMD_START:
case GF_DEFRAG_CMD_START_LAYOUT_FIX:
- case GF_DEFRAG_CMD_START_MIGRATE_DATA:
- case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
+ case GF_DEFRAG_CMD_START_FORCE:
+ ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+ if (ret) {
+ gf_log (this->name, GF_LOG_DEBUG, "Missing rebalance "
+ "id");
+ ret = 0;
+ } else {
+ uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
+ volinfo->rebal.op = GD_OP_REBALANCE;
+ }
ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
- cmd, NULL);
+ cmd, NULL, GD_OP_REBALANCE);
break;
- case GF_DEFRAG_CMD_STOP:
- ret = glusterd_defrag_stop (volinfo, &files, &size,
- msg, sizeof (msg));
- if (!ret && rsp_dict) {
- ret = dict_set_uint64 (rsp_dict, "files", files);
- if (ret)
- gf_log (THIS->name, GF_LOG_WARNING,
- "failed to set file count");
-
- ret = dict_set_uint64 (rsp_dict, "size", size);
- if (ret)
- gf_log (THIS->name, GF_LOG_WARNING,
- "failed to set xfer size");
-
- /* Don't want to propagate errors from dict_set() */
+ case GF_DEFRAG_CMD_STOP:
+ /* Clear task-id only on explicitly stopping rebalance.
+ * Also clear the stored operation, so it doesn't cause trouble
+ * with future rebalance/remove-brick starts
+ */
+ uuid_clear (volinfo->rebal.rebalance_id);
+ volinfo->rebal.op = GD_OP_NONE;
+
+ /* Fall back to the old volume file in case of decommission*/
+ list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
+ brick_list) {
+ if (!brickinfo->decommissioned)
+ continue;
+ brickinfo->decommissioned = 0;
+ volfile_update = _gf_true;
+ }
+
+ if (volfile_update == _gf_false) {
ret = 0;
+ break;
}
- break;
- case GF_DEFRAG_CMD_STATUS:
- if (rsp_dict)
- tmp_dict = rsp_dict;
+ ret = glusterd_create_volfiles_and_notify_services (volinfo);
+ if (ret) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "failed to create volfiles");
+ goto out;
+ }
+
+ ret = glusterd_store_volinfo (volinfo,
+ GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+ if (ret) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "failed to store volinfo");
+ goto out;
+ }
- /* On source node, there will be no 'rsp_dict' */
- if (!tmp_dict)
- tmp_dict = glusterd_op_get_ctx (GD_OP_REBALANCE);
+ ret = 0;
+ break;
- ret = glusterd_defrag_status_get (volinfo, tmp_dict);
+ case GF_DEFRAG_CMD_STATUS:
break;
default:
break;
}
- glusterd_rebalance_cmd_log (cmd, volname, ret);
-
out:
if (ret && op_errstr && msg[0])
*op_errstr = gf_strdup (msg);
return ret;
}
+
+int32_t
+glusterd_defrag_event_notify_handle (dict_t *dict)
+{
+ glusterd_volinfo_t *volinfo = NULL;
+ char *volname = NULL;
+ int32_t ret = -1;
+
+ ret = dict_get_str (dict, "volname", &volname);
+ if (ret) {
+ gf_log ("", GF_LOG_ERROR, "Failed to get volname");
+ return ret;
+ }
+
+ ret = glusterd_volinfo_find (volname, &volinfo);
+ if (ret) {
+ gf_log ("", GF_LOG_ERROR, "Failed to get volinfo for %s"
+ , volname);
+ return ret;
+ }
+
+ ret = glusterd_defrag_volume_status_update (volinfo, dict);
+
+ if (ret)
+ gf_log ("", GF_LOG_ERROR, "Failed to update status");
+ return ret;
+}
generated by cgit (git 2.34.1) at 2026-01-05 04:06:30 +0000