summaryrefslogtreecommitdiffstats
path: root/xlators/mgmt/glusterd/src/glusterd.c
diff options
context:
space:
mode:
Diffstat (limited to 'xlators/mgmt/glusterd/src/glusterd.c')
-rw-r--r--xlators/mgmt/glusterd/src/glusterd.c830
1 files changed, 645 insertions, 185 deletions
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 83bbd1b22..59288ada0 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1,22 +1,12 @@
/*
- Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
- This file is part of GlusterFS.
-
- GlusterFS is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published
- by the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- GlusterFS is distributed in the hope that it will be useful, but
- WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see
- <http://www.gnu.org/licenses/>.
-*/
+ Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
#ifndef _CONFIG_H
#define _CONFIG_H
@@ -44,20 +34,29 @@
#include "glusterd-sm.h"
#include "glusterd-op-sm.h"
#include "glusterd-store.h"
+#include "glusterd-hooks.h"
#include "glusterd-utils.h"
+#include "glusterd-locks.h"
#include "common-utils.h"
#include "run.h"
+#include "syncop.h"
+
#include "glusterd-mountbroker.h"
-static uuid_t glusterd_uuid;
-extern struct rpcsvc_program glusterd1_mop_prog;
-extern struct rpcsvc_program gd_svc_mgmt_prog;
-extern struct rpcsvc_program gd_svc_cli_prog;
extern struct rpcsvc_program gluster_handshake_prog;
+extern struct rpcsvc_program gluster_cli_getspec_prog;
extern struct rpcsvc_program gluster_pmap_prog;
extern glusterd_op_info_t opinfo;
-extern struct rpc_clnt_program glusterd_glusterfs_3_1_mgmt_prog;
+extern struct rpcsvc_program gd_svc_mgmt_prog;
+extern struct rpcsvc_program gd_svc_mgmt_v3_prog;
+extern struct rpcsvc_program gd_svc_peer_prog;
+extern struct rpcsvc_program gd_svc_cli_prog;
+extern struct rpcsvc_program gd_svc_cli_prog_ro;
+extern struct rpc_clnt_program gd_brick_prog;
+extern struct rpcsvc_program glusterd_mgmt_hndsk_prog;
+
+extern char snap_mount_folder[PATH_MAX];
rpcsvc_cbk_program_t glusterd_cbk_prog = {
.progname = "Gluster Callback",
@@ -65,6 +64,57 @@ rpcsvc_cbk_program_t glusterd_cbk_prog = {
.progver = GLUSTER_CBK_VERSION,
};
+struct rpcsvc_program *gd_inet_programs[] = {
+ &gd_svc_peer_prog,
+ &gd_svc_cli_prog_ro,
+ &gd_svc_mgmt_prog,
+ &gd_svc_mgmt_v3_prog,
+ &gluster_pmap_prog,
+ &gluster_handshake_prog,
+ &glusterd_mgmt_hndsk_prog,
+};
+int gd_inet_programs_count = (sizeof (gd_inet_programs) /
+ sizeof (gd_inet_programs[0]));
+
+struct rpcsvc_program *gd_uds_programs[] = {
+ &gd_svc_cli_prog,
+ &gluster_cli_getspec_prog,
+};
+int gd_uds_programs_count = (sizeof (gd_uds_programs) /
+ sizeof (gd_uds_programs[0]));
+
+const char *gd_op_list[GD_OP_MAX + 1] = {
+ [GD_OP_NONE] = "Invalid op",
+ [GD_OP_CREATE_VOLUME] = "Create",
+ [GD_OP_START_BRICK] = "Start Brick",
+ [GD_OP_STOP_BRICK] = "Stop Brick",
+ [GD_OP_DELETE_VOLUME] = "Delete",
+ [GD_OP_START_VOLUME] = "Start",
+ [GD_OP_STOP_VOLUME] = "Stop",
+ [GD_OP_DEFRAG_VOLUME] = "Rebalance",
+ [GD_OP_ADD_BRICK] = "Add brick",
+ [GD_OP_REMOVE_BRICK] = "Remove brick",
+ [GD_OP_REPLACE_BRICK] = "Replace brick",
+ [GD_OP_SET_VOLUME] = "Set",
+ [GD_OP_RESET_VOLUME] = "Reset",
+ [GD_OP_SYNC_VOLUME] = "Sync",
+ [GD_OP_LOG_ROTATE] = "Log rotate",
+ [GD_OP_GSYNC_SET] = "Geo-replication",
+ [GD_OP_PROFILE_VOLUME] = "Profile",
+ [GD_OP_QUOTA] = "Quota",
+ [GD_OP_STATUS_VOLUME] = "Status",
+ [GD_OP_REBALANCE] = "Rebalance",
+ [GD_OP_HEAL_VOLUME] = "Heal",
+ [GD_OP_STATEDUMP_VOLUME] = "Statedump",
+ [GD_OP_LIST_VOLUME] = "Lists",
+ [GD_OP_CLEARLOCKS_VOLUME] = "Clear locks",
+ [GD_OP_DEFRAG_BRICK_VOLUME] = "Rebalance",
+ [GD_OP_COPY_FILE] = "Copy File",
+ [GD_OP_SYS_EXEC] = "Execute system commands",
+ [GD_OP_GSYNC_CREATE] = "Geo-replication Create",
+ [GD_OP_SNAP] = "Snapshot",
+ [GD_OP_MAX] = "Invalid op"
+};
static int
glusterd_opinfo_init ()
@@ -76,35 +126,30 @@ glusterd_opinfo_init ()
return ret;
}
-static int
-glusterd_uuid_init (int flag)
+
+int
+glusterd_uuid_init ()
{
int ret = -1;
+ xlator_t *this = NULL;
glusterd_conf_t *priv = NULL;
- priv = THIS->private;
+ this = THIS;
+ GF_ASSERT (this);
+ priv = this->private;
- if (!flag) {
- ret = glusterd_retrieve_uuid ();
- if (!ret) {
- uuid_copy (glusterd_uuid, priv->uuid);
- gf_log ("glusterd", GF_LOG_INFO,
- "retrieved UUID: %s", uuid_utoa (priv->uuid));
- return 0;
- }
+ ret = glusterd_retrieve_uuid ();
+ if (ret == 0) {
+ gf_log (this->name, GF_LOG_INFO,
+ "retrieved UUID: %s", uuid_utoa (priv->uuid));
+ return 0;
}
- uuid_generate (glusterd_uuid);
-
- gf_log ("glusterd", GF_LOG_INFO,
- "generated UUID: %s", uuid_utoa (glusterd_uuid));
- uuid_copy (priv->uuid, glusterd_uuid);
-
- ret = glusterd_store_uuid ();
+ ret = glusterd_uuid_generate_save ();
if (ret) {
gf_log ("glusterd", GF_LOG_ERROR,
- "Unable to store generated UUID");
+ "Unable to generate and save new UUID");
return ret;
}
@@ -112,6 +157,63 @@ glusterd_uuid_init (int flag)
}
int
+glusterd_uuid_generate_save ()
+{
+ int ret = -1;
+ glusterd_conf_t *priv = NULL;
+ xlator_t *this = NULL;
+
+ this = THIS;
+ GF_ASSERT (this);
+ priv = this->private;
+ GF_ASSERT (priv);
+
+ uuid_generate (priv->uuid);
+
+ gf_log (this->name, GF_LOG_INFO, "generated UUID: %s",
+ uuid_utoa (priv->uuid));
+
+ ret = glusterd_store_global_info (this);
+
+ if (ret)
+ gf_log (this->name, GF_LOG_ERROR,
+ "Unable to store the generated uuid %s",
+ uuid_utoa (priv->uuid));
+
+ return ret;
+}
+
+int
+glusterd_options_init (xlator_t *this)
+{
+ int ret = -1;
+ glusterd_conf_t *priv = NULL;
+ char *initial_version = "0";
+
+ priv = this->private;
+
+ priv->opts = dict_new ();
+ if (!priv->opts)
+ goto out;
+
+ ret = glusterd_store_retrieve_options (this);
+ if (ret == 0)
+ goto out;
+
+ ret = dict_set_str (priv->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+ initial_version);
+ if (ret)
+ goto out;
+ ret = glusterd_store_options (this, priv->opts);
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR, "Unable to store version");
+ return ret;
+ }
+out:
+
+ return 0;
+}
+int
glusterd_fetchspec_notify (xlator_t *this)
{
int ret = -1;
@@ -120,10 +222,15 @@ glusterd_fetchspec_notify (xlator_t *this)
priv = this->private;
- list_for_each_entry (trans, &priv->xprt_list, list) {
- rpcsvc_callback_submit (priv->rpc, trans, &glusterd_cbk_prog,
- GF_CBK_FETCHSPEC, NULL, 0);
+ pthread_mutex_lock (&priv->xprt_lock);
+ {
+ list_for_each_entry (trans, &priv->xprt_list, list) {
+ rpcsvc_callback_submit (priv->rpc, trans,
+ &glusterd_cbk_prog,
+ GF_CBK_FETCHSPEC, NULL, 0);
+ }
}
+ pthread_mutex_unlock (&priv->xprt_lock);
ret = 0;
@@ -181,12 +288,16 @@ glusterd_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
{
INIT_LIST_HEAD (&xprt->list);
+ pthread_mutex_lock (&priv->xprt_lock);
list_add_tail (&xprt->list, &priv->xprt_list);
+ pthread_mutex_unlock (&priv->xprt_lock);
break;
}
case RPCSVC_EVENT_DISCONNECT:
{
+ pthread_mutex_lock (&priv->xprt_lock);
list_del (&xprt->list);
+ pthread_mutex_unlock (&priv->xprt_lock);
pmap_registry_remove (this, 0, NULL, GF_PMAP_PORT_NONE, xprt);
break;
}
@@ -225,12 +336,6 @@ glusterd_rpcsvc_options_build (dict_t *options)
int ret = 0;
uint32_t backlog = 0;
- if (!dict_get (options, "rpc-auth-allow-insecure")) {
- ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
- if (ret)
- goto out;
- }
-
ret = dict_get_uint32 (options, "transport.socket.listen-backlog",
&backlog);
@@ -249,18 +354,6 @@ out:
return ret;
}
-/* defined in usterd-utils.c -- no
- * glusterd header where it would be
- * appropriate to put to, and too
- * accidental routine to place in
- * libglusterfs.
- *
- * (Indeed, XXX: we'd rather need a general
- * "mkdir -p" like routine in
- * libglusterfs)
- */
-extern int mkdir_if_missing (char *path);
-
#if SYNCDAEMON_COMPILE
static int
glusterd_check_gsync_present (int *valid_state)
@@ -353,13 +446,13 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
if (strlen (conf->workdir)+2 > PATH_MAX-strlen(GEOREP)) {
ret = -1;
gf_log ("glusterd", GF_LOG_CRITICAL,
- "Unable to create "GEOREP" directory %s",
- georepdir);
+ "directory path %s/"GEOREP" is longer than PATH_MAX",
+ conf->workdir);
goto out;
}
snprintf (georepdir, PATH_MAX, "%s/"GEOREP, conf->workdir);
- ret = mkdir_if_missing (georepdir);
+ ret = mkdir_p (georepdir, 0777, _gf_true);
if (-1 == ret) {
gf_log ("glusterd", GF_LOG_CRITICAL,
"Unable to create "GEOREP" directory %s",
@@ -370,31 +463,49 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
if (strlen (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP) >= PATH_MAX) {
ret = -1;
gf_log ("glusterd", GF_LOG_CRITICAL,
- "Unable to create "GEOREP" directory %s",
- georepdir);
+ "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
+ GEOREP" is longer than PATH_MAX");
goto out;
}
- ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP);
+ ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP, 0777, _gf_true);
if (-1 == ret) {
gf_log ("glusterd", GF_LOG_CRITICAL,
"Unable to create "GEOREP" log directory");
goto out;
}
+ /* Slave log file directory */
if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves") >= PATH_MAX) {
ret = -1;
gf_log ("glusterd", GF_LOG_CRITICAL,
- "Unable to create "GEOREP" directory %s",
- georepdir);
+ "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
+ GEOREP"-slaves"" is longer than PATH_MAX");
goto out;
}
- ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves");
+ ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves", 0777,
+ _gf_true);
if (-1 == ret) {
gf_log ("glusterd", GF_LOG_CRITICAL,
"Unable to create "GEOREP" slave log directory");
goto out;
}
+ /* MountBroker log file directory */
+ if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr") >= PATH_MAX) {
+ ret = -1;
+ gf_log ("glusterd", GF_LOG_CRITICAL,
+ "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP
+ "-slaves/mbr"" is longer than PATH_MAX");
+ goto out;
+ }
+ ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr", 0777,
+ _gf_true);
+ if (-1 == ret) {
+ gf_log ("glusterd", GF_LOG_CRITICAL,
+ "Unable to create "GEOREP" mountbroker slave log directory");
+ goto out;
+ }
+
ret = dict_get_str (THIS->options, GEOREP"-log-group", &greplg_s);
if (ret)
ret = 0;
@@ -412,20 +523,22 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
if (ret == 0)
ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
GEOREP"-slaves", gr->gr_gid);
+ if (ret == 0)
+ ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
+ GEOREP"-slaves/mbr", gr->gr_gid);
}
out:
gf_log("", GF_LOG_DEBUG, "Returning %d", ret);
return ret;
}
-#endif
static void
runinit_gsyncd_setrx (runner_t *runner, glusterd_conf_t *conf)
{
runinit (runner);
runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
- runner_argprintf (runner, "%s/"GSYNC_CONF,conf->workdir);
+ runner_argprintf (runner, "%s/"GSYNC_CONF_TEMPLATE, conf->workdir);
runner_add_arg (runner, "--config-set-rx");
}
@@ -442,7 +555,6 @@ configure_syncdaemon (glusterd_conf_t *conf)
} while (0)
{
int ret = 0;
-#if SYNCDAEMON_COMPILE
runner_t runner = {0,};
char georepdir[PATH_MAX] = {0,};
int valid_state = 0;
@@ -475,8 +587,8 @@ configure_syncdaemon (glusterd_conf_t *conf)
RUN_GSYNCD_CMD;
runinit_gsyncd_setrx (&runner, conf);
- runner_add_args (&runner, "remote-gsyncd",
- "/usr/local/libexec/glusterfs/gsyncd", ".", "^ssh:", NULL);
+ runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
+ ".", "^ssh:", NULL);
RUN_GSYNCD_CMD;
/* gluster-command-dir */
@@ -488,7 +600,7 @@ configure_syncdaemon (glusterd_conf_t *conf)
/* gluster-params */
runinit_gsyncd_setrx (&runner, conf);
runner_add_args (&runner, "gluster-params",
- "xlator-option=*-dht.assert-no-child-down=true",
+ "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
".", ".", NULL);
RUN_GSYNCD_CMD;
@@ -505,17 +617,45 @@ configure_syncdaemon (glusterd_conf_t *conf)
/* pid-file */
runinit_gsyncd_setrx (&runner, conf);
runner_add_arg (&runner, "pid-file");
- runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.pid", georepdir);
+ runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
runner_add_args (&runner, ".", ".", NULL);
RUN_GSYNCD_CMD;
/* state-file */
runinit_gsyncd_setrx (&runner, conf);
runner_add_arg (&runner, "state-file");
- runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.status", georepdir);
+ runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
+ runner_add_args (&runner, ".", ".", NULL);
+ RUN_GSYNCD_CMD;
+
+ /* state-detail-file */
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_arg (&runner, "state-detail-file");
+ runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+ georepdir);
+ runner_add_args (&runner, ".", ".", NULL);
+ RUN_GSYNCD_CMD;
+
+ /* state-detail-file */
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_arg (&runner, "state-detail-file");
+ runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+ georepdir);
+ runner_add_args (&runner, ".", ".", NULL);
+ RUN_GSYNCD_CMD;
+
+ /* state-socket */
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_arg (&runner, "state-socket-unencoded");
+ runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.socket", georepdir);
runner_add_args (&runner, ".", ".", NULL);
RUN_GSYNCD_CMD;
+ /* socketdir */
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+ RUN_GSYNCD_CMD;
+
/* log-file */
runinit_gsyncd_setrx (&runner, conf);
runner_add_args (&runner,
@@ -528,10 +668,32 @@ configure_syncdaemon (glusterd_conf_t *conf)
runinit_gsyncd_setrx (&runner, conf);
runner_add_args (&runner,
"gluster-log-file",
- DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.gluster.log",
+ DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
".", ".", NULL);
RUN_GSYNCD_CMD;
+ /* ignore-deletes */
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
+ RUN_GSYNCD_CMD;
+
+ /* special-sync-mode */
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
+ RUN_GSYNCD_CMD;
+
+ /* change-detector == changelog */
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+ RUN_GSYNCD_CMD;
+
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_arg(&runner, "working-dir");
+ runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+ DEFAULT_VAR_RUN_DIRECTORY);
+ runner_add_args (&runner, ".", ".", NULL);
+ RUN_GSYNCD_CMD;
+
/************
* slave pre-configuration
************/
@@ -545,7 +707,7 @@ configure_syncdaemon (glusterd_conf_t *conf)
/* gluster-params */
runinit_gsyncd_setrx (&runner, conf);
runner_add_args (&runner, "gluster-params",
- "xlator-option=*-dht.assert-no-child-down=true",
+ "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
".", NULL);
RUN_GSYNCD_CMD;
@@ -557,6 +719,14 @@ configure_syncdaemon (glusterd_conf_t *conf)
".", NULL);
RUN_GSYNCD_CMD;
+ /* MountBroker log-file */
+ runinit_gsyncd_setrx (&runner, conf);
+ runner_add_args (&runner,
+ "log-file-mbr",
+ DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
+ ".", NULL);
+ RUN_GSYNCD_CMD;
+
/* gluster-log-file */
runinit_gsyncd_setrx (&runner, conf);
runner_add_args (&runner,
@@ -566,12 +736,17 @@ configure_syncdaemon (glusterd_conf_t *conf)
RUN_GSYNCD_CMD;
out:
-#else
- (void)conf;
-#endif
return ret ? -1 : 0;
}
#undef RUN_GSYNCD_CMD
+#else /* SYNCDAEMON_COMPILE */
+static int
+configure_syncdaemon (glusterd_conf_t *conf)
+{
+ return 0;
+}
+#endif /* !SYNCDAEMON_COMPILE */
+
static int
check_prepare_mountbroker_root (char *mountbroker_root)
@@ -603,6 +778,11 @@ check_prepare_mountbroker_root (char *mountbroker_root)
ret = -1;
goto out;
}
+ if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
+ gf_log ("", GF_LOG_WARNING,
+ "permissions on mountbroker-root directory %s are "
+ "probably too strict", mountbroker_root);
+ }
dfd0 = dup (dfd);
@@ -631,6 +811,11 @@ check_prepare_mountbroker_root (char *mountbroker_root)
ret = -1;
goto out;
}
+ if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
+ gf_log ("", GF_LOG_WARNING,
+ "permissions on ancestors of mountbroker-root "
+ "directory are probably too strict");
+ }
close (dfd);
dfd = dfd2;
@@ -653,14 +838,17 @@ check_prepare_mountbroker_root (char *mountbroker_root)
ret = 0;
out:
- close (dfd0);
- close (dfd);
- close (dfd2);
+ if (dfd0 != -1)
+ close (dfd0);
+ if (dfd != -1)
+ close (dfd);
+ if (dfd2 != -1)
+ close (dfd2);
return ret;
}
-static void
+static int
_install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
{
glusterd_conf_t *priv = THIS->private;
@@ -669,15 +857,11 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
gf_boolean_t ghadoop = _gf_false;
char *pdesc = value->data;
char *volname = NULL;
- int *ret = data;
int rv = 0;
gf_mount_spec_t *mspec = NULL;
char *user = NULL;
char *volfile_server = NULL;
- if (*ret == -1)
- return;
-
label = strtail (key, "mountbroker.");
/* check for presence of geo-rep/hadoop label */
@@ -685,14 +869,15 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
label = strtail (key, "mountbroker-"GEOREP".");
if (label)
georep = _gf_true;
-
- label = strtail (key, "mountbroker-"GHADOOP".");
- if (label)
- ghadoop = _gf_true;
+ else {
+ label = strtail (key, "mountbroker-"GHADOOP".");
+ if (label)
+ ghadoop = _gf_true;
+ }
}
if (!label)
- return;
+ return 0;
mspec = GF_CALLOC (1, sizeof (*mspec), gf_gld_mt_mount_spec);
if (!mspec)
@@ -731,14 +916,222 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
list_add_tail (&mspec->speclist, &priv->mount_specs);
- return;
+ return 0;
err:
gf_log ("", GF_LOG_ERROR,
"adding %smount spec failed: label: %s desc: %s",
georep ? GEOREP" " : "", label, pdesc);
- *ret = -1;
+ return -1;
+}
+
+
+static int
+gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+ glusterd_conf_t *priv = THIS->private;
+ synclock_unlock (&priv->big_lock);
+ return ret;
+}
+
+static void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
+{
+ xlator_t *this = NULL;
+ glusterd_conf_t *priv = NULL;
+ int ret = -1;
+
+ this = THIS;
+ priv = this->private;
+
+ synclock_lock (&priv->big_lock);
+ ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
+ opaque);
+ if (ret)
+ gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
+ " and other volume related services");
+}
+
+int
+glusterd_uds_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+ void *data)
+{
+ /* glusterd_rpcsvc_notify() does stuff that calls coming in from the
+ * unix domain socket don't need. This is just an empty function to be
+ * used for the uds listener. This will be used later if required.
+ */
+ return 0;
+}
+
+/* The glusterd unix domain socket listener only listens for cli */
+rpcsvc_t *
+glusterd_init_uds_listener (xlator_t *this)
+{
+ int ret = -1;
+ dict_t *options = NULL;
+ rpcsvc_t *rpc = NULL;
+ data_t *sock_data = NULL;
+ char sockfile[PATH_MAX+1] = {0,};
+ int i = 0;
+
+
+ GF_ASSERT (this);
+
+ sock_data = dict_get (this->options, "glusterd-sockfile");
+ if (!sock_data) {
+ strncpy (sockfile, DEFAULT_GLUSTERD_SOCKFILE, PATH_MAX);
+ } else {
+ strncpy (sockfile, sock_data->data, PATH_MAX);
+ }
+
+ options = dict_new ();
+ if (!options)
+ goto out;
+
+ ret = rpcsvc_transport_unix_options_build (&options, sockfile);
+ if (ret)
+ goto out;
+
+ rpc = rpcsvc_init (this, this->ctx, options, 8);
+ if (rpc == NULL) {
+ ret = -1;
+ goto out;
+ }
+
+ ret = rpcsvc_register_notify (rpc, glusterd_uds_rpcsvc_notify,
+ this);
+ if (ret) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "Failed to register notify function");
+ goto out;
+ }
+
+ ret = rpcsvc_create_listeners (rpc, options, this->name);
+ if (ret != 1) {
+ gf_log (this->name, GF_LOG_DEBUG, "Failed to create listener");
+ goto out;
+ }
+ ret = 0;
+
+ for (i = 0; i < gd_uds_programs_count; i++) {
+ ret = glusterd_program_register (this, rpc, gd_uds_programs[i]);
+ if (ret) {
+ i--;
+ for (; i >= 0; i--)
+ rpcsvc_program_unregister (rpc,
+ gd_uds_programs[i]);
+
+ goto out;
+ }
+ }
+
+out:
+ if (options)
+ dict_unref (options);
+
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR, "Failed to start glusterd "
+ "unix domain socket listener.");
+ if (rpc) {
+ GF_FREE (rpc);
+ rpc = NULL;
+ }
+ }
+ return rpc;
+}
+
+void
+glusterd_stop_uds_listener (xlator_t *this)
+{
+ glusterd_conf_t *conf = NULL;
+ rpcsvc_listener_t *listener = NULL;
+ rpcsvc_listener_t *next = NULL;
+
+ GF_ASSERT (this);
+ conf = this->private;
+
+ (void) rpcsvc_program_unregister (conf->uds_rpc, &gd_svc_cli_prog);
+ (void) rpcsvc_program_unregister (conf->uds_rpc, &gluster_handshake_prog);
+
+ list_for_each_entry_safe (listener, next, &conf->uds_rpc->listeners,
+ list) {
+ rpcsvc_listener_destroy (listener);
+ }
+
+ (void) rpcsvc_unregister_notify (conf->uds_rpc, glusterd_rpcsvc_notify,
+ this);
+
+ unlink (DEFAULT_GLUSTERD_SOCKFILE);
+
+ GF_FREE (conf->uds_rpc);
+ conf->uds_rpc = NULL;
+
+ return;
+}
+
+static int
+glusterd_init_snap_folder (xlator_t *this)
+{
+ int ret = -1;
+ struct stat buf = {0,};
+
+ GF_ASSERT (this);
+
+ /* Snapshot volumes are mounted under /var/run/gluster/snaps folder.
+ * But /var/run is normally a symbolic link to /run folder, which
+ * creates problems as the entry point in the mtab for the mount point
+ * and glusterd maintained entry point will be different. Therefore
+ * identify the correct run folder and use it for snap volume mounting.
+ */
+ ret = lstat (GLUSTERD_VAR_RUN_DIR, &buf);
+ if (ret != 0) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "stat fails on %s, exiting. (errno = %d)",
+ GLUSTERD_VAR_RUN_DIR, errno);
+ goto out;
+ }
+
+ /* If /var/run is symlink then use /run folder */
+ if (S_ISLNK (buf.st_mode)) {
+ strcpy (snap_mount_folder, GLUSTERD_RUN_DIR);
+ } else {
+ strcpy (snap_mount_folder, GLUSTERD_VAR_RUN_DIR);
+ }
+
+ strcat (snap_mount_folder, GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
+
+ ret = stat (snap_mount_folder, &buf);
+ if ((ret != 0) && (ENOENT != errno)) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "stat fails on %s, exiting. (errno = %d)",
+ snap_mount_folder, errno);
+ ret = -1;
+ goto out;
+ }
+
+ if ((!ret) && (!S_ISDIR(buf.st_mode))) {
+ gf_log (this->name, GF_LOG_CRITICAL,
+ "Provided snap path %s is not a directory,"
+ "exiting", snap_mount_folder);
+ ret = -1;
+ goto out;
+ }
+
+ if ((-1 == ret) && (ENOENT == errno)) {
+ /* Create missing folders */
+ ret = mkdir_p (snap_mount_folder, 0777, _gf_false);
+
+ if (-1 == ret) {
+ gf_log (this->name, GF_LOG_CRITICAL,
+ "Unable to create directory %s"
+ " ,errno = %d", snap_mount_folder, errno);
+ goto out;
+ }
+ }
+
+out:
+ return ret;
}
/*
@@ -752,57 +1145,67 @@ init (xlator_t *this)
{
int32_t ret = -1;
rpcsvc_t *rpc = NULL;
+ rpcsvc_t *uds_rpc = NULL;
glusterd_conf_t *conf = NULL;
data_t *dir_data = NULL;
struct stat buf = {0,};
- char voldir [PATH_MAX] = {0,};
- char dirname [PATH_MAX];
+ char storedir [PATH_MAX] = {0,};
+ char workdir [PATH_MAX] = {0,};
+ char hooks_dir [PATH_MAX] = {0,};
char cmd_log_filename [PATH_MAX] = {0,};
int first_time = 0;
char *mountbroker_root = NULL;
-
-#ifdef DEBUG
+ int i = 0;
char *valgrind_str = NULL;
-#endif
+
dir_data = dict_get (this->options, "working-directory");
if (!dir_data) {
//Use default working dir
- strncpy (dirname, GLUSTERD_DEFAULT_WORKDIR, PATH_MAX);
+ strncpy (workdir, GLUSTERD_DEFAULT_WORKDIR, PATH_MAX);
} else {
- strncpy (dirname, dir_data->data, PATH_MAX);
+ strncpy (workdir, dir_data->data, PATH_MAX);
}
- ret = stat (dirname, &buf);
+ ret = stat (workdir, &buf);
if ((ret != 0) && (ENOENT != errno)) {
gf_log (this->name, GF_LOG_ERROR,
"stat fails on %s, exiting. (errno = %d)",
- dirname, errno);
+ workdir, errno);
exit (1);
}
if ((!ret) && (!S_ISDIR(buf.st_mode))) {
gf_log (this->name, GF_LOG_CRITICAL,
"Provided working area %s is not a directory,"
- "exiting", dirname);
+ "exiting", workdir);
exit (1);
}
if ((-1 == ret) && (ENOENT == errno)) {
- ret = mkdir (dirname, 0777);
+ ret = mkdir (workdir, 0777);
if (-1 == ret) {
gf_log (this->name, GF_LOG_CRITICAL,
"Unable to create directory %s"
- " ,errno = %d", dirname, errno);
+ " ,errno = %d", workdir, errno);
exit (1);
}
+
first_time = 1;
}
+ setenv ("GLUSTERD_WORKING_DIR", workdir, 1);
gf_log (this->name, GF_LOG_INFO, "Using %s as working directory",
- dirname);
+ workdir);
+
+ ret = glusterd_init_snap_folder (this);
+ if (ret) {
+ gf_log (this->name, GF_LOG_CRITICAL, "Unable to create "
+ "snap backend folder");
+ exit (1);
+ }
snprintf (cmd_log_filename, PATH_MAX,"%s/.cmd_log_history",
DEFAULT_LOG_FILE_DIRECTORY);
@@ -814,59 +1217,68 @@ init (xlator_t *this)
exit (1);
}
- snprintf (voldir, PATH_MAX, "%s/vols", dirname);
+ snprintf (storedir, PATH_MAX, "%s/vols", workdir);
- ret = mkdir (voldir, 0777);
+ ret = mkdir (storedir, 0777);
if ((-1 == ret) && (errno != EEXIST)) {
gf_log (this->name, GF_LOG_CRITICAL,
"Unable to create volume directory %s"
- " ,errno = %d", voldir, errno);
+ " ,errno = %d", storedir, errno);
exit (1);
}
- snprintf (voldir, PATH_MAX, "%s/peers", dirname);
+ snprintf (storedir, PATH_MAX, "%s/peers", workdir);
- ret = mkdir (voldir, 0777);
+ ret = mkdir (storedir, 0777);
if ((-1 == ret) && (errno != EEXIST)) {
gf_log (this->name, GF_LOG_CRITICAL,
"Unable to create peers directory %s"
- " ,errno = %d", voldir, errno);
+ " ,errno = %d", storedir, errno);
exit (1);
}
- snprintf (voldir, PATH_MAX, "%s/bricks", DEFAULT_LOG_FILE_DIRECTORY);
- ret = mkdir (voldir, 0777);
+ snprintf (storedir, PATH_MAX, "%s/bricks", DEFAULT_LOG_FILE_DIRECTORY);
+ ret = mkdir (storedir, 0777);
if ((-1 == ret) && (errno != EEXIST)) {
gf_log (this->name, GF_LOG_CRITICAL,
"Unable to create logs directory %s"
- " ,errno = %d", voldir, errno);
+ " ,errno = %d", storedir, errno);
exit (1);
}
- snprintf (voldir, PATH_MAX, "%s/nfs", dirname);
- ret = mkdir (voldir, 0777);
+ snprintf (storedir, PATH_MAX, "%s/nfs", workdir);
+ ret = mkdir (storedir, 0777);
if ((-1 == ret) && (errno != EEXIST)) {
gf_log (this->name, GF_LOG_CRITICAL,
"Unable to create nfs directory %s"
- " ,errno = %d", voldir, errno);
+ " ,errno = %d", storedir, errno);
+ exit (1);
+ }
+
+ snprintf (storedir, PATH_MAX, "%s/glustershd", workdir);
+ ret = mkdir (storedir, 0777);
+ if ((-1 == ret) && (errno != EEXIST)) {
+ gf_log (this->name, GF_LOG_CRITICAL,
+ "Unable to create glustershd directory %s"
+ " ,errno = %d", storedir, errno);
exit (1);
}
- snprintf (voldir, PATH_MAX, "%s/glustershd", dirname);
- ret = mkdir (voldir, 0777);
+ snprintf (storedir, PATH_MAX, "%s/groups", workdir);
+ ret = mkdir (storedir, 0777);
if ((-1 == ret) && (errno != EEXIST)) {
gf_log (this->name, GF_LOG_CRITICAL,
"Unable to create glustershd directory %s"
- " ,errno = %d", voldir, errno);
+ " ,errno = %d", storedir, errno);
exit (1);
}
ret = glusterd_rpcsvc_options_build (this->options);
if (ret)
goto out;
- rpc = rpcsvc_init (this, this->ctx, this->options);
+ rpc = rpcsvc_init (this, this->ctx, this->options, 64);
if (rpc == NULL) {
gf_log (this->name, GF_LOG_ERROR,
"failed to init rpc");
@@ -892,38 +1304,26 @@ init (xlator_t *this)
goto out;
}
- ret = glusterd_program_register (this, rpc, &glusterd1_mop_prog);
- if (ret) {
- goto out;
- }
-
- ret = glusterd_program_register (this, rpc, &gd_svc_cli_prog);
- if (ret) {
- rpcsvc_program_unregister (rpc, &glusterd1_mop_prog);
- goto out;
- }
-
- ret = glusterd_program_register (this, rpc, &gd_svc_mgmt_prog);
- if (ret) {
- rpcsvc_program_unregister (rpc, &glusterd1_mop_prog);
- rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
- goto out;
- }
+ for (i = 0; i < gd_inet_programs_count; i++) {
+ ret = glusterd_program_register (this, rpc,
+ gd_inet_programs[i]);
+ if (ret) {
+ i--;
+ for (; i >= 0; i--)
+ rpcsvc_program_unregister (rpc,
+ gd_inet_programs[i]);
- ret = glusterd_program_register (this, rpc, &gluster_pmap_prog);
- if (ret) {
- rpcsvc_program_unregister (rpc, &glusterd1_mop_prog);
- rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
- rpcsvc_program_unregister (rpc, &gd_svc_mgmt_prog);
- goto out;
+ goto out;
+ }
}
- ret = glusterd_program_register (this, rpc, &gluster_handshake_prog);
- if (ret) {
- rpcsvc_program_unregister (rpc, &glusterd1_mop_prog);
- rpcsvc_program_unregister (rpc, &gluster_pmap_prog);
- rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
- rpcsvc_program_unregister (rpc, &gd_svc_mgmt_prog);
+ /* Start a unix domain socket listener just for cli commands
+ * This should prevent ports from being wasted by being in TIMED_WAIT
+ * when cli commands are done continuously
+ */
+ uds_rpc = glusterd_init_uds_listener (this);
+ if (uds_rpc == NULL) {
+ ret = -1;
goto out;
}
@@ -933,15 +1333,30 @@ init (xlator_t *this)
conf->shd = GF_CALLOC (1, sizeof (nodesrv_t),
gf_gld_mt_nodesrv_t);
GF_VALIDATE_OR_GOTO(this->name, conf->shd, out);
+ conf->nfs = GF_CALLOC (1, sizeof (nodesrv_t),
+ gf_gld_mt_nodesrv_t);
+ GF_VALIDATE_OR_GOTO(this->name, conf->nfs, out);
INIT_LIST_HEAD (&conf->peers);
INIT_LIST_HEAD (&conf->volumes);
+ INIT_LIST_HEAD (&conf->snapshots);
+ INIT_LIST_HEAD (&conf->missed_snaps_list);
+
pthread_mutex_init (&conf->mutex, NULL);
conf->rpc = rpc;
- conf->gfs_mgmt = &glusterd_glusterfs_3_1_mgmt_prog;
- strncpy (conf->workdir, dirname, PATH_MAX);
+ conf->uds_rpc = uds_rpc;
+ conf->gfs_mgmt = &gd_brick_prog;
+ strncpy (conf->workdir, workdir, PATH_MAX);
+ synclock_init (&conf->big_lock);
+ pthread_mutex_init (&conf->xprt_lock, NULL);
INIT_LIST_HEAD (&conf->xprt_list);
+
+ glusterd_friend_sm_init ();
+ glusterd_op_sm_init ();
+ glusterd_opinfo_init ();
+ glusterd_mgmt_v3_lock_init ();
+ glusterd_txn_opinfo_dict_init ();
ret = glusterd_sm_tr_log_init (&conf->op_sm_log,
glusterd_op_sm_state_name_get,
glusterd_op_sm_event_name_get,
@@ -949,31 +1364,42 @@ init (xlator_t *this)
if (ret)
goto out;
+ conf->base_port = GF_IANA_PRIV_PORTS_START;
+ if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
+ gf_log (this->name, GF_LOG_INFO,
+ "base-port override: %d", conf->base_port);
+ }
+
/* Set option to run bricks on valgrind if enabled in glusterd.vol */
-#ifdef DEBUG
conf->valgrind = _gf_false;
- ret = dict_get_str (this->options, "brick-with-valgrind", &valgrind_str);
+ ret = dict_get_str (this->options, "run-with-valgrind", &valgrind_str);
if (ret < 0) {
- gf_log (THIS->name, GF_LOG_ERROR,
- "cannot get brick-with-valgrind value");
+ gf_log (this->name, GF_LOG_DEBUG,
+ "cannot get run-with-valgrind value");
}
if (valgrind_str) {
if (gf_string2boolean (valgrind_str, &(conf->valgrind))) {
- gf_log (THIS->name, GF_LOG_WARNING,
- "brick-with-valgrind value not a boolean string");
+ gf_log (this->name, GF_LOG_WARNING,
+ "run-with-valgrind value not a boolean string");
}
}
-#endif
+
this->private = conf;
- (void) glusterd_shd_set_running (_gf_false);
- /* this->ctx->top = this;*/
+ (void) glusterd_nodesvc_set_online_status ("glustershd", _gf_false);
- ret = glusterd_uuid_init (first_time);
- if (ret < 0)
- goto out;
+ GLUSTERD_GET_HOOKS_DIR (hooks_dir, GLUSTERD_HOOK_VER, conf);
+ if (stat (hooks_dir, &buf)) {
+ ret = glusterd_hooks_create_hooks_directory (conf->workdir);
+ if (-1 == ret) {
+ gf_log (this->name, GF_LOG_CRITICAL,
+ "Unable to create hooks directory ");
+ exit (1);
+ }
+ }
INIT_LIST_HEAD (&conf->mount_specs);
- dict_foreach (this->options, _install_mount_spec, &ret);
+
+ ret = dict_foreach (this->options, _install_mount_spec, NULL);
if (ret)
goto out;
ret = dict_get_str (this->options, "mountbroker-root",
@@ -993,18 +1419,25 @@ init (xlator_t *this)
if (ret < 0)
goto out;
- glusterd_friend_sm_init ();
- glusterd_op_sm_init ();
- glusterd_opinfo_init ();
+ /* If there are no 'friends', this would be the best time to
+ * spawn process/bricks that may need (re)starting since last
+ * time (this) glusterd was up.*/
+
+ if (list_empty (&conf->peers)) {
+ glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
+ }
+ ret = glusterd_options_init (this);
+ if (ret < 0)
+ goto out;
ret = glusterd_handle_upgrade_downgrade (this->options, conf);
if (ret)
goto out;
- glusterd_restart_bricks (conf);
- ret = glusterd_restart_gsyncds (conf);
+ ret = glusterd_hooks_spawn_worker (this);
if (ret)
goto out;
+
ret = 0;
out:
if (ret < 0) {
@@ -1036,12 +1469,17 @@ fini (xlator_t *this)
goto out;
conf = this->private;
- if (conf->pmap)
- FREE (conf->pmap);
+
+ glusterd_stop_uds_listener (this);
+
+ FREE (conf->pmap);
if (conf->handle)
- glusterd_store_handle_destroy (conf->handle);
+ gf_store_handle_destroy (conf->handle);
glusterd_sm_tr_log_delete (&conf->op_sm_log);
+ glusterd_mgmt_v3_lock_fini ();
+ glusterd_txn_opinfo_dict_fini ();
GF_FREE (conf);
+
this->private = NULL;
out:
return;
@@ -1079,11 +1517,9 @@ notify (xlator_t *this, int32_t event, void *data, ...)
}
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
struct xlator_dumpops dumpops = {
.priv = glusterd_priv,
@@ -1130,8 +1566,32 @@ struct volume_options options[] = {
{ .key = {GEOREP"-log-group"},
.type = GF_OPTION_TYPE_ANY,
},
- { .key = {"brick-with-valgrind"},
+ { .key = {"run-with-valgrind"},
.type = GF_OPTION_TYPE_BOOL,
},
+ { .key = {"server-quorum-type"},
+ .type = GF_OPTION_TYPE_STR,
+ .value = { "none", "server"},
+ .description = "If set to server, enables the specified "
+ "volume to participate in quorum."
+ },
+ { .key = {"server-quorum-ratio"},
+ .type = GF_OPTION_TYPE_PERCENT,
+ .description = "Sets the quorum percentage for the trusted "
+ "storage pool."
+ },
+ { .key = {"glusterd-sockfile"},
+ .type = GF_OPTION_TYPE_PATH,
+ .description = "The socket file on which glusterd should listen for "
+ "cli requests. Default is "DEFAULT_GLUSTERD_SOCKFILE "."
+ },
+ { .key = {"base-port"},
+ .type = GF_OPTION_TYPE_INT,
+ .description = "Sets the base port for portmap query"
+ },
+ { .key = {"snap-brick-path"},
+ .type = GF_OPTION_TYPE_STR,
+ .description = "directory where the bricks for the snapshots will be created"
+ },
{ .key = {NULL} },
};
generated by cgit (git 2.34.1) at 2026-01-04 05:29:46 +0000