From 22ac7241b2f8c1bb3db2678b8b6b9a364f14876c Mon Sep 17 00:00:00 2001 From: Gaurav Kumar Garg Date: Wed, 2 Mar 2016 17:42:07 +0530 Subject: glusterd: Gluster should keep PID file in correct location Currently Gluster keeps process pid information of all the daemons and brick processes in Gluster configuration file directory (ie., /var/lib/glusterd/*). These pid files should be seperate from configuration files. Deletion of the configuration file directory might result into serious problems. Also, /var/run/gluster is the default placeholder directory for pid files. So, with this fix Gluster will keep all process pid information of all processes in /var/run/gluster/* directory. > BUG: 1258561 > Signed-off-by: Gaurav Kumar Garg > Signed-off-by: Saravanakumar Arumugam > Reviewed-on: https://review.gluster.org/13580 > Tested-by: MOHIT AGRAWAL > Smoke: Gluster Build System > CentOS-regression: Gluster Build System > Reviewed-by: Atin Mukherjee > cherry pick from commit 220d406ad13d840e950eef001a2b36f87570058d BUG: 1480459 Change-Id: Idb09e3fccb6a7355fbac1df31082637c8d7ab5b4 Signed-off-by: Mohit Agrawal Reviewed-on: https://review.gluster.org/18023 Reviewed-by: Atin Mukherjee Smoke: Gluster Build System CentOS-regression: Gluster Build System --- .../mgmt/glusterd/src/glusterd-snapd-svc-helper.c | 5 +- xlators/mgmt/glusterd/src/glusterd-store.c | 34 ++++++++++ xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c | 9 +-- xlators/mgmt/glusterd/src/glusterd-utils.c | 14 ++-- xlators/mgmt/glusterd/src/glusterd.c | 77 ++++++++++++++++++++++ xlators/mgmt/glusterd/src/glusterd.h | 37 ++++++++--- 6 files changed, 152 insertions(+), 24 deletions(-) (limited to 'xlators/mgmt/glusterd/src') diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c index 826b4ca7463..b2e06825c32 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c +++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c @@ -19,9 +19,8 @@ glusterd_svc_build_snapd_rundir (glusterd_volinfo_t *volinfo, char workdir[PATH_MAX] = {0,}; glusterd_conf_t *priv = THIS->private; - GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv); - - snprintf (path, path_len, "%s/run", workdir); + GLUSTERD_GET_VOLUME_PID_DIR (workdir, volinfo, priv); + snprintf (path, path_len, "%s", workdir); } void diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index 8eb301f040f..14ccac8a9e3 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -1069,6 +1069,19 @@ glusterd_store_voldirpath_set (glusterd_volinfo_t *volinfo, char *voldirpath, GLUSTERD_GET_VOLUME_DIR (voldirpath, volinfo, priv); } +static void +glusterd_store_piddirpath_set (glusterd_volinfo_t *volinfo, char *piddirpath, + size_t len) +{ + glusterd_conf_t *priv = NULL; + + GF_ASSERT (volinfo); + priv = THIS->private; + GF_ASSERT (priv); + + GLUSTERD_GET_VOLUME_PID_DIR (piddirpath, volinfo, priv); +} + static int32_t glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo) { @@ -1085,6 +1098,23 @@ glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo) return ret; } +static int32_t +glusterd_store_create_volume_run_dir (glusterd_volinfo_t *volinfo) +{ + int32_t ret = -1; + char piddirpath[PATH_MAX] = {0,}; + + GF_ASSERT (volinfo); + + glusterd_store_piddirpath_set (volinfo, piddirpath, + sizeof (piddirpath)); + + ret = gf_store_mkdir (piddirpath); + + gf_msg_debug (THIS->name, 0, "Returning with %d", ret); + return ret; +} + int32_t glusterd_store_create_snap_dir (glusterd_snap_t *snap) { @@ -1762,6 +1792,10 @@ glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t a if (ret) goto out; + ret = glusterd_store_create_volume_run_dir (volinfo); + if (ret) + goto out; + ret = glusterd_store_create_vol_shandle_on_absence (volinfo); if (ret) goto out; diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c index 4d60c856171..f2298653ed7 100644 --- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c @@ -86,8 +86,9 @@ glusterd_svc_init_common (glusterd_svc_t *svc, goto out; /* Initialize the process mgmt */ - glusterd_svc_build_pidfile_path (svc_name, workdir, pidfile, - sizeof(pidfile)); + glusterd_svc_build_pidfile_path (svc_name, priv->rundir, + pidfile, sizeof(pidfile)); + glusterd_svc_build_volfile_path (svc_name, workdir, volfile, sizeof (volfile)); @@ -133,7 +134,7 @@ int glusterd_svc_init (glusterd_svc_t *svc, char *svc_name) priv = this->private; GF_ASSERT (priv); - glusterd_svc_build_rundir (svc_name, priv->workdir, rundir, + glusterd_svc_build_rundir (svc_name, priv->rundir, rundir, sizeof (rundir)); ret = glusterd_svc_init_common (svc, svc_name, priv->workdir, rundir, DEFAULT_LOG_FILE_DIRECTORY, NULL); @@ -276,7 +277,7 @@ glusterd_svc_build_rundir (char *server, char *workdir, char *path, size_t len) GF_ASSERT (len == PATH_MAX); glusterd_svc_build_svcdir (server, workdir, dir, sizeof (dir)); - snprintf (path, len, "%s/run", dir); + snprintf (path, len, "%s", dir); } int diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index e0f166f1d4a..64ff103b5a9 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -1845,7 +1845,7 @@ glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo, priv = this->private; - GLUSTERD_GET_VOLUME_DIR (volume_dir, volinfo, priv); + GLUSTERD_GET_VOLUME_PID_DIR (volume_dir, volinfo, priv); GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, export_path); snprintf (sock_filepath, PATH_MAX, "%s/run/%s-%s", volume_dir, brickinfo->hostname, export_path); @@ -1904,7 +1904,6 @@ out: static int _mk_rundir_p (glusterd_volinfo_t *volinfo) { - char voldir[PATH_MAX] = {0,}; char rundir[PATH_MAX] = {0,}; glusterd_conf_t *priv = NULL; xlator_t *this = NULL; @@ -1912,8 +1911,7 @@ _mk_rundir_p (glusterd_volinfo_t *volinfo) this = THIS; priv = this->private; - GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv); - snprintf (rundir, sizeof (rundir)-1, "%s/run", voldir); + GLUSTERD_GET_VOLUME_PID_DIR (rundir, volinfo, priv); ret = mkdir_p (rundir, 0777, _gf_true); if (ret) gf_msg (this->name, GF_LOG_ERROR, errno, @@ -4999,8 +4997,8 @@ glusterd_add_node_to_dict (char *server, dict_t *dict, int count, priv = this->private; GF_ASSERT (priv); - glusterd_svc_build_pidfile_path (server, priv->workdir, pidfile, - sizeof (pidfile)); + glusterd_svc_build_pidfile_path (server, priv->rundir, + pidfile, sizeof (pidfile)); if (strcmp(server, priv->shd_svc.name) == 0) svc = &(priv->shd_svc); @@ -8101,7 +8099,7 @@ glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr) } GLUSTERD_GET_NFS_DIR (path, conf); - GLUSTERD_GET_NFS_PIDFILE (pidfile_path, path); + GLUSTERD_GET_NFS_PIDFILE (pidfile_path, path, conf); pidfile = fopen (pidfile_path, "r"); if (!pidfile) { @@ -8226,7 +8224,7 @@ glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr) } GLUSTERD_GET_QUOTAD_DIR (path, conf); - GLUSTERD_GET_QUOTAD_PIDFILE (pidfile_path, path); + GLUSTERD_GET_QUOTAD_PIDFILE (pidfile_path, path, conf); pidfile = fopen (pidfile_path, "r"); if (!pidfile) { diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c index 68998f0a877..4887ff413bf 100644 --- a/xlators/mgmt/glusterd/src/glusterd.c +++ b/xlators/mgmt/glusterd/src/glusterd.c @@ -1390,6 +1390,7 @@ init (xlator_t *this) struct stat buf = {0,}; char storedir[PATH_MAX] = {0,}; char workdir[PATH_MAX] = {0,}; + char rundir[PATH_MAX] = {0,}; char cmd_log_filename[PATH_MAX] = {0,}; char *mountbroker_root = NULL; int i = 0; @@ -1423,6 +1424,17 @@ init (xlator_t *this) } #endif + dir_data = dict_get (this->options, "run-directory"); + + if (!dir_data) { + /* Use default working dir */ + strncpy (rundir, DEFAULT_VAR_RUN_DIRECTORY, PATH_MAX); + } else { + strncpy (rundir, dir_data->data, PATH_MAX); + } + + dir_data = NULL; + dir_data = dict_get (this->options, "working-directory"); if (!dir_data) { @@ -1467,6 +1479,11 @@ init (xlator_t *this) GD_MSG_CURR_WORK_DIR_INFO, "Using %s as working directory", workdir); + setenv ("DEFAULT_VAR_RUN_DIRECTORY", rundir, 1); + gf_msg (this->name, GF_LOG_INFO, 0, + GD_MSG_CURR_WORK_DIR_INFO, "Using %s as pid file working " + "directory", rundir); + ret = glusterd_find_correct_var_run_dir (this, var_run_dir); if (ret) { gf_msg (this->name, GF_LOG_CRITICAL, 0, @@ -1477,6 +1494,7 @@ init (xlator_t *this) ret = glusterd_init_var_run_dirs (this, var_run_dir, GLUSTERD_DEFAULT_SNAPS_BRICK_DIR); + if (ret) { gf_msg (this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED, "Unable to create " @@ -1496,6 +1514,51 @@ init (xlator_t *this) exit (1); } + ret = glusterd_init_var_run_dirs (this, rundir, + GLUSTERD_BITD_RUN_DIR); + if (ret) { + gf_msg (this->name, GF_LOG_CRITICAL, 0, + GD_MSG_CREATE_DIR_FAILED, "Unable to create " + "bitd running directory"); + exit (1); + } + + ret = glusterd_init_var_run_dirs (this, rundir, + GLUSTERD_SCRUB_RUN_DIR); + if (ret) { + gf_msg (this->name, GF_LOG_CRITICAL, 0, + GD_MSG_CREATE_DIR_FAILED, "Unable to create " + "scrub running directory"); + exit (1); + } + + ret = glusterd_init_var_run_dirs (this, rundir, + GLUSTERD_GLUSTERSHD_RUN_DIR); + if (ret) { + gf_msg (this->name, GF_LOG_CRITICAL, 0, + GD_MSG_CREATE_DIR_FAILED, "Unable to create " + "glustershd running directory"); + exit (1); + } + + ret = glusterd_init_var_run_dirs (this, rundir, + GLUSTERD_NFS_RUN_DIR); + if (ret) { + gf_msg (this->name, GF_LOG_CRITICAL, 0, + GD_MSG_CREATE_DIR_FAILED, "Unable to create " + "nfs running directory"); + exit (1); + } + + ret = glusterd_init_var_run_dirs (this, rundir, + GLUSTERD_QUOTAD_RUN_DIR); + if (ret) { + gf_msg (this->name, GF_LOG_CRITICAL, 0, + GD_MSG_CREATE_DIR_FAILED, "Unable to create " + "quota running directory"); + exit (1); + } + snprintf (cmd_log_filename, PATH_MAX, "%s/cmd_history.log", DEFAULT_LOG_FILE_DIRECTORY); ret = gf_cmd_log_init (cmd_log_filename); @@ -1519,6 +1582,19 @@ init (xlator_t *this) exit (1); } + /*keeping individual volume pid file information in /var/run/gluster* */ + snprintf (storedir, PATH_MAX, "%s/vols", rundir); + + ret = sys_mkdir (storedir, 0777); + + if ((-1 == ret) && (errno != EEXIST)) { + gf_msg (this->name, GF_LOG_CRITICAL, errno, + GD_MSG_CREATE_DIR_FAILED, + "Unable to create volume directory %s" + " ,errno = %d", storedir, errno); + exit (1); + } + snprintf (storedir, PATH_MAX, "%s/snaps", workdir); ret = sys_mkdir (storedir, 0777); @@ -1731,6 +1807,7 @@ init (xlator_t *this) conf->uds_rpc = uds_rpc; conf->gfs_mgmt = &gd_brick_prog; strncpy (conf->workdir, workdir, PATH_MAX); + strncpy (conf->rundir, rundir, PATH_MAX); synclock_init (&conf->big_lock, SYNC_LOCK_RECURSIVE); pthread_mutex_init (&conf->xprt_lock, NULL); diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index dd1ff7a66be..722d2f8f420 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -148,6 +148,7 @@ typedef struct { gf_boolean_t trace; uuid_t uuid; char workdir[PATH_MAX]; + char rundir[PATH_MAX]; rpcsvc_t *rpc; glusterd_svc_t shd_svc; glusterd_svc_t nfs_svc; @@ -578,6 +579,11 @@ typedef enum { #define GLUSTERD_VOL_SNAP_DIR_PREFIX "snaps" #define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR "/gluster/snaps" +#define GLUSTERD_BITD_RUN_DIR "/bitd" +#define GLUSTERD_SCRUB_RUN_DIR "/scrub" +#define GLUSTERD_GLUSTERSHD_RUN_DIR "/glustershd" +#define GLUSTERD_NFS_RUN_DIR "/nfs" +#define GLUSTERD_QUOTAD_RUN_DIR "/quotad" #define GLUSTER_SHARED_STORAGE_BRICK_DIR GLUSTERD_DEFAULT_WORKDIR"/ss_brick" #define GLUSTERD_VAR_RUN_DIR "/var/run" #define GLUSTERD_RUN_DIR "/run" @@ -613,6 +619,19 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args); volinfo->volname); \ } while (0) +#define GLUSTERD_GET_VOLUME_PID_DIR(path, volinfo, priv) \ +do { \ + if (volinfo->is_snap_volume) { \ + snprintf (path, PATH_MAX, "%s/snaps/%s/%s", \ + priv->rundir, \ + volinfo->snapshot->snapname, volinfo->volname); \ + } else { \ + snprintf (path, PATH_MAX, "%s/vols/%s", \ + priv->rundir, \ + volinfo->volname); \ + } \ +} while (0) + #define GLUSTERD_GET_SNAP_DIR(path, snap, priv) \ snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir, \ snap->snapname); @@ -665,20 +684,20 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args); #define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volinfo,brickinfo, priv) do { \ char exp_path[PATH_MAX] = {0,}; \ char volpath[PATH_MAX] = {0,}; \ - GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv); \ + GLUSTERD_GET_VOLUME_PID_DIR (volpath, volinfo, priv); \ GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path); \ - snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid", \ - volpath, brickinfo->hostname, exp_path); \ + snprintf (pidfile, PATH_MAX, "%s/%s-%s.pid", \ + volpath, brickinfo->hostname, exp_path); \ } while (0) -#define GLUSTERD_GET_NFS_PIDFILE(pidfile,nfspath) { \ - snprintf (pidfile, PATH_MAX, "%s/run/nfs.pid", \ - nfspath); \ +#define GLUSTERD_GET_NFS_PIDFILE(pidfile, nfspath, priv) { \ + snprintf (pidfile, PATH_MAX, "%s/nfs/nfs.pid", \ + priv->rundir); \ } -#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile,quotadpath) { \ - snprintf (pidfile, PATH_MAX, "%s/run/quotad.pid", \ - quotadpath); \ +#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile, quotadpath, priv) { \ + snprintf (pidfile, PATH_MAX, "%s/quotad/quotad.pid", \ + priv->rundir); \ } #define GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(piddir, volinfo, type) do { \ -- cgit