glusterd: Gluster should keep PID file in correct location

Currently Gluster keeps process pid information of all the daemons
and brick processes in Gluster configuration file directory
(ie., /var/lib/glusterd/*).

These pid files should be seperate from configuration files.
Deletion of the configuration file directory might result into serious problems.
Also, /var/run/gluster is the default placeholder directory for pid files.

So, with this fix Gluster will keep all process pid information of all
processes in /var/run/gluster/* directory.

> BUG: 1258561
> Signed-off-by: Gaurav Kumar Garg <ggarg@redhat.com>
> Signed-off-by: Saravanakumar Arumugam <sarumuga@redhat.com>
> Reviewed-on: https://review.gluster.org/13580
> Tested-by: MOHIT AGRAWAL <moagrawa@redhat.com>
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
> cherry pick from commit 220d406ad13d840e950eef001a2b36f87570058d

BUG: 1480459
Change-Id: Idb09e3fccb6a7355fbac1df31082637c8d7ab5b4
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
Reviewed-on: https://review.gluster.org/18023
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

20 files changed, 190 insertions, 49 deletions

diff --git a/extras/ocf/volume.in b/extras/ocf/volume.in
index 72fd1213af2..de05373b9e3 100755
--- a/extras/ocf/volume.in
+++ b/extras/ocf/volume.in
@@ -78,6 +78,16 @@ volume_getdir() {
     return 0
 }
 
+volume_getpid_dir() {
+    local volpid_dir
+    volpid_dir="/var/run/gluster/vols/${OCF_RESKEY_volname}"
+
+    [ -d ${volpid_dir} ] || return 1
+
+    echo "${volpid_dir}"
+    return 0
+}
+
 volume_getbricks() {
     local infofile
     local voldir
@@ -92,17 +102,15 @@ volume_getbricks() {
 
 volume_getpids() {
     local bricks
-    local piddir
     local pidfile
     local infofile
-    local voldir
+    local volpid_dir
 
-    voldir=`volume_getdir`
+    volpid_dir=`volume_getpid_dir`
     bricks=`volume_getbricks`
-    piddir="${voldir}/run"
 
     for brick in ${bricks}; do
-	pidfile="${piddir}/${SHORTHOSTNAME}${brick}.pid"
+	pidfile="${volpid_dir}/${SHORTHOSTNAME}${brick}.pid"
 	[ -e $pidfile ] || return 1
 	cat $pidfile
     done
diff --git a/extras/stop-all-gluster-processes.sh b/extras/stop-all-gluster-processes.sh
index 69fd601273e..ea2b48ee88b 100755
--- a/extras/stop-all-gluster-processes.sh
+++ b/extras/stop-all-gluster-processes.sh
@@ -72,7 +72,7 @@ kill_bricks_and_services()
     local pidfile
     local pid
 
-    for pidfile in $(find /var/lib/glusterd/ -name '*.pid');
+    for pidfile in $(find /var/run/gluster/ -name '*.pid');
     do
         local pid=$(cat ${pidfile});
         echo "sending SIG${signal} to pid: ${pid}";
diff --git a/libglusterfs/src/store.c b/libglusterfs/src/store.c
index 1591627b741..199ba2960d1 100644
--- a/libglusterfs/src/store.c
+++ b/libglusterfs/src/store.c
@@ -24,7 +24,7 @@ gf_store_mkdir (char *path)
 {
         int32_t     ret = -1;
 
-        ret = sys_mkdir (path, 0777);
+        ret = mkdir_p (path, 0777, _gf_true);
 
         if ((-1 == ret) && (EEXIST != errno)) {
                 gf_msg ("", GF_LOG_ERROR, errno, LG_MSG_DIR_OP_FAILED, "mkdir()"
diff --git a/tests/basic/mount-nfs-auth.t b/tests/basic/mount-nfs-auth.t
index cd0189788ba..b275e985287 100755
--- a/tests/basic/mount-nfs-auth.t
+++ b/tests/basic/mount-nfs-auth.t
@@ -139,7 +139,7 @@ function stat_nfs () {
 
 # Restarts the NFS server
 function restart_nfs () {
-        local NFS_PID=$(cat ${GLUSTERD_WORKDIR}/nfs/run/nfs.pid)
+        local NFS_PID=$(cat $GLUSTERD_PIDFILEDIR/nfs/nfs.pid)
 
         # kill the NFS-server if it is running
         while ps -q ${NFS_PID} 2>&1 > /dev/null; do
diff --git a/tests/bugs/fuse/bug-858215.t b/tests/bugs/fuse/bug-858215.t
index c8613efb72e..389783be841 100755
--- a/tests/bugs/fuse/bug-858215.t
+++ b/tests/bugs/fuse/bug-858215.t
@@ -40,8 +40,8 @@ TEST touch $M0/newfile;
 TEST stat $M0/newfile;
 TEST rm $M0/newfile;
 
-nfs_pid=$(cat $GLUSTERD_WORKDIR/nfs/run/nfs.pid);
-glustershd_pid=$(cat $GLUSTERD_WORKDIR/glustershd/run/glustershd.pid);
+nfs_pid=$(cat $GLUSTERD_PIDFILEDIR/nfs/nfs.pid);
+glustershd_pid=$(cat $GLUSTERD_PIDFILEDIR/glustershd/glustershd.pid);
 
 pids=$(pidof glusterfs);
 for i in $pids
diff --git a/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t b/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t
index 47403b44ee6..d168866ab63 100644
--- a/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t
+++ b/tests/bugs/glusterd/bug-1225716-brick-online-validation-remove-brick.t
@@ -28,7 +28,6 @@ EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" remove_brick_status_completed_field
 
 #kill a brick process
 kill_brick $V0 $H0 $B0/${V0}1
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "0" brick_up_status $V0 $H0 $B0/${V0}1
 
 #remove-brick commit should pass even if the brick is down
 TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}1 commit
diff --git a/tests/bugs/glusterd/bug-916549.t b/tests/bugs/glusterd/bug-916549.t
index bedbdd60bb6..6e3612dce94 100755
--- a/tests/bugs/glusterd/bug-916549.t
+++ b/tests/bugs/glusterd/bug-916549.t
@@ -8,8 +8,8 @@ TEST glusterd;
 TEST $CLI volume create $V0 $H0:$B0/${V0}1;
 TEST $CLI volume start $V0;
 
-pid_file=$(ls $GLUSTERD_WORKDIR/vols/$V0/run);
-brick_pid=$(cat $GLUSTERD_WORKDIR/vols/$V0/run/$pid_file);
+pid_file=$(ls $GLUSTERD_PIDFILEDIR/vols/$V0/);
+brick_pid=$(cat $GLUSTERD_PIDFILEDIR/vols/$V0/$pid_file);
 
 
 kill -SIGKILL $brick_pid;
diff --git a/tests/bugs/glusterfs-server/bug-861542.t b/tests/bugs/glusterfs-server/bug-861542.t
index d33593257b9..60d1b132fb4 100755
--- a/tests/bugs/glusterfs-server/bug-861542.t
+++ b/tests/bugs/glusterfs-server/bug-861542.t
@@ -39,7 +39,7 @@ TEST $CLI --xml volume status $V0;
 TEST $CLI --xml volume status $V0 detail;
 
 # Kill the brick process. After this, port number for the killed (in this case brick) process must be "N/A".
-kill `cat $GLUSTERD_WORKDIR/vols/$V0/run/$H0-d-backends-brick0.pid`
+kill `cat $GLUSTERD_PIDFILEDIR/vols/$V0/$H0-d-backends-brick0.pid`
 
 EXPECT "N/A" port_field $V0 '0'; # volume status
 EXPECT "N/A" port_field $V0 '1'; # volume status detail
diff --git a/tests/bugs/snapshot/bug-1399598-uss-with-ssl.t b/tests/bugs/snapshot/bug-1399598-uss-with-ssl.t
index 7d6252638b5..f4e4e6ec4d2 100755
--- a/tests/bugs/snapshot/bug-1399598-uss-with-ssl.t
+++ b/tests/bugs/snapshot/bug-1399598-uss-with-ssl.t
@@ -20,7 +20,7 @@ function total_online_bricks
 {
         # This will count snapd, which isn't really a brick, but callers can
         # account for that so it's OK.
-        find $GLUSTERD_WORKDIR -name '*.pid' | wc -l
+        find $GLUSTERD_PIDFILEDIR -name '*.pid' | wc -l
 }
 
 cleanup;
diff --git a/tests/bugs/trace/bug-797171.t b/tests/bugs/trace/bug-797171.t
index 29f96b1be57..b823e477229 100755
--- a/tests/bugs/trace/bug-797171.t
+++ b/tests/bugs/trace/bug-797171.t
@@ -21,8 +21,8 @@ $M0;
 touch $M0/{1..22};
 rm -f $M0/*;
 
-pid_file=$(ls $GLUSTERD_WORKDIR/vols/$V0/run);
-brick_pid=$(cat $GLUSTERD_WORKDIR/vols/$V0/run/$pid_file);
+pid_file=$(ls $GLUSTERD_PIDFILEDIR/vols/$V0/);
+brick_pid=$(cat $GLUSTERD_PIDFILEDIR/vols/$V0/$pid_file);
 
 mkdir $statedumpdir/statedump_tmp/;
 echo "path=$statedumpdir/statedump_tmp" > $statedumpdir/glusterdump.options;
diff --git a/tests/cluster.rc b/tests/cluster.rc
index cd71e6494e8..c1ff8ab5b74 100644
--- a/tests/cluster.rc
+++ b/tests/cluster.rc
@@ -27,6 +27,7 @@ function define_backends() {
     for i in `seq 1 $count`; do
         b="B$i";
         mkdir -p ${!b}/glusterd;
+        mkdir -p ${!b}/run;
     done
 }
 
@@ -43,6 +44,7 @@ function define_glusterds() {
         b="B$i";
         h="H$i";
         wopt="management.working-directory=${!b}/glusterd";
+        ropt="management.run-directory=${!b}/run/gluster";
         bopt="management.transport.socket.bind-address=${!h}";
         popt="--pid-file=${!b}/glusterd.pid";
         sopt="management.glusterd-sockfile=${!b}/glusterd/gd.sock"
@@ -52,11 +54,11 @@ function define_glusterds() {
         logfile=`echo ${0##*/}`_glusterd$i.log
         lopt="--log-file=$logdir/$logfile"
         if [ "$2" == "-LDEBUG" ]; then
-            eval "glusterd_$i='glusterd -LDEBUG --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";
-            eval "glusterd$i='glusterd -LDEBUG --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";
+            eval "glusterd_$i='glusterd -LDEBUG --xlator-option $wopt --xlator-option $bopt --xlator-option $ropt --xlator-option $sopt $lopt $popt'";
+            eval "glusterd$i='glusterd -LDEBUG --xlator-option $wopt --xlator-option $bopt --xlator-option $ropt --xlator-option $sopt $lopt $popt'";
         else
-            eval "glusterd_$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";
-            eval "glusterd$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $sopt $lopt $popt'";
+            eval "glusterd_$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $ropt --xlator-option $sopt $lopt $popt'";
+            eval "glusterd$i='glusterd --xlator-option $wopt --xlator-option $bopt --xlator-option $ropt --xlator-option $sopt $lopt $popt'";
         fi
     done
 }
diff --git a/tests/env.rc.in b/tests/env.rc.in
index 82971c4a8de..3f3394bd5ec 100644
--- a/tests/env.rc.in
+++ b/tests/env.rc.in
@@ -5,6 +5,9 @@ libdir=@libdir@
 PATH=@sbindir@:$PATH
 export PATH
 
+GLUSTERD_PIDFILEDIR=@localstatedir@/run/gluster
+export GLUSTERD_PIDFILEDIR
+
 LD_LIBRARY_PATH=@libdir@:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH
 
diff --git a/tests/include.rc b/tests/include.rc
index ffba54e8176..7470ea13e5f 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -576,6 +576,7 @@ function cleanup()
         # Complete cleanup time
         rm -rf "$B0/*" "/etc/glusterd/*";
         rm -rf $WORKDIRS
+        find $GLUSTERD_PIDFILEDIR -name "*.pid" | xargs rm -rf
         leftover=""
         for d in $WORKDIRS ; do
                 if test -d $d ; then
diff --git a/tests/volume.rc b/tests/volume.rc
index 369e5a706c6..1cee648993b 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -103,7 +103,7 @@ function get_nfs_pid ()
 
 function read_nfs_pidfile ()
 {
-        echo `cat $GLUSTERD_WORKDIR/nfs/run/nfs.pid`
+        echo `cat $GLUSTERD_PIDFILEDIR/nfs/nfs.pid`
 }
 
 function cleanup_statedump {
@@ -230,7 +230,7 @@ function ec_child_up_count_shd {
 }
 
 function get_shd_process_pid {
-        ps auxww | grep glusterfs | grep -E "glustershd/run/glustershd.pid" | awk '{print $2}' | head -1
+        ps auxww | grep glusterfs | grep -E "glustershd/glustershd.pid" | awk '{print $2}' | head -1
 }
 
 function generate_shd_statedump {
@@ -279,7 +279,7 @@ function get_brick_pidfile {
         local host=$2
         local brick=$3
         local brick_hiphenated=$(echo $brick | tr '/' '-')
-        echo $GLUSTERD_WORKDIR/vols/$vol/run/${host}${brick_hiphenated}.pid
+        echo $GLUSTERD_PIDFILEDIR/vols/$vol/${host}${brick_hiphenated}.pid
 }
 
 function get_brick_pid {
@@ -534,7 +534,7 @@ function volume_exists() {
 
 function killall_gluster() {
         pkill gluster
-        find $GLUSTERD_WORKDIR -name '*.pid' | xargs rm -f
+        find $GLUSTERD_PIDFILEDIR -name '*.pid' | xargs rm -f
         sleep 1
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
index 826b4ca7463..b2e06825c32 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
@@ -19,9 +19,8 @@ glusterd_svc_build_snapd_rundir (glusterd_volinfo_t *volinfo,
         char                    workdir[PATH_MAX]      = {0,};
         glusterd_conf_t        *priv                   = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
-
-        snprintf (path, path_len, "%s/run", workdir);
+        GLUSTERD_GET_VOLUME_PID_DIR (workdir, volinfo, priv);
+        snprintf (path, path_len, "%s", workdir);
 }
 
 void
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 8eb301f040f..14ccac8a9e3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1069,6 +1069,19 @@ glusterd_store_voldirpath_set (glusterd_volinfo_t *volinfo, char *voldirpath,
         GLUSTERD_GET_VOLUME_DIR (voldirpath, volinfo, priv);
 }
 
+static void
+glusterd_store_piddirpath_set (glusterd_volinfo_t *volinfo, char *piddirpath,
+                               size_t len)
+{
+        glusterd_conf_t         *priv = NULL;
+
+        GF_ASSERT (volinfo);
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        GLUSTERD_GET_VOLUME_PID_DIR (piddirpath, volinfo, priv);
+}
+
 static int32_t
 glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)
 {
@@ -1085,6 +1098,23 @@ glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)
         return ret;
 }
 
+static int32_t
+glusterd_store_create_volume_run_dir (glusterd_volinfo_t *volinfo)
+{
+        int32_t                 ret = -1;
+        char                    piddirpath[PATH_MAX] = {0,};
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_piddirpath_set (volinfo, piddirpath,
+                                       sizeof (piddirpath));
+
+        ret = gf_store_mkdir (piddirpath);
+
+        gf_msg_debug (THIS->name, 0, "Returning with %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_store_create_snap_dir (glusterd_snap_t *snap)
 {
@@ -1762,6 +1792,10 @@ glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t a
         if (ret)
                 goto out;
 
+        ret = glusterd_store_create_volume_run_dir (volinfo);
+        if (ret)
+                goto out;
+
         ret = glusterd_store_create_vol_shandle_on_absence (volinfo);
         if (ret)
                 goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
index 4d60c856171..f2298653ed7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
@@ -86,8 +86,9 @@ glusterd_svc_init_common (glusterd_svc_t *svc,
                 goto out;
 
         /* Initialize the process mgmt */
-        glusterd_svc_build_pidfile_path (svc_name, workdir, pidfile,
-                                         sizeof(pidfile));
+        glusterd_svc_build_pidfile_path (svc_name, priv->rundir,
+                                         pidfile, sizeof(pidfile));
+
         glusterd_svc_build_volfile_path (svc_name, workdir, volfile,
                                          sizeof (volfile));
 
@@ -133,7 +134,7 @@ int glusterd_svc_init (glusterd_svc_t *svc, char *svc_name)
         priv = this->private;
         GF_ASSERT (priv);
 
-        glusterd_svc_build_rundir (svc_name, priv->workdir, rundir,
+        glusterd_svc_build_rundir (svc_name, priv->rundir, rundir,
                                    sizeof (rundir));
         ret = glusterd_svc_init_common (svc, svc_name, priv->workdir, rundir,
                                         DEFAULT_LOG_FILE_DIRECTORY, NULL);
@@ -276,7 +277,7 @@ glusterd_svc_build_rundir (char *server, char *workdir, char *path, size_t len)
         GF_ASSERT (len == PATH_MAX);
 
         glusterd_svc_build_svcdir (server, workdir, dir, sizeof (dir));
-        snprintf (path, len, "%s/run", dir);
+        snprintf (path, len, "%s", dir);
 }
 
 int
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index e0f166f1d4a..64ff103b5a9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1845,7 +1845,7 @@ glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
 
         priv = this->private;
 
-        GLUSTERD_GET_VOLUME_DIR (volume_dir, volinfo, priv);
+        GLUSTERD_GET_VOLUME_PID_DIR (volume_dir, volinfo, priv);
         GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, export_path);
         snprintf (sock_filepath, PATH_MAX, "%s/run/%s-%s",
                   volume_dir, brickinfo->hostname, export_path);
@@ -1904,7 +1904,6 @@ out:
 static int
 _mk_rundir_p (glusterd_volinfo_t *volinfo)
 {
-        char voldir[PATH_MAX]   = {0,};
         char rundir[PATH_MAX]   = {0,};
         glusterd_conf_t *priv   = NULL;
         xlator_t        *this   = NULL;
@@ -1912,8 +1911,7 @@ _mk_rundir_p (glusterd_volinfo_t *volinfo)
 
         this = THIS;
         priv = this->private;
-        GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv);
-        snprintf (rundir, sizeof (rundir)-1, "%s/run", voldir);
+        GLUSTERD_GET_VOLUME_PID_DIR (rundir, volinfo, priv);
         ret =  mkdir_p (rundir, 0777, _gf_true);
         if (ret)
                 gf_msg (this->name, GF_LOG_ERROR, errno,
@@ -4999,8 +4997,8 @@ glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
         priv = this->private;
         GF_ASSERT (priv);
 
-        glusterd_svc_build_pidfile_path (server, priv->workdir, pidfile,
-                                         sizeof (pidfile));
+        glusterd_svc_build_pidfile_path (server, priv->rundir,
+                                         pidfile, sizeof (pidfile));
 
         if (strcmp(server, priv->shd_svc.name) == 0)
                 svc = &(priv->shd_svc);
@@ -8101,7 +8099,7 @@ glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr)
         }
 
         GLUSTERD_GET_NFS_DIR (path, conf);
-        GLUSTERD_GET_NFS_PIDFILE (pidfile_path, path);
+        GLUSTERD_GET_NFS_PIDFILE (pidfile_path, path, conf);
 
         pidfile = fopen (pidfile_path, "r");
         if (!pidfile) {
@@ -8226,7 +8224,7 @@ glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr)
         }
 
         GLUSTERD_GET_QUOTAD_DIR (path, conf);
-        GLUSTERD_GET_QUOTAD_PIDFILE (pidfile_path, path);
+        GLUSTERD_GET_QUOTAD_PIDFILE (pidfile_path, path, conf);
 
         pidfile = fopen (pidfile_path, "r");
         if (!pidfile) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 68998f0a877..4887ff413bf 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1390,6 +1390,7 @@ init (xlator_t *this)
         struct stat        buf                        = {0,};
         char               storedir[PATH_MAX]         = {0,};
         char               workdir[PATH_MAX]          = {0,};
+        char               rundir[PATH_MAX]           = {0,};
         char               cmd_log_filename[PATH_MAX] = {0,};
         char              *mountbroker_root           = NULL;
         int                i                          = 0;
@@ -1423,6 +1424,17 @@ init (xlator_t *this)
         }
 #endif
 
+        dir_data = dict_get (this->options, "run-directory");
+
+        if (!dir_data) {
+                /* Use default working dir */
+                strncpy (rundir, DEFAULT_VAR_RUN_DIRECTORY, PATH_MAX);
+        } else {
+                strncpy (rundir, dir_data->data, PATH_MAX);
+        }
+
+        dir_data = NULL;
+
         dir_data = dict_get (this->options, "working-directory");
 
         if (!dir_data) {
@@ -1467,6 +1479,11 @@ init (xlator_t *this)
                 GD_MSG_CURR_WORK_DIR_INFO, "Using %s as working directory",
                 workdir);
 
+        setenv ("DEFAULT_VAR_RUN_DIRECTORY", rundir, 1);
+        gf_msg (this->name, GF_LOG_INFO, 0,
+                GD_MSG_CURR_WORK_DIR_INFO, "Using %s as pid file working "
+                "directory", rundir);
+
         ret = glusterd_find_correct_var_run_dir (this, var_run_dir);
         if (ret) {
                 gf_msg (this->name, GF_LOG_CRITICAL, 0,
@@ -1477,6 +1494,7 @@ init (xlator_t *this)
 
         ret = glusterd_init_var_run_dirs (this, var_run_dir,
                                       GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
+
         if (ret) {
                 gf_msg (this->name, GF_LOG_CRITICAL, 0,
                         GD_MSG_CREATE_DIR_FAILED, "Unable to create "
@@ -1496,6 +1514,51 @@ init (xlator_t *this)
                 exit (1);
         }
 
+        ret = glusterd_init_var_run_dirs (this, rundir,
+                                          GLUSTERD_BITD_RUN_DIR);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_CRITICAL, 0,
+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "
+                        "bitd running directory");
+                exit (1);
+        }
+
+        ret = glusterd_init_var_run_dirs (this, rundir,
+                                          GLUSTERD_SCRUB_RUN_DIR);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_CRITICAL, 0,
+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "
+                        "scrub running directory");
+                exit (1);
+        }
+
+        ret = glusterd_init_var_run_dirs (this, rundir,
+                                          GLUSTERD_GLUSTERSHD_RUN_DIR);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_CRITICAL, 0,
+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "
+                        "glustershd running directory");
+                exit (1);
+        }
+
+        ret = glusterd_init_var_run_dirs (this, rundir,
+                                          GLUSTERD_NFS_RUN_DIR);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_CRITICAL, 0,
+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "
+                        "nfs running directory");
+                exit (1);
+        }
+
+        ret = glusterd_init_var_run_dirs (this, rundir,
+                                          GLUSTERD_QUOTAD_RUN_DIR);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_CRITICAL, 0,
+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create "
+                        "quota running directory");
+                exit (1);
+        }
+
         snprintf (cmd_log_filename, PATH_MAX, "%s/cmd_history.log",
                   DEFAULT_LOG_FILE_DIRECTORY);
         ret = gf_cmd_log_init (cmd_log_filename);
@@ -1519,6 +1582,19 @@ init (xlator_t *this)
                 exit (1);
         }
 
+        /*keeping individual volume pid file information in /var/run/gluster* */
+        snprintf (storedir, PATH_MAX, "%s/vols", rundir);
+
+        ret = sys_mkdir (storedir, 0777);
+
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_msg (this->name, GF_LOG_CRITICAL, errno,
+                        GD_MSG_CREATE_DIR_FAILED,
+                        "Unable to create volume directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
         snprintf (storedir, PATH_MAX, "%s/snaps", workdir);
 
         ret = sys_mkdir (storedir, 0777);
@@ -1731,6 +1807,7 @@ init (xlator_t *this)
         conf->uds_rpc = uds_rpc;
         conf->gfs_mgmt = &gd_brick_prog;
         strncpy (conf->workdir, workdir, PATH_MAX);
+        strncpy (conf->rundir, rundir, PATH_MAX);
 
         synclock_init (&conf->big_lock, SYNC_LOCK_RECURSIVE);
         pthread_mutex_init (&conf->xprt_lock, NULL);
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index dd1ff7a66be..722d2f8f420 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -148,6 +148,7 @@ typedef struct {
         gf_boolean_t             trace;
         uuid_t                   uuid;
         char                     workdir[PATH_MAX];
+        char                     rundir[PATH_MAX];
         rpcsvc_t                *rpc;
         glusterd_svc_t           shd_svc;
         glusterd_svc_t           nfs_svc;
@@ -578,6 +579,11 @@ typedef enum {
 #define GLUSTERD_VOL_SNAP_DIR_PREFIX "snaps"
 
 #define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR     "/gluster/snaps"
+#define GLUSTERD_BITD_RUN_DIR                "/bitd"
+#define GLUSTERD_SCRUB_RUN_DIR               "/scrub"
+#define GLUSTERD_GLUSTERSHD_RUN_DIR          "/glustershd"
+#define GLUSTERD_NFS_RUN_DIR                 "/nfs"
+#define GLUSTERD_QUOTAD_RUN_DIR              "/quotad"
 #define GLUSTER_SHARED_STORAGE_BRICK_DIR     GLUSTERD_DEFAULT_WORKDIR"/ss_brick"
 #define GLUSTERD_VAR_RUN_DIR                 "/var/run"
 #define GLUSTERD_RUN_DIR                     "/run"
@@ -613,6 +619,19 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                           volinfo->volname);                            \
         } while (0)
 
+#define GLUSTERD_GET_VOLUME_PID_DIR(path, volinfo, priv)                   \
+do {                                                                       \
+        if (volinfo->is_snap_volume) {                                     \
+                snprintf (path, PATH_MAX, "%s/snaps/%s/%s",                \
+                          priv->rundir,                                    \
+                          volinfo->snapshot->snapname, volinfo->volname);  \
+        } else {                                                           \
+                snprintf (path, PATH_MAX, "%s/vols/%s",                    \
+                          priv->rundir,                                    \
+                          volinfo->volname);                               \
+        }                                                                  \
+} while (0)
+
 #define GLUSTERD_GET_SNAP_DIR(path, snap, priv)                           \
                 snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,   \
                           snap->snapname);
@@ -665,20 +684,20 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
 #define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volinfo,brickinfo, priv) do {      \
                 char exp_path[PATH_MAX] = {0,};                               \
                 char volpath[PATH_MAX]  = {0,};                               \
-                GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv);             \
+                GLUSTERD_GET_VOLUME_PID_DIR (volpath, volinfo, priv);         \
                 GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);  \
-                snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid",              \
-                          volpath, brickinfo->hostname, exp_path);      \
+                snprintf (pidfile, PATH_MAX, "%s/%s-%s.pid",                  \
+                          volpath, brickinfo->hostname, exp_path);            \
         } while (0)
 
-#define GLUSTERD_GET_NFS_PIDFILE(pidfile,nfspath) {                     \
-                snprintf (pidfile, PATH_MAX, "%s/run/nfs.pid",          \
-                          nfspath);                                     \
+#define GLUSTERD_GET_NFS_PIDFILE(pidfile, nfspath, priv) {            \
+                snprintf (pidfile, PATH_MAX, "%s/nfs/nfs.pid",        \
+                          priv->rundir);                              \
         }
 
-#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile,quotadpath) {                     \
-                snprintf (pidfile, PATH_MAX, "%s/run/quotad.pid",          \
-                          quotadpath);                                     \
+#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile, quotadpath, priv) {         \
+                snprintf (pidfile, PATH_MAX, "%s/quotad/quotad.pid",     \
+                           priv->rundir);                                \
         }
 
 #define GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(piddir, volinfo, type) do {           \