features/quota: Add the quota config xattr to newly added brick

Issue:
Quota directory limit configuration is stored in the xattrs. When a new brick
is added these 'limit-set' xattrs have to be created to the directory in the
new brick. This is done by the dht directory healing when the directory is
created in the new brick. Since 'root' directory is already created DHT doesn't
heal the limit-set xattr root.

Solution:
When the add-brick command is issued run the below hook script to heal the
'limit-set' xattr. The hook script does the following only if limit is
configured on root.
    1. Create an auxiliary mount.
    2. getxattr 'limit-set' on the root
    3. setxattr the same value on the root
But this script needs the volume to be started to make the auxiliary mount.

To handle the case when the add-brick is issued when the volume was stopped,
symlink is created by the 'master' script to the corresponding location and
these two are by default disabled.

So, a 'master' script is added in the add-brick/pre. When add-brick command is
issued, it enables one of the scripts mentioned above based on the condition,
    if volume is started - enable add-brick/post script
    else                 - enable start/post script
After the actual script completes its job, it disables itself.

Note:
The enabling and disabling of the script is based on the glusterd's logic, that
it only runs the scripts which starts its name with 'S'. So,
    Enable     - symlink the file to 'S'*
    Disable    - unlink the symlink.

Change-Id: I2d3947a4d686c54417ec95f530af3bdd3444f4e2
BUG: 969461
Signed-off-by: Varun Shastry <vshastry@redhat.com>
Reviewed-on: http://review.gluster.org/6104
Reviewed-by: Brian Foster <bfoster@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

8 files changed, 277 insertions, 0 deletions

diff --git a/configure.ac b/configure.ac
index 5cc87d1aaef..7bfee047ae2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -162,6 +162,9 @@ AC_CONFIG_FILES([Makefile
                 extras/ocf/volume
                 extras/LinuxRPM/Makefile
                 extras/geo-rep/Makefile
+                extras/hook-scripts/add-brick/Makefile
+                extras/hook-scripts/add-brick/pre/Makefile
+                extras/hook-scripts/add-brick/post/Makefile
                 contrib/fuse-util/Makefile
                 contrib/uuid/uuid_types.h
                 glusterfs-api.pc
diff --git a/extras/hook-scripts/Makefile.am b/extras/hook-scripts/Makefile.am
index f6bded20cbe..0e542b4ccac 100644
--- a/extras/hook-scripts/Makefile.am
+++ b/extras/hook-scripts/Makefile.am
@@ -1 +1,2 @@
 EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh S30samba-stop.sh S30samba-set.sh S56glusterd-geo-rep-create-post.sh
+SUBDIRS = add-brick
diff --git a/extras/hook-scripts/add-brick/Makefile.am b/extras/hook-scripts/add-brick/Makefile.am
new file mode 100644
index 00000000000..6e2701e909e
--- /dev/null
+++ b/extras/hook-scripts/add-brick/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = post pre
+CLEANFILES =
diff --git a/extras/hook-scripts/add-brick/post/Makefile.am b/extras/hook-scripts/add-brick/post/Makefile.am
new file mode 100644
index 00000000000..12f510291a9
--- /dev/null
+++ b/extras/hook-scripts/add-brick/post/Makefile.am
@@ -0,0 +1 @@
+EXTRA_DIST = disabled-quota-root-xattr-heal.sh
diff --git a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh
new file mode 100755
index 00000000000..1866f6abd7e
--- /dev/null
+++ b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh
@@ -0,0 +1,141 @@
+#!/bin/sh
+
+##---------------------------------------------------------------------------
+## This script updates the 'limit-set' xattr on the newly added node. Please
+## refer hook-scripts/add-brick/pre/S28Quota-root-xattr-heal.sh for the complete
+## description.
+## Do the following only if limit configured on root.
+## 1. Do an auxiliary mount.
+## 2. Get 'limit-set' xattr on root
+## 3. Set xattrs with the same value on the root.
+## 4. Disable itself
+##---------------------------------------------------------------------------
+
+QUOTA_CONFIG_XATTR="trusted.glusterfs.quota.limit-set";
+MOUNT_DIR=`mktemp --directory --tmpdir`;
+OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
+PROGNAME="Quota-xattr-heal-add-brick"
+VOL_NAME=
+VERSION=
+VOLUME_OP=
+GLUSTERD_WORKING_DIR=
+ENABLED_NAME="S28Quota-root-xattr-heal.sh"
+
+
+cleanup_mountpoint ()
+{
+        umount -f $MOUNT_DIR;
+        if [ 0 -ne $? ]
+        then
+                return $?
+        fi
+
+        rmdir $MOUNT_DIR;
+        if [ 0 -ne $? ]
+        then
+                return $?
+        fi
+}
+
+##------------------------------------------
+## Parse the arguments
+##------------------------------------------
+ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+eval set -- "$ARGS"
+
+while true;
+do
+        case $1 in
+        --volname)
+                 shift
+                 VOL_NAME=$1
+                 ;;
+        --version)
+                 shift
+                 VERSION=$1
+                 ;;
+         --gd-workdir)
+                 shift
+                 GLUSTERD_WORKING_DIR=$1
+                 ;;
+         --volume-op)
+                 shift
+                 VOLUME_OP=$1
+                 ;;
+        *)
+                 shift
+                 break
+                 ;;
+        esac
+        shift
+done
+##----------------------------------------
+
+ENABLED_STATE="$GLUSTERD_WORKING_DIR/hooks/$VERSION/$VOLUME_OP/post/$ENABLED_NAME"
+
+
+FLAG=`gluster volume quota $VOL_NAME list / 2>&1 | grep \
+      '\(No quota configured on volume\)\|\(Limit not set\)'`;
+if ! [ -z $FLAG ]
+then
+        ls $ENABLED_STATE;
+        RET=$?
+        if [ 0 -eq $RET ]
+        then
+                unlink $ENABLED_STATE;
+                exit $?
+        fi
+
+        exit $RET;
+fi
+
+## -----------------------------------
+## Mount the volume in temp directory.
+## -----------------------------------
+glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-42 $MOUNT_DIR;
+if [ 0 -ne $? ]
+then
+        exit $?;
+fi
+## -----------------------------------
+
+## ------------------
+## Getfattr the value
+## ------------------
+VALUE=`getfattr -n "$QUOTA_CONFIG_XATTR" -e hex --absolute-names $MOUNT_DIR \
+       2>&1 | grep $QUOTA_CONFIG_XATTR | awk -F'=' '{print $2}'`
+RET=$?
+if [ 0 -ne $RET ]
+then
+        ## Clean up and exit
+        cleanup_mountpoint;
+
+        exit $RET;
+fi
+## ------------------
+
+## ---------
+## Set xattr
+## ---------
+setfattr -n "$QUOTA_CONFIG_XATTR" -v $VALUE $MOUNT_DIR;
+RET=$?
+if [ 0 -ne $RET ]
+then
+        ## Clean up and exit
+        cleanup_mountpoint;
+
+        exit $RET;
+fi
+## ---------
+
+cleanup_mountpoint;
+
+## Disable
+ls $ENABLED_STATE;
+RET=$?
+if [ 0 -eq $RET ]
+then
+        unlink $ENABLED_STATE;
+        exit $?
+fi
+exit $?
diff --git a/extras/hook-scripts/add-brick/pre/Makefile.am b/extras/hook-scripts/add-brick/pre/Makefile.am
new file mode 100644
index 00000000000..4d22d3c489b
--- /dev/null
+++ b/extras/hook-scripts/add-brick/pre/Makefile.am
@@ -0,0 +1 @@
+EXTRA_DIST = S28Quota-enable-root-xattr-heal.sh
diff --git a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
new file mode 100755
index 00000000000..2580a4c5885
--- /dev/null
+++ b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh
@@ -0,0 +1,100 @@
+#!/bin/sh
+
+###############################################################################
+## ----------------------------------------------------------------------------
+## The scripts
+## I.   add-brick/pre/S28Quota-root-xattr-heal.sh (itself)
+## II.  add-brick/post/disabled-root-xattr-heal.sh AND
+## collectively acheives the job of healing the 'limit-set' xattr upon
+## add-brick to the gluster volume.
+##
+## This script is the 'controlling' script. Upon add-brick this script enables
+## the corresponding script based on the status of the volume.
+## If volume is started - enable add-brick/post script
+## else                 - enable start/post script.
+##
+## The enabling and disabling of a script is based on the glusterd's logic,
+## that it only runs the scripts which starts its name with 'S'. So,
+## Enable - symlink the file to 'S'*.
+## Disable- unlink symlink
+## ----------------------------------------------------------------------------
+###############################################################################
+
+OPTSPEC="volname:,version:,gd-workdir:,volume-op:"
+PROGNAME="Quota-xattr-heal-add-brick-pre"
+VOL_NAME=
+GLUSTERD_WORKING_DIR=
+VOLUME_OP=
+VERSION=
+ENABLED_NAME="S28Quota-root-xattr-heal.sh"
+DISABLED_NAME="disabled-quota-root-xattr-heal.sh"
+
+enable ()
+{
+        ln -sf $DISABLED_STATE $1;
+}
+
+##------------------------------------------
+## Parse the arguments
+##------------------------------------------
+ARGS=$(getopt -l $OPTSPEC  -name $PROGNAME $@)
+eval set -- "$ARGS"
+
+while true;
+do
+        case $1 in
+        --volname)
+                 shift
+                 VOL_NAME=$1
+                 ;;
+        --gd-workdir)
+                 shift
+                 GLUSTERD_WORKING_DIR=$1
+                 ;;
+        --volume-op)
+                 shift
+                 VOLUME_OP=$1
+                 ;;
+        --version)
+                 shift
+                 VERSION=$1
+                 ;;
+        *)
+                 shift
+                 break
+                 ;;
+        esac
+        shift
+done
+##----------------------------------------
+
+DISABLED_STATE="$GLUSTERD_WORKING_DIR/hooks/$VERSION/add-brick/post/$DISABLED_NAME"
+ENABLED_STATE_START="$GLUSTERD_WORKING_DIR/hooks/$VERSION/start/post/$ENABLED_NAME"
+ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKING_DIR/hooks/$VERSION/add-brick/post/$ENABLED_NAME";
+
+## Why to proceed if the required script itself is not present?
+ls $DISABLED_STATE;
+if [ 0 -ne $? ]
+then
+        exit $?;
+fi
+
+## Is quota enabled?
+FLAG=`cat $GLUSTERD_WORKING_DIR/vols/$VOL_NAME/info | grep "^features.quota" \
+      | awk -F'=' '{print $NF}'`;
+if [ "$FLAG" != "on" ]
+then
+        exit $EXIT_SUCCESS;
+fi
+
+## Is volume started?
+FLAG=`cat $GLUSTERD_WORKING_DIR/vols/$VOL_NAME/info | grep "^status" \
+      | awk -F'=' '{print $NF}'`;
+if [ "$FLAG" != "1" ]
+then
+        enable $ENABLED_STATE_START;
+        exit $?
+fi
+
+enable $ENABLED_STATE_ADD_BRICK;
+exit $?
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.c b/xlators/mgmt/glusterd/src/glusterd-hooks.c
index 2b43a452e0c..352b6ba1197 100644
--- a/xlators/mgmt/glusterd/src/glusterd-hooks.c
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.c
@@ -143,6 +143,24 @@ glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op)
         return glusterd_hook_dirnames[op];
 }
 
+void
+glusterd_hooks_add_working_dir (runner_t *runner, glusterd_conf_t *priv)
+{
+        runner_argprintf (runner, "--gd-workdir=%s", priv->workdir);
+}
+
+void
+glusterd_hooks_add_op (runner_t *runner, char *op)
+{
+        runner_argprintf (runner, "--volume-op=%s", op);
+}
+
+void
+glusterd_hooks_add_hooks_version (runner_t* runner)
+{
+        runner_argprintf (runner, "--version=%d", GLUSTERD_HOOK_VER);
+}
+
 int
 glusterd_hooks_set_volume_args (dict_t *dict, runner_t *runner)
 {
@@ -216,6 +234,11 @@ glusterd_hooks_add_op_args (runner_t *runner, glusterd_op_t op,
 
                         runner_argprintf (runner, "--first=%s",
                                           truth? "yes":"no");
+
+                        glusterd_hooks_add_hooks_version (runner);
+                        glusterd_hooks_add_op (runner, "start");
+                        glusterd_hooks_add_working_dir (runner, priv);
+
                         break;
 
                 case GD_OP_STOP_VOLUME:
@@ -250,6 +273,11 @@ glusterd_hooks_add_op_args (runner_t *runner, glusterd_op_t op,
                                 runner_argprintf (runner, "%s", hooks_args);
                         break;
 
+                case GD_OP_ADD_BRICK:
+                        glusterd_hooks_add_hooks_version (runner);
+                        glusterd_hooks_add_op (runner, "add-brick");
+                        glusterd_hooks_add_working_dir (runner, priv);
+
                 default:
                         break;