cli/glusterd: CLI interface for BitRot detection

* BitRot enable/disable CLI per volume
* Volfile generation for Scrubber
* Relevant glusterd infrastructure

Change-Id: I1212af63f93ecc52b22ee6da920e1664f66a1e39
BUG: 1170075
Original-Author: Raghavendra Bhat <raghavendra@redhat.com>
Original-Author: Venky Shankar <vshankar@redhat.com>
Original-Author: Gaurav Kumar Garg <ggarg@redhat.com>
Original-Author: Anand Nekkunti <anekkunt@redhat.com>
Reviewed-on: http://review.gluster.org/9986
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

10 files changed, 337 insertions, 29 deletions

diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
index 90afa4f298b..58a1d2fdb2a 100644
--- a/xlators/mgmt/glusterd/src/Makefile.am
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -16,7 +16,7 @@ glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \
 	glusterd-proc-mgmt.c glusterd-svc-mgmt.c glusterd-shd-svc.c \
 	glusterd-nfs-svc.c glusterd-quotad-svc.c glusterd-svc-helper.c \
 	glusterd-conn-helper.c glusterd-snapd-svc.c glusterd-snapd-svc-helper.c \
-	glusterd-bitd-svc.c
+	glusterd-bitd-svc.c glusterd-scrub-svc.c
 
 
 glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
@@ -37,7 +37,7 @@ noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \
 	glusterd-svc-mgmt.h glusterd-shd-svc.h glusterd-nfs-svc.h \
 	glusterd-quotad-svc.h glusterd-svc-helper.h glusterd-snapd-svc.h \
 	glusterd-snapd-svc-helper.h glusterd-rcu.h glusterd-bitd-svc.h \
-	$(CONTRIBDIR)/userspace-rcu/rculist-extra.h
+	glusterd-scrub-svc.h $(CONTRIBDIR)/userspace-rcu/rculist-extra.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(rpclibdir) -I$(CONTRIBDIR)/rbtree \
diff --git a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
index ab06ab64db0..d007f3a0a0d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
@@ -70,10 +70,10 @@ out:
 int
 glusterd_bitdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
 {
-        int          ret    = -1;
+        int          ret    = -EINVAL;
 
-        if (glusterd_are_all_volumes_stopped ()) {
-                ret = svc->stop (svc, SIGKILL);
+        if (glusterd_all_volumes_with_bitrot_stopped ()) {
+                ret = svc->stop (svc, SIGTERM);
         } else {
                 ret = glusterd_bitdsvc_create_volfile ();
                 if (ret)
@@ -91,6 +91,7 @@ glusterd_bitdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
                 if (ret)
                         goto out;
         }
+
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-bitrot.c b/xlators/mgmt/glusterd/src/glusterd-bitrot.c
index 44fefe82b7e..5bd906312a4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-bitrot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-bitrot.c
@@ -192,6 +192,27 @@ out:
         return ret;
 }
 
+gf_boolean_t
+glusterd_all_volumes_with_bitrot_stopped ()
+{
+        glusterd_conf_t *conf = THIS->private;
+        glusterd_volinfo_t *volinfo = NULL;
+        gf_boolean_t        stopped = _gf_true;
+
+        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                if (!glusterd_is_bitrot_enabled (volinfo))
+                        continue;
+                else if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+                else {
+                        stopped = _gf_false;
+                        break;
+                }
+        }
+
+        return stopped;
+}
+
 static int
 glusterd_manage_bitrot (int opcode)
 {
@@ -207,28 +228,13 @@ glusterd_manage_bitrot (int opcode)
 
         switch (opcode) {
         case GF_BITROT_OPTION_TYPE_ENABLE:
-                /* TO DO:
-                 * Start bitd service. once bitd volfile generation patch
-                 * merge or this patch become dependent of bitd volfile
-                 * generation patch below comment will remove.
-                 * http://review.gluster.org/#/c/9710/
-                 */
-                /*ret = priv->bitd_svc.manager (&(priv->bitd_svc),
-                                                NULL, PROC_START);*/
         case GF_BITROT_OPTION_TYPE_DISABLE:
-
-                /* TO DO:
-                 * Stop bitd service. once bitd volfile generation patch
-                 * merge or this patch become dependent of bitd volfile
-                 * generation patch below comment will remove.
-                 * http://review.gluster.org/#/c/9710/
-                 */
-
-                /*if (glusterd_all_volumes_with_bitrot_stopped ())
-                        ret = glusterd_svc_stop (&(priv->bitd_svc),
-                                                 SIGTERM);
-                */
-                ret = 0;
+                ret = priv->bitd_svc.manager (&(priv->bitd_svc),
+                                                NULL, PROC_START_NO_WAIT);
+                if (ret)
+                        break;
+                ret = priv->scrub_svc.manager (&(priv->scrub_svc), NULL,
+                                               PROC_START_NO_WAIT);
                 break;
         default:
                 ret = 0;
diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
new file mode 100644
index 00000000000..bd1ed390f46
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
@@ -0,0 +1,119 @@
+/*
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "globals.h"
+#include "run.h"
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "glusterd-scrub-svc.h"
+
+char *scrub_svc_name = "scrub";
+
+int
+glusterd_scrubsvc_init (glusterd_svc_t *svc)
+{
+        return glusterd_svc_init (svc, scrub_svc_name,
+                                  glusterd_scrubsvc_manager,
+                                  glusterd_scrubsvc_start,
+                                  glusterd_scrubsvc_stop);
+}
+
+static int
+glusterd_scrubsvc_create_volfile ()
+{
+        char              filepath[PATH_MAX] = {0,};
+        int               ret                = -1;
+        glusterd_conf_t   *conf              = NULL;
+        dict_t            *mod_dict          = NULL;
+        xlator_t          *this              = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+
+
+        mod_dict = dict_new ();
+        if (!mod_dict) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to allocate new "
+                        "dict");
+                goto out;
+        }
+
+        ret = dict_set_uint32 (mod_dict, "trusted-client", GF_CLIENT_TRUSTED);
+        if (ret)
+                goto free_dict;
+
+        glusterd_svc_build_volfile_path (scrub_svc_name, conf->workdir,
+                                         filepath, sizeof (filepath));
+
+        ret = glusterd_create_global_volfile (build_scrub_graph,
+                                              filepath, mod_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create volfile");
+                goto free_dict;
+        }
+
+free_dict:
+        dict_unref (mod_dict);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_scrubsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+{
+        int          ret    = -EINVAL;
+
+        if (glusterd_all_volumes_with_bitrot_stopped ()) {
+                ret = svc->stop (svc, SIGTERM);
+        } else {
+                ret = glusterd_scrubsvc_create_volfile ();
+                if (ret)
+                        goto out;
+
+                ret = svc->stop (svc, SIGKILL);
+                if (ret)
+                        goto out;
+
+                ret = svc->start (svc, flags);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_conn_connect (&(svc->conn));
+                if (ret)
+                        goto out;
+        }
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_scrubsvc_start (glusterd_svc_t *svc, int flags)
+{
+        return glusterd_svc_start (svc, flags, NULL);
+}
+
+int
+glusterd_scrubsvc_stop (glusterd_svc_t *svc, int sig)
+{
+        return glusterd_svc_stop (svc, sig);
+}
+
+int
+glusterd_scrubsvc_reconfigure ()
+{
+        return glusterd_svc_reconfigure (glusterd_scrubsvc_create_volfile);
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h
new file mode 100644
index 00000000000..e2e3b0d2ba2
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h
@@ -0,0 +1,47 @@
+/*
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERD_SCRUB_SVC_H_
+#define _GLUSTERD_SCRUB_SVC_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterd-svc-mgmt.h"
+
+typedef struct glusterd_scrubsvc_ glusterd_scrubsvc_t;
+
+struct glusterd_scrubsvc_{
+        glusterd_svc_t      svc;
+        gf_store_handle_t   *handle;
+};
+
+int
+glusterd_scrubsvc_init (glusterd_svc_t *svc);
+
+int
+glusterd_scrubsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+
+int
+glusterd_scrubsvc_start (glusterd_svc_t *svc, int flags);
+
+int
+glusterd_scrubsvc_stop (glusterd_svc_t *svc, int sig);
+
+int
+glusterd_scrubsvc_reconfigure ();
+
+void
+glusterd_scrubsvc_build_volfile_path (char *server, char *workdir,
+                                     char *volfile, size_t len);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
index 39384545c86..8ee715f1a94 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
@@ -18,6 +18,7 @@
 #include "glusterd-quotad-svc.h"
 #include "glusterd-nfs-svc.h"
 #include "glusterd-bitd-svc.h"
+#include "glusterd-scrub-svc.h"
 
 int
 glusterd_svcs_reconfigure (glusterd_volinfo_t *volinfo)
@@ -55,6 +56,10 @@ glusterd_svcs_reconfigure (glusterd_volinfo_t *volinfo)
         ret = glusterd_bitdsvc_reconfigure ();
         if (ret)
                 goto out;
+
+        ret = glusterd_scrubsvc_reconfigure ();
+        if (ret)
+                goto out;
 out:
         return ret;
 }
@@ -87,6 +92,9 @@ glusterd_svcs_stop ()
         ret = glusterd_svc_stop (&(priv->bitd_svc), SIGTERM);
         if (ret)
                 goto out;
+
+        ret = glusterd_svc_stop (&(priv->scrub_svc), SIGTERM);
+
 out:
         return ret;
 }
@@ -128,12 +136,18 @@ glusterd_svcs_manager (glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        ret = conf->bitd_svc.manager (&(conf->bitd_svc), volinfo,
-                                        PROC_START_NO_WAIT);
+        ret = conf->bitd_svc.manager (&(conf->bitd_svc), NULL,
+                                      PROC_START_NO_WAIT);
         if (ret == -EINVAL)
                 ret = 0;
         if (ret)
                 goto out;
+
+        ret = conf->scrub_svc.manager (&(conf->scrub_svc), NULL,
+                                       PROC_START_NO_WAIT);
+        if (ret == -EINVAL)
+                ret = 0;
+
 out:
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index ea00eea432f..65f446e8548 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -1670,7 +1670,6 @@ static int
 brick_graph_add_acl (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                       dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-
         xlator_t        *xl = NULL;
         int             ret = -1;
 
@@ -4751,6 +4750,99 @@ build_bitd_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 if (voliter->status != GLUSTERD_STATUS_STARTED)
                         continue;
 
+                if (!glusterd_is_bitrot_enabled (voliter))
+                        continue;
+
+                memset (transt, '\0', 16);
+
+                get_transport_type (voliter, set_dict, transt, _gf_false);
+                if (!strcmp (transt, "tcp,rdma"))
+                        strcpy (transt, "tcp");
+
+
+                list_for_each_entry (brickinfo, &voliter->bricks, brick_list) {
+                        if (!glusterd_is_local_brick (this, voliter, brickinfo))
+                                continue;
+                        xl  = volgen_graph_build_client (graph, voliter,
+                                                         brickinfo->hostname,
+                                                         brickinfo->path,
+                                                         brickinfo->brick_id, transt,
+                                                         set_dict);
+                        if (!xl) {
+                                ret = -1;
+                                goto out;
+                        }
+
+                        count++;
+                }
+        }
+
+        bitd_xl = volgen_graph_add_nolink (graph, br_args[0], br_args[1]);
+        if (!bitd_xl) {
+                ret = -1;
+                goto out;
+        }
+
+        txl = first_of (graph);
+        for (trav = txl; count; trav = trav->next)
+                count--;
+
+        for (; trav != txl; trav = trav->prev) {
+                ret = volgen_xlator_link (bitd_xl, trav);
+                if (ret)
+                        goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (set_dict)
+                dict_unref (set_dict);
+
+        gf_log(this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int
+build_scrub_graph (volgen_graph_t *graph, dict_t *mod_dict)
+{
+        volgen_graph_t        cgraph        = {0};
+        glusterd_volinfo_t   *voliter       = NULL;
+        xlator_t             *this          = NULL;
+        glusterd_conf_t      *priv          = NULL;
+        dict_t               *set_dict      = NULL;
+        int                   ret           = 0;
+        xlator_t             *bitd_xl       = NULL;
+        xlator_t             *xl            = NULL;
+        xlator_t             *trav          = NULL;
+        xlator_t             *txl           = NULL;
+        char                 *skey          = NULL;
+        char                  transt[16]    = {0,};
+        glusterd_brickinfo_t *brickinfo     = NULL;
+        char                 *br_args[]     = {"features/bit-rot",
+                                               "bit-rot"};
+        int32_t               count         = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        set_dict = dict_new ();
+        if (!set_dict) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        if (mod_dict)
+                dict_copy (mod_dict, set_dict);
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (voliter->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+
                 memset (transt, '\0', 16);
 
                 get_transport_type (voliter, set_dict, transt, _gf_false);
@@ -4787,6 +4879,10 @@ build_bitd_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 goto out;
         }
 
+        ret = xlator_set_option (bitd_xl, "scrubber", "true");
+        if (ret)
+                goto out;
+
         txl = first_of (graph);
         for (trav = txl; count; trav = trav->next)
                 count--;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 93381fd03eb..02f8df0cf7d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -34,6 +34,7 @@
 #define VKEY_FEATURES_QUOTA       "features.quota"
 #define VKEY_FEATURES_TRASH       "features.trash"
 #define VKEY_FEATURES_BITROT      "features.bitrot"
+#define VKEY_FEATURES_SCRUB       "features.scrub"
 
 #define AUTH_ALLOW_MAP_KEY "auth.allow"
 #define AUTH_REJECT_MAP_KEY "auth.reject"
@@ -185,6 +186,9 @@ int
 build_bitd_graph (volgen_graph_t *graph, dict_t *mod_dict);
 
 int
+build_scrub_graph (volgen_graph_t *graph, dict_t *mod_dict);
+
+int
 glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
                              glusterd_brickinfo_t *brickinfo);
 int
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 223ee98d0a4..6f37338ee09 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -42,6 +42,7 @@
 #include "glusterd-shd-svc.h"
 #include "glusterd-nfs-svc.h"
 #include "glusterd-bitd-svc.h"
+#include "glusterd-scrub-svc.h"
 #include "glusterd-quotad-svc.h"
 #include "glusterd-snapd-svc.h"
 #include "common-utils.h"
@@ -1228,6 +1229,13 @@ glusterd_svc_init_all ()
         }
         gf_log (THIS->name, GF_LOG_DEBUG, "BitD service initialized");
 
+        ret = glusterd_scrubsvc_init (&(priv->scrub_svc));
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to initialized scrub "
+                        "service");
+                goto out;
+        }
+        gf_log (THIS->name, GF_LOG_DEBUG, "scrub service initialized");
 
 out:
         return ret;
@@ -1397,6 +1405,15 @@ init (xlator_t *this)
                 exit (1);
         }
 
+        snprintf (storedir, PATH_MAX, "%s/scrub", workdir);
+        ret = mkdir (storedir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create scrub directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
         snprintf (storedir, PATH_MAX, "%s/glustershd", workdir);
         ret = mkdir (storedir, 0777);
         if ((-1 == ret) && (errno != EEXIST)) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index e7a4a13d3ec..0de34474cb8 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -135,6 +135,7 @@ typedef struct {
         glusterd_svc_t           shd_svc;
         glusterd_svc_t           nfs_svc;
         glusterd_svc_t           bitd_svc;
+        glusterd_svc_t           scrub_svc;
         glusterd_svc_t           quotad_svc;
         struct pmap_registry    *pmap;
         struct cds_list_head     volumes;
@@ -1099,4 +1100,7 @@ glusterd_add_brick_status_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
 int32_t
 glusterd_handle_snap_limit (dict_t *dict, dict_t *rsp_dict);
 
+gf_boolean_t
+glusterd_all_volumes_with_bitrot_stopped ();
+
 #endif