WORM/Retention Translator: Implementation of file level WORM

To activate the file level worm feature, the features.read-only and
features.worm options should be switched "off" on the volume and
the features.worm-file-level should be switched "on". Both read-only
and worm or worm-file-level cannot be switched "on" together. The
files which are created when the worm-file-level option is set on the
volume will have their own retention profile.

If both worm and worm-file-level are "on" at that time the worm
which is the volume level worm will have priority over file level
worm. If worm-file level is switched "off" after some time and the
read-only option is switched "on" then read-only will have priority.

The current implementation allows the users to manually transmit
a file to a WORM-Retained state by removing all the write bits of
the file using the chmod command. The file will have a retention
profile which contains the state of the file, mode of retention,
and the default retention time.

The file will be made WORM-Retained for a default of 120 seconds
during which it will be immutable and undeletable and it sets the
atime of the file to the time till which it is retained.
After that period if any fop request comes for that file, will
make the transition from WORM-Retained state to WORM state, where
the file will be immutable but deletable and, it will reset
the atime to the actual atime of the file. If a WORM file needs
to be made undeletable again, it can be done by using the chmod
command with all the write bits removed.

There are two modes of retention:
1. Relax: where the retention time of a WORM-Retained file can be
   increased or decreased.
2. Enterprise: where the retention time of a WORM-Retained file
   can be increased but not be decreased.
Whenever a utime change(touch -a, -t, ...)request comes for a
file it checks the mode of retention before setting the utimes.
This is done only if the file is WORM-Retained but for a WORM file
it will change the utimes.

Lazy auto commit:
Whenever a file gets created it will store the creation time of the
file or if a file already exists then any of the next unlink, link,
truncate or rename fops will set the current time as the start time
in an xattr. The next rename/unlink/truncate/link call will check for the
auto commit period and if is is expired, then it will automatically do
the state transition. If it is a normal file then it gets converted
to WORM-Retained state. If it is a WORM-Retained file and its retention
period is expired, then it gets converted to WORM state.

Added the volume set options for the WORM translator. It allows the users
to change the default values of auto-commit-period, default-retention-period,
retention-mode. To make use of the file-level WORM first we have to set the
'worm-file' option to 'on'. The files which are created when the worm-file
option is set on the volume will get WORM-Retained. Other files will work
as usual and will not be WORMed. The auto-commit-period, retention-mode,
and the default-retention-period values for the file will be set to the values
which are set on the volume when the file is created.

Added the tests to check the basic functionalities of the WORM/Retention feature.

Change-Id: I77bd9777f9395a944d76b5cc35a5b48a3c14d148
BUG: 1326308
Signed-off-by: karthik-us <ksubrahm@redhat.com>
Reviewed-on: http://review.gluster.org/13429
Reviewed-by: Niels de Vos <ndevos@redhat.com>
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>

2 files changed, 114 insertions, 9 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 23890238933..e88116e02ba 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -2075,7 +2075,8 @@ brick_graph_add_ro (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 goto out;
 
         if (dict_get_str_boolean (set_dict, "features.read-only", 0) &&
-            dict_get_str_boolean (set_dict, "features.worm", 0)) {
+            (dict_get_str_boolean (set_dict, "features.worm", 0) ||
+             dict_get_str_boolean (set_dict, "features.worm-file-level", 0))) {
                 gf_msg (THIS->name, GF_LOG_ERROR, errno,
                         GD_MSG_DICT_GET_FAILED,
                         "read-only and worm cannot be set together");
@@ -2107,7 +2108,8 @@ brick_graph_add_worm (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 goto out;
 
         if (dict_get_str_boolean (set_dict, "features.read-only", 0) &&
-            dict_get_str_boolean (set_dict, "features.worm", 0)) {
+            (dict_get_str_boolean (set_dict, "features.worm", 0) ||
+             dict_get_str_boolean (set_dict, "features.worm-file-level", 0))) {
                 gf_msg (THIS->name, GF_LOG_ERROR, 0,
                         GD_MSG_INCOMPATIBLE_VALUE,
                         "read-only and worm cannot be set together");
@@ -2402,8 +2404,6 @@ static volgen_brick_xlator_t server_graph_table[] = {
         {brick_graph_add_server, NULL},
         {brick_graph_add_io_stats, NULL},
         {brick_graph_add_cdc, NULL},
-        {brick_graph_add_ro, NULL},
-        {brick_graph_add_worm, NULL},
         {brick_graph_add_quota, "quota"},
         {brick_graph_add_index, "index"},
         {brick_graph_add_barrier, NULL},
@@ -2412,6 +2412,8 @@ static volgen_brick_xlator_t server_graph_table[] = {
         {brick_graph_add_iot, "io-threads"},
         {brick_graph_add_upcall, "upcall"},
         {brick_graph_add_pump, NULL},
+        {brick_graph_add_ro, NULL},
+        {brick_graph_add_worm, NULL},
         {brick_graph_add_locks, "locks"},
         {brick_graph_add_acl, "acl"},
         {brick_graph_add_bitrot_stub, "bitrot-stub"},
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 5bb9d9077b5..68dec22ecaf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -863,6 +863,82 @@ out:
         return ret;
 }
 
+
+static int
+validate_worm (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+               char *value, char **op_errstr)
+{
+        xlator_t *this      =       NULL;
+        gf_boolean_t b      =       _gf_false;
+        int ret             =       -1;
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+        ret = gf_string2boolean (value, &b);
+        if (ret) {
+                gf_asprintf (op_errstr, "%s is not a valid boolean value. %s "
+                             "expects a valid boolean value.", value, key);
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_INVALID_ENTRY, "%s", *op_errstr);
+        }
+out:
+        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+
+        return ret;
+}
+
+
+static int
+validate_worm_period (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+               char *value, char **op_errstr)
+{
+        xlator_t *this      =       NULL;
+        uint64_t period     =       -1;
+        int ret             =       -1;
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+        ret = gf_string2uint64 (value, &period);
+        if (ret) {
+                gf_asprintf (op_errstr, "%s is not a valid uint64_t value."
+                          " %s expects a valid uint64_t value.", value, key);
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_INVALID_ENTRY, "%s", *op_errstr);
+        }
+out:
+        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+
+        return ret;
+}
+
+
+static int
+validate_reten_mode (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+               char *value, char **op_errstr)
+{
+        xlator_t *this      =       NULL;
+        int ret             =       -1;
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+        if ((strcmp (value, "relax") &&
+            strcmp (value, "enterprise"))) {
+                gf_asprintf (op_errstr, "The value of retention mode should be "
+                             "either relax or enterprise. But the value"
+                             " of %s is %s", key, value);
+                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                        "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+
+        return ret;
+}
+
+
 /* dispatch table for VOLUME SET
  * -----------------------------
  *
@@ -2314,13 +2390,40 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .op_version = 1,
           .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
         },
-        { .key        = "features.worm",
-          .voltype    = "features/worm",
-          .option     = "worm",
-          .value      = "off",
-          .op_version = 2,
+        { .key         = "features.worm",
+          .voltype     = "features/worm",
+          .option      = "worm",
+          .value       = "off",
+          .validate_fn = validate_worm,
+          .op_version  = 2,
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "features.worm-file-level",
+          .voltype     = "features/worm",
+          .option      = "worm-file-level",
+          .value       = "off",
+          .validate_fn = validate_worm,
+          .op_version  = GD_OP_VERSION_3_8_0,
           .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
         },
+        { .key         = "features.default-retention-period",
+          .voltype     = "features/worm",
+          .option      = "default-retention-period",
+          .validate_fn = validate_worm_period,
+          .op_version  = GD_OP_VERSION_3_8_0,
+        },
+        { .key         = "features.retention-mode",
+          .voltype     = "features/worm",
+          .option      = "retention-mode",
+          .validate_fn = validate_reten_mode,
+          .op_version  = GD_OP_VERSION_3_8_0,
+        },
+        { .key         = "features.auto-commit-period",
+          .voltype     = "features/worm",
+          .option      = "auto-commit-period",
+          .validate_fn = validate_worm_period,
+          .op_version  = GD_OP_VERSION_3_8_0,
+        },
         { .key         = "storage.linux-aio",
           .voltype     = "storage/posix",
           .op_version  = 1