features/libgfchangelog: APIs to process history changelogs.

1. Create directories in following fashion for history API's
   usage when consumer is registered with libgfchangelog
   shared library through gf_changelog_register.
       scratch_dir/.history
       scratch_dir/.history/.current
       scratch_dir/.history/.processed
       scratch_dir/.history/.processing

2. Added new file 'gf-history-changelog.c' and following APIs
   are provided for consumers to process history changelogs.

    1. gf_history_changelog_scan:
            Move processed history changelog file from
            .processing to .processed
    2. gf_history_changelog_next_change:
            Return the next history changelog file entry.
            Zero means all history chanelogs are consumed.
    3. gf_history_changelog_done:
            Scan .processing directory and generate a list of
            change entries.
    4. gf_history_changelog_start_fresh:
            For a set of changelogs, start from the begining.

NOTE: Though this patch provides above funcationalities.
      It is considered functionally full with the
      patch (http://review.gluster.org/#/c/6930/).

Change-Id: I200780c7278e0a6c008910d93faad5858a4b3e76
Original-author: Kotresh H R <khiremat@redhat.com>
Signed-off-by: Kotresh H R <khiremat@redhat.com>
Signed-off-by: Ajeet Jha <ajha@redhat.com>
Reviewed-on: http://review.gluster.org/6998
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Venky Shankar <vshankar@redhat.com>
Tested-by: Venky Shankar <vshankar@redhat.com>

4 files changed, 336 insertions, 2 deletions

diff --git a/xlators/features/changelog/lib/src/Makefile.am b/xlators/features/changelog/lib/src/Makefile.am
index 775f026cf72..28d5a70aab2 100644
--- a/xlators/features/changelog/lib/src/Makefile.am
+++ b/xlators/features/changelog/lib/src/Makefile.am
@@ -17,7 +17,8 @@ lib_LTLIBRARIES = libgfchangelog.la
 CONTRIB_BUILDDIR = $(top_builddir)/contrib
 
 libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-process.c \
-		gf-changelog-helpers.c $(CONTRIBDIR)/uuid/clear.c \
+		gf-changelog-helpers.c gf-history-changelog.c \
+		$(CONTRIBDIR)/uuid/clear.c \
 		$(CONTRIBDIR)/uuid/copy.c $(CONTRIBDIR)/uuid/gen_uuid.c \
 		$(CONTRIBDIR)/uuid/pack.c $(CONTRIBDIR)/uuid/parse.c \
 		$(CONTRIBDIR)/uuid/unparse.c $(CONTRIBDIR)/uuid/uuid_time.c \
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.h b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
index 7e13d937640..2d545da9e82 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-helpers.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
@@ -23,6 +23,7 @@
 #define GF_CHANGELOG_CURRENT_DIR    ".current"
 #define GF_CHANGELOG_PROCESSED_DIR  ".processed"
 #define GF_CHANGELOG_PROCESSING_DIR ".processing"
+#define GF_CHANGELOG_HISTORY_DIR    ".history"
 
 #ifndef MAXLINE
 #define MAXLINE 4096
@@ -68,6 +69,9 @@ typedef struct gf_changelog {
         char gfc_processing_dir[PATH_MAX];
 
         pthread_t gfc_changelog_processor;
+
+        /* Holds gfc for History API */
+        struct gf_changelog *hist_gfc;
 } gf_changelog_t;
 
 int
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
index 4b2b25ad573..0827f2cac6c 100644
--- a/xlators/features/changelog/lib/src/gf-changelog.c
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -82,6 +82,10 @@ __attribute__ ((destructor)) gf_changelog_dtor (void)
         gfc = this->private;
 
         if (gfc) {
+                if (gfc->hist_gfc) {
+                        gf_changelog_cleanup(gfc->hist_gfc);
+                        GF_FREE (gfc->hist_gfc);
+                }
                 gf_changelog_cleanup (gfc);
                 GF_FREE (gfc);
         }
@@ -437,6 +441,7 @@ gf_changelog_register (char *brick_path, char *scratch_dir,
         int             errn = 0;
         xlator_t       *this = NULL;
         gf_changelog_t *gfc  = NULL;
+        char hist_scratch_dir[PATH_MAX] = {0,};
 
         this = THIS;
         if (!this->ctx)
@@ -460,6 +465,52 @@ gf_changelog_register (char *brick_path, char *scratch_dir,
                 goto cleanup;
         }
 
+        /* Begin: Changes for History API */
+        gfc->hist_gfc = NULL;
+
+        gfc->hist_gfc = GF_CALLOC (1, sizeof (*gfc),
+                         gf_changelog_mt_libgfchangelog_t);
+        if (!gfc->hist_gfc)
+                goto cleanup;
+
+        gfc->hist_gfc->gfc_dir = NULL;
+        gfc->hist_gfc->gfc_fd = gfc->hist_gfc->gfc_sockfd = -1;
+        gfc->hist_gfc->this = NULL;
+
+        (void) strncpy (hist_scratch_dir, scratch_dir, PATH_MAX);
+        (void) snprintf (hist_scratch_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_HISTORY_DIR"/",
+                         gfc->gfc_working_dir);
+
+        ret = mkdir_p (hist_scratch_dir, 0600, _gf_false);
+        if (ret) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        gfc->hist_gfc->gfc_working_dir = realpath (hist_scratch_dir, NULL);
+        if (!gfc->hist_gfc->gfc_working_dir) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_changelog_open_dirs (gfc->hist_gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not create entries in history scratch dir");
+                goto cleanup;
+        }
+
+        (void) strncpy (gfc->hist_gfc->gfc_brickpath, brick_path, PATH_MAX);
+
+        for (i=0; i < 256; i++) {
+                gfc->hist_gfc->rfc3986[i] =
+                        (isalnum(i) || i == '~' ||
+                        i == '-' || i == '.' || i == '_') ? i : 0;
+        }
+        /* End: Changes for History API*/
+
         ret = gf_changelog_open_dirs (gfc);
         if (ret) {
                 errn = errno;
@@ -494,7 +545,7 @@ gf_changelog_register (char *brick_path, char *scratch_dir,
                 goto cleanup;
         }
 
-        for (; i < 256; i++) {
+        for (i=0; i < 256; i++) {
                 gfc->rfc3986[i] =
                         (isalnum(i) || i == '~' ||
                         i == '-' || i == '.' || i == '_') ? i : 0;
@@ -506,6 +557,10 @@ gf_changelog_register (char *brick_path, char *scratch_dir,
         goto out;
 
  cleanup:
+        if (gfc->hist_gfc) {
+                gf_changelog_cleanup (gfc->hist_gfc);
+                GF_FREE (gfc->hist_gfc);
+        }
         gf_changelog_cleanup (gfc);
         GF_FREE (gfc);
         this->private = NULL;
diff --git a/xlators/features/changelog/lib/src/gf-history-changelog.c b/xlators/features/changelog/lib/src/gf-history-changelog.c
new file mode 100644
index 00000000000..bfc4cd37dc3
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-history-changelog.c
@@ -0,0 +1,274 @@
+#include <errno.h>
+#include <dirent.h>
+#include <stddef.h>
+#include <sys/types.h>
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+#include "changelog-mem-types.h"
+
+/*@API
+ * gf_history_changelog_done:
+ *    Move processed history changelog file from .processing
+ *    to .processed
+ *
+ * ARGUMENTS:
+ *    file(IN): path to processed history changelog file in
+ *    .processing directory.
+ *
+ * RETURN VALUE:
+ *    0: On success.
+ *   -1: On error.
+ */
+int
+gf_history_changelog_done (char *file)
+{
+        int                     ret                     = -1;
+        char                    *buffer                 = NULL;
+        xlator_t                *this                   = NULL;
+        gf_changelog_t          *gfc                    = NULL;
+        gf_changelog_t          *hist_gfc               = NULL;
+        char                    to_path[PATH_MAX]       = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        hist_gfc = gfc->hist_gfc;
+        if (!hist_gfc)
+                goto out;
+
+        if (!file || !strlen (file))
+                goto out;
+
+        /* make sure 'file' is inside ->gfc_working_dir */
+        buffer = realpath (file, NULL);
+        if (!buffer)
+                goto out;
+
+        if (strncmp (hist_gfc->gfc_working_dir,
+                     buffer, strlen (hist_gfc->gfc_working_dir)))
+                goto out;
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         hist_gfc->gfc_processed_dir, basename (buffer));
+        gf_log (this->name, GF_LOG_DEBUG,
+                "moving %s to processed directory", file);
+        ret = rename (buffer, to_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot move %s to %s (reason: %s)",
+                        file, to_path, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        if (buffer)
+                free (buffer); /* allocated by realpath() */
+        return ret;
+}
+/**
+ * @API
+ *  gf_history_changelog_start_fresh:
+ *     For a set of changelogs, start from the begining.
+ *     It will truncates the history tracker fd.
+ *
+ *  RETURN VALUES:
+ *     0: On success.
+ *    -1: On error.
+ */
+int
+gf_history_changelog_start_fresh ()
+{
+        xlator_t                *this                   = NULL;
+        gf_changelog_t          *gfc                    = NULL;
+        gf_changelog_t          *hist_gfc               = NULL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        errno = EINVAL;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        hist_gfc = gfc->hist_gfc;
+        if (!hist_gfc)
+                goto out;
+
+        if (gf_ftruncate (hist_gfc->gfc_fd, 0))
+                goto out;
+
+        return 0;
+
+ out:
+        return -1;
+}
+
+/*
+ * @API
+ *  gf_history_changelog_next_change:
+ *     Return the next history changelog file entry. Zero means all
+ *     history chanelogs are consumed.
+ *
+ *  ARGUMENTS:
+ *     bufptr(OUT): Path to unprocessed history changelog file
+ *                  from tracker file.
+ *     maxlen(IN): Usually PATH_MAX.
+ *
+ *  RETURN VALUES:
+ *     size: On success.
+ *     -1  : On error.
+ */
+ssize_t
+gf_history_changelog_next_change (char *bufptr, size_t maxlen)
+{
+        ssize_t                 size                    = 0;
+        int                     tracker_fd              = 0;
+        xlator_t                *this                   = NULL;
+        gf_changelog_t          *gfc                    = NULL;
+        gf_changelog_t          *hist_gfc               = NULL;
+        char                    buffer[PATH_MAX]        = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        hist_gfc = gfc->hist_gfc;
+        if (!hist_gfc)
+                goto out;
+
+        tracker_fd = hist_gfc->gfc_fd;
+
+        size = gf_readline (tracker_fd, buffer, maxlen);
+        if (size < 0)
+                goto out;
+        if (size == 0)
+                return 0;
+
+        memcpy (bufptr, buffer, size - 1);
+        *(buffer + size) = '\0';
+
+        return size;
+
+ out:
+        return -1;
+}
+
+/*
+ * @API
+ *  gf_history_changelog_scan:
+ *     Scan and generate a list of change entries.
+ *     Calling this api multiple times (without calling gf_changlog_done())
+ *     would result new changelogs(s) being refreshed in the tracker file.
+ *     This call also acts as a cancellation point for the consumer.
+ *
+ *  RETURN VALUES:
+ *     nr_entries: On success.
+ *     -1        : On error.
+ */
+ssize_t
+gf_history_changelog_scan ()
+{
+        int             ret        = 0;
+        int             tracker_fd = 0;
+        size_t          len        = 0;
+        size_t          off        = 0;
+        xlator_t       *this       = NULL;
+        size_t          nr_entries = 0;
+        gf_changelog_t *gfc        = NULL;
+        gf_changelog_t *hist_gfc        = NULL;
+        struct dirent  *entryp     = NULL;
+        struct dirent  *result     = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        hist_gfc = gfc->hist_gfc;
+        if (!hist_gfc)
+                goto out;
+
+        errno = EINVAL;
+
+        tracker_fd = hist_gfc->gfc_fd;
+
+        if (gf_ftruncate (tracker_fd, 0))
+                goto out;
+
+        len = offsetof(struct dirent, d_name)
+                + pathconf(hist_gfc->gfc_processing_dir, _PC_NAME_MAX) + 1;
+        entryp = GF_CALLOC (1, len,
+                            gf_changelog_mt_libgfchangelog_dirent_t);
+        if (!entryp)
+                goto out;
+
+        rewinddir (hist_gfc->gfc_dir);
+        while (1) {
+                ret = readdir_r (hist_gfc->gfc_dir, entryp, &result);
+                if (ret || !result)
+                        break;
+
+                if ( !strcmp (basename (entryp->d_name), ".")
+                     || !strcmp (basename (entryp->d_name), "..") )
+                        continue;
+
+                nr_entries++;
+
+                GF_CHANGELOG_FILL_BUFFER (hist_gfc->gfc_processing_dir,
+                                          buffer, off,
+                                          strlen (hist_gfc->gfc_processing_dir));
+                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
+                                          off, strlen (entryp->d_name));
+                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
+
+                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error writing changelog filename"
+                                " to tracker file");
+                        break;
+                }
+                off = 0;
+        }
+
+        GF_FREE (entryp);
+
+        if (!result) {
+                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
+                        return nr_entries;
+        }
+ out:
+        return -1;
+}