core/statedump: statedump enahancements

* append timestamp to the statedump filename to prevent old files
  getting over written
* Add start and end markers to statedump to indicate beginning and
  finishing of statedump information
* Make glusterfs take options through /tmp/glusterdump.options file and
  treating those options with higher prioriry
* do not dump the entire inode table in the statedump. Instead just dump
  the ltable and the fdtable

Signed-off-by: Raghavendra Bhat <raghavendra@redhat.com>

3 files changed, 172 insertions, 26 deletions

diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index b0b75a5ad3e..aafda5ecc27 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -960,6 +960,13 @@ fd_dump (fd_t *fd, char *prefix)
         gf_proc_dump_write("pid", "%llu", fd->pid);
         gf_proc_dump_write("refcount", "%d", fd->refcount);
         gf_proc_dump_write("flags", "%d", fd->flags);
+
+        if (fd->inode) {
+                gf_proc_dump_build_key (key, "inode", NULL);
+                gf_proc_dump_add_section(key);
+                inode_dump (fd->inode, key);
+        }
+
 }
 
 
diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c
index 899d8ef2b1b..8e0b7aeca24 100644
--- a/libglusterfs/src/statedump.c
+++ b/libglusterfs/src/statedump.c
@@ -60,8 +60,9 @@ gf_proc_dump_open (char *dump_dir, char *brickname)
         char path[PATH_MAX] = {0,};
         int  dump_fd = -1;
 
-        snprintf (path, sizeof (path), "%s/%s.%d.dump", (dump_dir ?
-                  dump_dir : "/tmp"), brickname, getpid());
+        snprintf (path, sizeof (path), "%s/%s.%d.dump.%"PRIu64,
+                  (dump_dir ? dump_dir : "/tmp"), brickname, getpid(),
+                  (uint64_t) time (NULL));
 
         dump_fd = open (path, O_CREAT|O_RDWR|O_TRUNC|O_APPEND, 0600);
         if (dump_fd < 0)
@@ -79,6 +80,45 @@ gf_proc_dump_close (void)
         gf_dump_fd = -1;
 }
 
+static int
+gf_proc_dump_set_path (char *dump_options_file)
+{
+        int     ret = -1;
+        FILE    *fp = NULL;
+        char    buf[256];
+        char    *key = NULL, *value = NULL;
+        char    *saveptr = NULL;
+
+        fp = fopen (dump_options_file, "r");
+        if (!fp)
+                goto out;
+
+        ret = fscanf (fp, "%s", buf);
+
+        while (ret != EOF) {
+                key = strtok_r (buf, "=", &saveptr);
+                if (!key) {
+                        ret = fscanf (fp, "%s", buf);
+                        continue;
+                }
+
+                value = strtok_r (NULL, "=", &saveptr);
+
+                if (!value) {
+                        ret = fscanf (fp, "%s", buf);
+                        continue;
+                }
+                if (!strcmp (key, "path")) {
+                        dump_options.dump_path = gf_strdup (value);
+                        break;
+                }
+        }
+
+out:
+        if (fp)
+                fclose (fp);
+        return ret;
+}
 
 int
 gf_proc_dump_add_section (char *key, ...)
@@ -395,8 +435,6 @@ gf_proc_dump_xlator_info (xlator_t *top)
                     (trav->itable)) {
                         snprintf (itable_key, 1024, "%d.%s.itable",
                                   ctx->graph_id, trav->name);
-
-                        inode_table_dump (trav->itable, itable_key);
                 }
 
                 if (!trav->dumpops) {
@@ -430,24 +468,20 @@ static void
 gf_proc_dump_oldgraph_xlator_info (xlator_t *top)
 {
         xlator_t        *trav = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-        char             itable_key[1024] = {0,};
 
         if (!top)
                 return;
 
-        ctx = glusterfs_ctx_get ();
-
         trav = top;
         while (trav) {
                 gf_proc_dump_xlator_mem_info_only_in_use (trav);
 
                 if (GF_PROC_DUMP_IS_XL_OPTION_ENABLED (inode) &&
                     (trav->itable)) {
-                        snprintf (itable_key, 1024, "%d.%s.itable",
-                                  ctx->graph_id, trav->name);
-
-                        inode_table_dump (trav->itable, itable_key);
+                        /*TODO: dump inode table info if necessary by
+                          printing the graph id (taken by glusterfs_cbtx_t)
+                          in the key
+                        */
                 }
 
                 if (!trav->dumpops) {
@@ -488,6 +522,44 @@ gf_proc_dump_enable_all_options ()
         return 0;
 }
 
+gf_boolean_t
+is_gf_proc_dump_all_disabled ()
+{
+        gf_boolean_t all_disabled = _gf_true;
+
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_mem, all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_iobuf, all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_callpool, all_disabled,
+                                   out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_priv,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_inode,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_fd,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_inodectx,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_fdctx,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_history,
+                                   all_disabled, out);
+
+out:
+        return all_disabled;
+}
+
+/* These options are dumped by default if /tmp/glusterdump.options
+   file exists and it is emtpty
+*/
+static int
+gf_proc_dump_enable_default_options ()
+{
+        GF_PROC_DUMP_SET_OPTION (dump_options.dump_mem, _gf_true);
+        GF_PROC_DUMP_SET_OPTION (dump_options.dump_callpool, _gf_true);
+
+        return 0;
+}
+
 static int
 gf_proc_dump_disable_all_options ()
 {
@@ -566,28 +638,43 @@ gf_proc_dump_options_init ()
         int     ret = -1;
         FILE    *fp = NULL;
         char    buf[256];
-        char    dumpbuf[GF_DUMP_MAX_BUF_LEN];
         char    *key = NULL, *value = NULL;
         char    *saveptr = NULL;
         char    dump_option_file[PATH_MAX];
 
+        /* glusterd will create a file /tmp/glusterdump.<pid>.options and
+           sets the statedump options for the process and the file is removed
+           after the statedump is taken. Direct issue of SIGUSR1 does not have
+           mechanism for considering the statedump options. So to have a way
+           of configuring the statedump of all the glusterfs processes through
+           both cli command and SIGUSR1, /tmp/glusterdump.options file
+           is searched and the options mentioned in it are given the higher
+           priority.
+        */
         snprintf (dump_option_file, sizeof (dump_option_file),
-                  "/tmp/glusterdump.%d.options", getpid ());
-
+                  "/tmp/glusterdump.options");
         fp = fopen (dump_option_file, "r");
-
         if (!fp) {
-                //ENOENT, return success
-                (void) gf_proc_dump_enable_all_options ();
-                return 0;
+                snprintf (dump_option_file, sizeof (dump_option_file),
+                          "/tmp/glusterdump.%d.options", getpid ());
+
+                fp = fopen (dump_option_file, "r");
+
+                if (!fp) {
+                        //ENOENT, return success
+                        (void) gf_proc_dump_enable_all_options ();
+                        return 0;
+                }
         }
 
         (void) gf_proc_dump_disable_all_options ();
 
+        // swallow the errors if setting statedump file path is failed.
+        ret = gf_proc_dump_set_path (dump_option_file);
+
         ret = fscanf (fp, "%s", buf);
 
         while (ret != EOF) {
-
                 key = strtok_r (buf, "=", &saveptr);
                 if (!key) {
                         ret = fscanf (fp, "%s", buf);
@@ -601,13 +688,15 @@ gf_proc_dump_options_init ()
                         continue;
                 }
 
-                snprintf (dumpbuf, sizeof (dumpbuf), "[Debug]:key=%s, value=%s\n",key,value);
-                ret = write (gf_dump_fd, dumpbuf, strlen (dumpbuf));
-
                 gf_proc_dump_parse_set_option (key, value);
-
         }
 
+        if (is_gf_proc_dump_all_disabled ())
+                (void) gf_proc_dump_enable_default_options ();
+
+        if (fp)
+                fclose (fp);
+
         return 0;
 }
 
@@ -619,6 +708,9 @@ gf_proc_dump_info (int signum)
         glusterfs_ctx_t   *ctx  = NULL;
         glusterfs_graph_t *trav = NULL;
         char               brick_name[PATH_MAX] = {0,};
+        struct timeval     tv   = {0,};
+        char timestr[256]       = {0,};
+        char sign_string[512]   = {0,};
 
         gf_proc_dump_lock ();
 
@@ -631,14 +723,37 @@ gf_proc_dump_info (int signum)
         } else
                 strncpy (brick_name, "glusterdump", sizeof (brick_name));
 
-        ret = gf_proc_dump_open (ctx->statedump_path, brick_name);
+        ret = gf_proc_dump_options_init ();
         if (ret < 0)
                 goto out;
 
-        ret = gf_proc_dump_options_init ();
+                if (dump_options.dump_path)
+                ret = gf_proc_dump_open (dump_options.dump_path, brick_name);
+        else
+                ret = gf_proc_dump_open (ctx->statedump_path, brick_name);
         if (ret < 0)
                 goto out;
 
+        //continue even though gettimeofday() has failed
+        ret = gettimeofday (&tv, NULL);
+        if (0 == ret) {
+                strftime (timestr, sizeof timestr, "%Y-%m-%d %H:%M:%S",
+                          localtime (&tv.tv_sec));
+                snprintf (timestr + strlen (timestr),
+                          sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
+        }
+
+        snprintf (sign_string, sizeof (sign_string), "DUMP-START-TIME: %s\n",
+                  timestr);
+
+        //swallow the errors of write for start and end marker
+        ret = write (gf_dump_fd, sign_string, strlen (sign_string));
+
+        memset (sign_string, 0, sizeof (sign_string));
+        memset (timestr, 0, sizeof (timestr));
+        memset (&tv, 0, sizeof (tv));
+
         if (GF_PROC_DUMP_IS_OPTION_ENABLED (mem)) {
                 gf_proc_dump_mem_info ();
                 gf_proc_dump_mempool_info (ctx);
@@ -670,9 +785,24 @@ gf_proc_dump_info (int signum)
                 i++;
         }
 
+        ret = gettimeofday (&tv, NULL);
+        if (0 == ret) {
+                strftime (timestr, sizeof timestr, "%Y-%m-%d %H:%M:%s",
+                          localtime (&tv.tv_sec));
+                snprintf (timestr + strlen (timestr),
+                          sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
+        }
+
+        snprintf (sign_string, sizeof (sign_string), "\nDUMP-END-TIME: %s",
+                  timestr);
+        ret = write (gf_dump_fd, sign_string, strlen (sign_string));
+
 out:
         if (gf_dump_fd != -1)
                 gf_proc_dump_close ();
+        GF_FREE (dump_options.dump_path);
+        dump_options.dump_path = NULL;
         gf_proc_dump_unlock ();
 
         return;
diff --git a/libglusterfs/src/statedump.h b/libglusterfs/src/statedump.h
index dc56bda0cbb..e8d304e66d4 100644
--- a/libglusterfs/src/statedump.h
+++ b/libglusterfs/src/statedump.h
@@ -31,6 +31,7 @@ typedef struct gf_dump_options_ {
         gf_boolean_t            dump_iobuf;
         gf_boolean_t            dump_callpool;
         gf_dump_xl_options_t    xl_options; //options for all xlators
+        char                    *dump_path;
 } gf_dump_options_t;
 
 extern gf_dump_options_t dump_options;
@@ -55,6 +56,14 @@ void _gf_proc_dump_build_key (char *key, const char *prefix, char *fmt,...)
 
 #define GF_PROC_DUMP_SET_OPTION(opt,val) opt = val
 
+#define GF_CHECK_DUMP_OPTION_ENABLED(option_dump, var, label)      \
+        do {                                                    \
+                if (option_dump == _gf_true) {                  \
+                        var = _gf_false;                        \
+                        goto label;                             \
+                }                                               \
+        } while (0);
+
 void gf_proc_dump_init();
 
 void gf_proc_dump_fini(void);