path: root/libglusterfs/src/statedump.c

/*
  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
  This file is part of GlusterFS.

  This file is licensed to you under your choice of the GNU Lesser
  General Public License, version 3 or any later version (LGPLv3 or
  later), or the GNU General Public License, version 2 (GPLv2), in all
  cases as published by the Free Software Foundation.
*/

#include <stdarg.h>
#include "glusterfs.h"
#include "logging.h"
#include "iobuf.h"
#include "statedump.h"
#include "stack.h"
#include "common-utils.h"

#ifdef HAVE_MALLOC_H
#include <malloc.h>
#endif /* MALLOC_H */

/* We don't want gf_log in this function because it may cause
   'deadlock' with statedump. This is because statedump happens
   inside a signal handler and cannot afford to block on a lock.*/
#ifdef gf_log
# undef gf_log
#endif

#define GF_PROC_DUMP_IS_OPTION_ENABLED(opt)     \
        (dump_options.dump_##opt == _gf_true)

#define GF_PROC_DUMP_IS_XL_OPTION_ENABLED(opt)                  \
        (dump_options.xl_options.dump_##opt == _gf_true)

extern xlator_t global_xlator;

static pthread_mutex_t  gf_proc_dump_mutex;
static int gf_dump_fd = -1;
gf_dump_options_t dump_options;


static void
gf_proc_dump_lock (void)
{
        pthread_mutex_lock (&gf_proc_dump_mutex);
}


static void
gf_proc_dump_unlock (void)
{
        pthread_mutex_unlock (&gf_proc_dump_mutex);
}

static int
gf_proc_dump_open (char *tmpname)
{
        int  dump_fd = -1;

        dump_fd = mkstemp (tmpname);
        if (dump_fd < 0)
                return -1;

        gf_dump_fd = dump_fd;
        return 0;
}

static void
gf_proc_dump_close (void)
{
        close (gf_dump_fd);
        gf_dump_fd = -1;
}

static int
gf_proc_dump_set_path (char *dump_options_file)
{
        int     ret = -1;
        FILE    *fp = NULL;
        char    buf[256];
        char    *key = NULL, *value = NULL;
        char    *saveptr = NULL;

        fp = fopen (dump_options_file, "r");
        if (!fp)
                goto out;

        ret = fscanf (fp, "%s", buf);

        while (ret != EOF) {
                key = strtok_r (buf, "=", &saveptr);
                if (!key) {
                        ret = fscanf (fp, "%s", buf);
                        continue;
                }

                value = strtok_r (NULL, "=", &saveptr);

                if (!value) {
                        ret = fscanf (fp, "%s", buf);
                        continue;
                }
                if (!strcmp (key, "path")) {
                        dump_options.dump_path = gf_strdup (value);
                        break;
                }
        }

out:
        if (fp)
                fclose (fp);
        return ret;
}

int
gf_proc_dump_add_section (char *key, ...)
{

        char buf[GF_DUMP_MAX_BUF_LEN];
        va_list ap;

        GF_ASSERT(key);

        memset (buf, 0, sizeof(buf));
        snprintf (buf, GF_DUMP_MAX_BUF_LEN, "\n[");
        va_start (ap, key);
        vsnprintf (buf + strlen(buf),
                   GF_DUMP_MAX_BUF_LEN - strlen (buf), key, ap);
        va_end (ap);
        snprintf (buf + strlen(buf),
                  GF_DUMP_MAX_BUF_LEN - strlen (buf),  "]\n");
        return write (gf_dump_fd, buf, strlen (buf));
}


int
gf_proc_dump_write (char *key, char *value,...)
{

        char         buf[GF_DUMP_MAX_BUF_LEN];
        int          offset = 0;
        va_list      ap;

        GF_ASSERT (key);

        offset = strlen (key);

        memset (buf, 0, GF_DUMP_MAX_BUF_LEN);
        snprintf (buf, GF_DUMP_MAX_BUF_LEN, "%s", key);
        snprintf (buf + offset, GF_DUMP_MAX_BUF_LEN - offset, "=");
        offset += 1;
        va_start (ap, value);
        vsnprintf (buf + offset, GF_DUMP_MAX_BUF_LEN - offset, value, ap);
        va_end (ap);

        offset = strlen (buf);
        snprintf (buf + offset, GF_DUMP_MAX_BUF_LEN - offset, "\n");
        return write (gf_dump_fd, buf, strlen (buf));
}

static void
gf_proc_dump_xlator_mem_info (xlator_t *xl)
{
        int     i = 0;
        struct mem_acct rec = {0,};

        if (!xl)
                return;

        if (!xl->mem_acct.rec)
                return;

        gf_proc_dump_add_section ("%s.%s - Memory usage", xl->type, xl->name);
        gf_proc_dump_write ("num_types", "%d", xl->mem_acct.num_types);

        for (i = 0; i < xl->mem_acct.num_types; i++) {
                if (!(memcmp (&xl->mem_acct.rec[i], &rec,
                              sizeof (struct mem_acct))))
                        continue;

                gf_proc_dump_add_section ("%s.%s - usage-type %d memusage",
                                          xl->type, xl->name, i);
                gf_proc_dump_write ("size", "%u", xl->mem_acct.rec[i].size);
                gf_proc_dump_write ("num_allocs", "%u",
                                    xl->mem_acct.rec[i].num_allocs);
                gf_proc_dump_write ("max_size", "%u",
                                    xl->mem_acct.rec[i].max_size);
                gf_proc_dump_write ("max_num_allocs", "%u",
                                    xl->mem_acct.rec[i].max_num_allocs);
                gf_proc_dump_write ("total_allocs", "%u",
                                    xl->mem_acct.rec[i].total_allocs);
        }

        return;
}

static void
gf_proc_dump_xlator_mem_info_only_in_use (xlator_t *xl)
{
        int     i = 0;

        if (!xl)
                return;

        if (!xl->mem_acct.rec)
                return;

        gf_proc_dump_add_section ("%s.%s - Memory usage", xl->type, xl->name);
        gf_proc_dump_write ("num_types", "%d", xl->mem_acct.num_types);

        for (i = 0; i < xl->mem_acct.num_types; i++) {
                if (!xl->mem_acct.rec[i].size)
                        continue;

                gf_proc_dump_add_section ("%s.%s - usage-type %d", xl->type,
                                          xl->name,i);

                gf_proc_dump_write ("size", "%u",
                                    xl->mem_acct.rec[i].size);
                gf_proc_dump_write ("max_size", "%u",
                                    xl->mem_acct.rec[i].max_size);
                gf_proc_dump_write ("num_allocs", "%u",
                                    xl->mem_acct.rec[i].num_allocs);
                gf_proc_dump_write ("max_num_allocs", "%u",
                                    xl->mem_acct.rec[i].max_num_allocs);
                gf_proc_dump_write ("total_allocs", "%u",
                                    xl->mem_acct.rec[i].total_allocs);
        }

        return;
}



/* Currently this dumps only mallinfo. More can be built on here */
void
gf_proc_dump_mem_info ()
{
#ifdef HAVE_MALLOC_STATS
        struct mallinfo info;

        memset (&info, 0, sizeof (struct mallinfo));
        info = mallinfo ();

        gf_proc_dump_add_section ("mallinfo");
        gf_proc_dump_write ("mallinfo_arena", "%d", info.arena);
        gf_proc_dump_write ("mallinfo_ordblks", "%d", info.ordblks);
        gf_proc_dump_write ("mallinfo_smblks", "%d", info.smblks);
        gf_proc_dump_write ("mallinfo_hblks", "%d", info.hblks);
        gf_proc_dump_write ("mallinfo_hblkhd", "%d", info.hblkhd);
        gf_proc_dump_write ("mallinfo_usmblks", "%d", info.usmblks);
        gf_proc_dump_write ("mallinfo_fsmblks", "%d", info.fsmblks);
        gf_proc_dump_write ("mallinfo_uordblks", "%d", info.uordblks);
        gf_proc_dump_write ("mallinfo_fordblks", "%d", info.fordblks);
        gf_proc_dump_write ("mallinfo_keepcost", "%d", info.keepcost);
#endif
        gf_proc_dump_xlator_mem_info(&global_xlator);

}

void
gf_proc_dump_mem_info_to_dict (dict_t *dict)
{
        if (!dict)
                return;
#ifdef HAVE_MALLOC_STATS
        struct  mallinfo info;
        int     ret = -1;

        memset (&info, 0, sizeof(struct mallinfo));
        info = mallinfo ();

        ret = dict_set_int32 (dict, "mallinfo.arena", info.arena);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.ordblks", info.ordblks);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.smblks", info.smblks);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.hblks", info.hblks);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.hblkhd", info.hblkhd);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.usmblks", info.usmblks);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.fsmblks", info.fsmblks);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.uordblks", info.uordblks);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.fordblks", info.fordblks);
        if (ret)
                return;

        ret = dict_set_int32 (dict, "mallinfo.keepcost", info.keepcost);
        if (ret)
                return;
#endif
        return;
}

void
gf_proc_dump_mempool_info (glusterfs_ctx_t *ctx)
{
        struct mem_pool *pool = NULL;

        gf_proc_dump_add_section ("mempool");

        list_for_each_entry (pool, &ctx->mempool_list, global_list) {
                gf_proc_dump_write ("-----", "-----");
                gf_proc_dump_write ("pool-name", "%s", pool->name);
                gf_proc_dump_write ("hot-count", "%d", pool->hot_count);
                gf_proc_dump_write ("cold-count", "%d", pool->cold_count);
                gf_proc_dump_write ("padded_sizeof", "%lu",
                                    pool->padded_sizeof_type);
                gf_proc_dump_write ("alloc-count", "%"PRIu64, pool->alloc_count);
                gf_proc_dump_write ("max-alloc", "%d", pool->max_alloc);

                gf_proc_dump_write ("pool-misses", "%"PRIu64, pool->pool_misses);
                gf_proc_dump_write ("max-stdalloc", "%d", pool->max_stdalloc);
        }
}

void
gf_proc_dump_mempool_info_to_dict (glusterfs_ctx_t *ctx, dict_t *dict)
{
        struct mem_pool *pool = NULL;
        char            key[GF_DUMP_MAX_BUF_LEN] = {0,};
        int             count = 0;
        int             ret = -1;

        if (!ctx || !dict)
                return;

        list_for_each_entry (pool, &ctx->mempool_list, global_list) {
                memset (key, 0, sizeof (key));
                snprintf (key, sizeof (key), "pool%d.name", count);
                ret = dict_set_str (dict, key, pool->name);
                if (ret)
                        return;

                memset (key, 0, sizeof (key));
                snprintf (key, sizeof (key), "pool%d.hotcount", count);
                ret = dict_set_int32 (dict, key, pool->hot_count);
                if (ret)
                        return;

                memset (key, 0, sizeof (key));
                snprintf (key, sizeof (key), "pool%d.coldcount", count);
                ret = dict_set_int32 (dict, key, pool->cold_count);
                if (ret)
                        return;

                memset (key, 0, sizeof (key));
                snprintf (key, sizeof (key), "pool%d.paddedsizeof", count);
                ret = dict_set_uint64 (dict, key, pool->padded_sizeof_type);
                if (ret)
                        return;

                memset (key, 0, sizeof (key));
                snprintf (key, sizeof (key), "pool%d.alloccount", count);
                ret = dict_set_uint64 (dict, key, pool->alloc_count);
                if (ret)
                        return;

                memset (key, 0, sizeof (key));
                snprintf (key, sizeof (key), "pool%d.max_alloc", count);
                ret = dict_set_int32 (dict, key, pool->max_alloc);
                if (ret)
                        return;

                memset (key, 0, sizeof (key));
                snprintf (key, sizeof (key), "pool%d.max-stdalloc", count);
                ret = dict_set_int32 (dict, key, pool->max_stdalloc);
                if (ret)
                        return;

                memset (key, 0, sizeof (key));
                snprintf (key, sizeof (key), "pool%d.pool-misses", count);
                ret = dict_set_uint64 (dict, key, pool->pool_misses);
                if (ret)
                        return;
                count++;
        }
        ret = dict_set_int32 (dict, "mempool-count", count);

        return;
}

void gf_proc_dump_latency_info (xlator_t *xl);

void
gf_proc_dump_xlator_info (xlator_t *top)
{
        xlator_t        *trav = NULL;
        glusterfs_ctx_t *ctx = NULL;
        char             itable_key[1024] = {0,};

        if (!top)
                return;

        ctx = top->ctx;

        trav = top;
        while (trav) {

                if (ctx->measure_latency)
                        gf_proc_dump_latency_info (trav);

                gf_proc_dump_xlator_mem_info(trav);

                if (GF_PROC_DUMP_IS_XL_OPTION_ENABLED (inode) &&
                    (trav->itable)) {
                        snprintf (itable_key, 1024, "%d.%s.itable",
                                  ctx->graph_id, trav->name);
                }

                if (!trav->dumpops) {
                        trav = trav->next;
                        continue;
                }

                if (trav->dumpops->priv &&
                    GF_PROC_DUMP_IS_XL_OPTION_ENABLED (priv))
                        trav->dumpops->priv (trav);

                if (GF_PROC_DUMP_IS_XL_OPTION_ENABLED (inode) &&
                    (trav->dumpops->inode))
                        trav->dumpops->inode (trav);

                if (trav->dumpops->fd &&
                    GF_PROC_DUMP_IS_XL_OPTION_ENABLED (fd))
                        trav->dumpops->fd (trav);

                if (trav->dumpops->history &&
                    GF_PROC_DUMP_IS_XL_OPTION_ENABLED (history))
                        trav->dumpops->history (trav);

                trav = trav->next;
        }

        return;
}

static void
gf_proc_dump_oldgraph_xlator_info (xlator_t *top)
{
        xlator_t        *trav = NULL;

        if (!top)
                return;

        trav = top;
        while (trav) {
                gf_proc_dump_xlator_mem_info_only_in_use (trav);

                if (GF_PROC_DUMP_IS_XL_OPTION_ENABLED (inode) &&
                    (trav->itable)) {
                        /*TODO: dump inode table info if necessary by
                          printing the graph id (taken by glusterfs_cbtx_t)
                          in the key
                        */
                }

                if (!trav->dumpops) {
                        trav = trav->next;
                        continue;
                }

                if (GF_PROC_DUMP_IS_XL_OPTION_ENABLED (inode) &&
                    (trav->dumpops->inode))
                        trav->dumpops->inode (trav);

                if (trav->dumpops->fd &&
                    GF_PROC_DUMP_IS_XL_OPTION_ENABLED (fd))
                        trav->dumpops->fd (trav);

                trav = trav->next;
        }

        return;
}

static int
gf_proc_dump_enable_all_options ()
{

        GF_PROC_DUMP_SET_OPTION (dump_options.dump_mem, _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.dump_iobuf, _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.dump_callpool, _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_priv, _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_inode, _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_fd, _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_inodectx,
                                 _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_fdctx, _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_history,
                                 _gf_true);

        return 0;
}

gf_boolean_t
is_gf_proc_dump_all_disabled ()
{
        gf_boolean_t all_disabled = _gf_true;

        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_mem, all_disabled, out);
        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_iobuf, all_disabled, out);
        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_callpool, all_disabled,
                                   out);
        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_priv,
                                   all_disabled, out);
        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_inode,
                                   all_disabled, out);
        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_fd,
                                   all_disabled, out);
        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_inodectx,
                                   all_disabled, out);
        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_fdctx,
                                   all_disabled, out);
        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_history,
                                   all_disabled, out);

out:
        return all_disabled;
}

/* These options are dumped by default if glusterdump.options
   file exists and it is emtpty
*/
static int
gf_proc_dump_enable_default_options ()
{
        GF_PROC_DUMP_SET_OPTION (dump_options.dump_mem, _gf_true);
        GF_PROC_DUMP_SET_OPTION (dump_options.dump_callpool, _gf_true);

        return 0;
}

static int
gf_proc_dump_disable_all_options ()
{

        GF_PROC_DUMP_SET_OPTION (dump_options.dump_mem, _gf_false);
        GF_PROC_DUMP_SET_OPTION (dump_options.dump_iobuf, _gf_false);
        GF_PROC_DUMP_SET_OPTION (dump_options.dump_callpool, _gf_false);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_priv, _gf_false);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_inode,
                                 _gf_false);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_fd, _gf_false);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_inodectx,
                                 _gf_false);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_fdctx, _gf_false);
        GF_PROC_DUMP_SET_OPTION (dump_options.xl_options.dump_history,
                                 _gf_false);
        return 0;
}

static int
gf_proc_dump_parse_set_option (char *key, char *value)
{
        gf_boolean_t    *opt_key = NULL;
        gf_boolean_t    opt_value = _gf_false;
        char buf[GF_DUMP_MAX_BUF_LEN];
        int ret = -1;

        if (!strcasecmp (key, "all")) {
                (void)gf_proc_dump_enable_all_options ();
                return 0;
        } else if (!strcasecmp (key, "mem")) {
                opt_key = &dump_options.dump_mem;
        } else if (!strcasecmp (key, "iobuf")) {
                opt_key = &dump_options.dump_iobuf;
        } else if (!strcasecmp (key, "callpool")) {
                opt_key = &dump_options.dump_callpool;
        } else if (!strcasecmp (key, "priv")) {
                opt_key = &dump_options.xl_options.dump_priv;
        } else if (!strcasecmp (key, "fd")) {
                opt_key = &dump_options.xl_options.dump_fd;
        } else if (!strcasecmp (key, "inode")) {
                opt_key = &dump_options.xl_options.dump_inode;
        } else if (!strcasecmp (key, "inodectx")) {
                opt_key = &dump_options.xl_options.dump_inodectx;
        } else if (!strcasecmp (key, "fdctx")) {
                opt_key = &dump_options.xl_options.dump_fdctx;
        } else if (!strcasecmp (key, "history")) {
                opt_key = &dump_options.xl_options.dump_history;
        }

        if (!opt_key) {
                //None of dump options match the key, return back
                snprintf (buf, sizeof (buf), "[Warning]:None of the options "
                          "matched key : %s\n", key);
                ret = write (gf_dump_fd, buf, strlen (buf));

                if (ret >= 0)
                        ret = -1;
                goto out;

        }

        opt_value = (strncasecmp (value, "yes", 3) ?
                     _gf_false: _gf_true);

        GF_PROC_DUMP_SET_OPTION (*opt_key, opt_value);

        ret = 0;
out:
        return ret;
}

static int
gf_proc_dump_options_init ()
{
        int     ret = -1;
        FILE    *fp = NULL;
        char    buf[256];
        char    *key = NULL, *value = NULL;
        char    *saveptr = NULL;
        char    dump_option_file[PATH_MAX];

        /* glusterd will create a file glusterdump.<pid>.options and
           sets the statedump options for the process and the file is removed
           after the statedump is taken. Direct issue of SIGUSR1 does not have
           mechanism for considering the statedump options. So to have a way
           of configuring the statedump of all the glusterfs processes through
           both cli command and SIGUSR1, glusterdump.options file is searched
           and the options mentioned in it are given the higher priority.
        */
        snprintf (dump_option_file, sizeof (dump_option_file),
                  DEFAULT_VAR_RUN_DIRECTORY
                  "/glusterdump.options");
        fp = fopen (dump_option_file, "r");
        if (!fp) {
                snprintf (dump_option_file, sizeof (dump_option_file),
                          DEFAULT_VAR_RUN_DIRECTORY
                          "/glusterdump.%d.options", getpid ());

                fp = fopen (dump_option_file, "r");

                if (!fp) {
                        //ENOENT, return success
                        (void) gf_proc_dump_enable_all_options ();
                        return 0;
                }
        }

        (void) gf_proc_dump_disable_all_options ();

        // swallow the errors if setting statedump file path is failed.
        ret = gf_proc_dump_set_path (dump_option_file);

        ret = fscanf (fp, "%s", buf);

        while (ret != EOF) {
                key = strtok_r (buf, "=", &saveptr);
                if (!key) {
                        ret = fscanf (fp, "%s", buf);
                        continue;
                }

                value = strtok_r (NULL, "=", &saveptr);

                if (!value) {
                        ret = fscanf (fp, "%s", buf);
                        continue;
                }

                gf_proc_dump_parse_set_option (key, value);
        }

        if (is_gf_proc_dump_all_disabled ())
                (void) gf_proc_dump_enable_default_options ();

        if (fp)
                fclose (fp);

        return 0;
}

void
gf_proc_dump_info (int signum, glusterfs_ctx_t *ctx)
{
        int                i                       = 0;
        int                ret                     = -1;
        glusterfs_graph_t *trav                    = NULL;
        char               brick_name[PATH_MAX]    = {0,};
        char               timestr[256]            = {0,};
        char               sign_string[512]        = {0,};
        char               tmp_dump_name[PATH_MAX] = {0,};
        char               path[PATH_MAX]          = {0,};
        struct timeval     tv                      = {0,};

        gf_proc_dump_lock ();

        if (!ctx)
                goto out;

        if (ctx->cmd_args.brick_name) {
                GF_REMOVE_SLASH_FROM_PATH (ctx->cmd_args.brick_name, brick_name);
        } else
                strncpy (brick_name, "glusterdump", sizeof (brick_name));

        ret = gf_proc_dump_options_init ();
        if (ret < 0)
                goto out;

        snprintf (path, sizeof (path), "%s/%s.%d.dump.%"PRIu64,
                  ((dump_options.dump_path != NULL)?dump_options.dump_path:
                   ((ctx->statedump_path != NULL)?ctx->statedump_path:
                    DEFAULT_VAR_RUN_DIRECTORY)), brick_name, getpid(),
                  (uint64_t) time (NULL));

        snprintf (tmp_dump_name, PATH_MAX, "%s/dumpXXXXXX",
                  ((dump_options.dump_path != NULL)?dump_options.dump_path:
                   ((ctx->statedump_path != NULL)?ctx->statedump_path:
                    DEFAULT_VAR_RUN_DIRECTORY)));

        ret = gf_proc_dump_open (tmp_dump_name);
        if (ret < 0)
                goto out;

        //continue even though gettimeofday() has failed
        ret = gettimeofday (&tv, NULL);
        if (0 == ret) {
                gf_time_fmt (timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT);
                snprintf (timestr + strlen (timestr),
                          sizeof timestr - strlen (timestr),
                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
        }

        snprintf (sign_string, sizeof (sign_string), "DUMP-START-TIME: %s\n",
                  timestr);

        //swallow the errors of write for start and end marker
        ret = write (gf_dump_fd, sign_string, strlen (sign_string));

        memset (sign_string, 0, sizeof (sign_string));
        memset (timestr, 0, sizeof (timestr));
        memset (&tv, 0, sizeof (tv));

        if (GF_PROC_DUMP_IS_OPTION_ENABLED (mem)) {
                gf_proc_dump_mem_info ();
                gf_proc_dump_mempool_info (ctx);
        }

        if (GF_PROC_DUMP_IS_OPTION_ENABLED (iobuf))
                iobuf_stats_dump (ctx->iobuf_pool);
        if (GF_PROC_DUMP_IS_OPTION_ENABLED (callpool))
                gf_proc_dump_pending_frames (ctx->pool);

        if (ctx->master) {
                gf_proc_dump_add_section ("fuse");
                gf_proc_dump_xlator_info (ctx->master);
        }

        if (ctx->active) {
                gf_proc_dump_add_section ("active graph - %d", ctx->graph_id);
                gf_proc_dump_xlator_info (ctx->active->top);
        }

        i = 0;
        list_for_each_entry (trav, &ctx->graphs, list) {
                if (trav == ctx->active)
                        continue;

                gf_proc_dump_add_section ("oldgraph[%d]", i);

                gf_proc_dump_oldgraph_xlator_info (trav->top);
                i++;
        }

        ret = gettimeofday (&tv, NULL);
        if (0 == ret) {
                gf_time_fmt (timestr, sizeof timestr, tv.tv_sec, gf_timefmt_FT);
                snprintf (timestr + strlen (timestr),
                          sizeof timestr - strlen (timestr),
                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
        }

        snprintf (sign_string, sizeof (sign_string), "\nDUMP-END-TIME: %s",
                  timestr);
        ret = write (gf_dump_fd, sign_string, strlen (sign_string));

out:
        if (gf_dump_fd != -1)
                gf_proc_dump_close ();
        rename (tmp_dump_name, path);
        GF_FREE (dump_options.dump_path);
        dump_options.dump_path = NULL;
        gf_proc_dump_unlock ();

        return;
}


void
gf_proc_dump_fini (void)
{
        pthread_mutex_destroy (&gf_proc_dump_mutex);
}


void
gf_proc_dump_init ()
{
        pthread_mutex_init (&gf_proc_dump_mutex, NULL);

        return;
}


void
gf_proc_dump_cleanup (void)
{
        pthread_mutex_destroy (&gf_proc_dump_mutex);
}