glusterd: Add last successful glusterd lock backtrace

Also, moved the backtrace fetching logic to a separate function.
Modified the backtrace fetching logic able to work under memory pressure
conditions.

Change-Id: Ie38bea425a085770f41831314aeda95595177ece
BUG:1145093
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/8794
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-by: Niels de Vos <ndevos@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

6 files changed, 146 insertions, 23 deletions

diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index be8bd6298ac..2875a304c3e 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -3275,3 +3275,119 @@ gf_set_timestamp  (const char *src, const char* dest)
 out:
         return ret;
 }
+
+static void
+gf_backtrace_end (char *buf, size_t frames)
+{
+        size_t pos = 0;
+
+        if (!buf)
+                return;
+
+        pos = strlen (buf);
+
+        frames = min(frames, GF_BACKTRACE_LEN - pos -1);
+
+        if (frames <= 0)
+                return;
+
+        memset (buf+pos, ')', frames);
+        buf[pos+frames] = '\0';
+}
+
+/*Returns bytes written*/
+static int
+gf_backtrace_append (char *buf, size_t pos, char *framestr)
+{
+        if (pos >= GF_BACKTRACE_LEN)
+                return -1;
+        return snprintf (buf+pos, GF_BACKTRACE_LEN-pos, "(--> %s ", framestr);
+}
+
+static int
+gf_backtrace_fillframes (char *buf)
+{
+        void    *array[GF_BACKTRACE_FRAME_COUNT];
+        size_t  frames                  = 0;
+        FILE    *fp                     = NULL;
+        char    callingfn[GF_BACKTRACE_FRAME_COUNT-2][1024] = {{0},};
+        int     ret                     = -1;
+        int     fd                      = -1;
+        size_t  idx                     = 0;
+        size_t  pos                     = 0;
+        size_t  inc                     = 0;
+        char    tmpl[32]                = "/tmp/btXXXXXX";
+
+        frames = backtrace (array, GF_BACKTRACE_FRAME_COUNT);
+        if (!frames)
+                return -1;
+
+        fd = gf_mkostemp (tmpl, 0, O_RDWR);
+        if (fd == -1)
+                return -1;
+
+        /*The most recent two frames are the calling function and
+         * gf_backtrace_save, which we can infer.*/
+
+        backtrace_symbols_fd (&array[2], frames-2, fd);
+
+        fp = fdopen (fd, "r");
+        if (!fp) {
+                close (fd);
+                ret = -1;
+                goto out;
+        }
+
+        ret = fseek (fp, 0L, SEEK_SET);
+        if (ret)
+                goto out;
+
+        pos = 0;
+        for (idx = 0; idx < frames - 2; idx++) {
+                ret = fscanf (fp, "%s", callingfn[idx]);
+                if (ret == EOF)
+                        break;
+                inc = gf_backtrace_append (buf, pos, callingfn[idx]);
+                if (inc == -1)
+                        break;
+                pos += inc;
+        }
+        gf_backtrace_end (buf, idx);
+
+out:
+        if (fp)
+                fclose (fp);
+
+        unlink (tmpl);
+
+        return (idx > 0)? 0: -1;
+
+}
+
+/* Optionally takes @buf to save backtrace.  If @buf is NULL, uses the
+ * pre-allocated ctx->btbuf to avoid allocating memory while printing
+ * backtrace.
+ * TODO: This API doesn't provide flexibility in terms of no. of frames
+ * of the backtrace is being saved in the buffer. Deferring fixing it
+ * when there is a real-use for that.*/
+
+char *
+gf_backtrace_save (char *buf)
+{
+        char *bt = NULL;
+
+        if (!buf) {
+                bt = THIS->ctx->btbuf;
+                GF_ASSERT (bt);
+
+        } else {
+                bt = buf;
+
+        }
+
+        if ((0 == gf_backtrace_fillframes (bt)))
+                return bt;
+
+        gf_log (THIS->name, GF_LOG_WARNING, "Failed to save the backtrace.");
+        return NULL;
+}
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 4eec5170c60..a669e741e9a 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -644,4 +644,10 @@ gf_check_logger (const char *value);
 gf_boolean_t
 gf_compare_sockaddr (const struct sockaddr *addr1,
                      const struct sockaddr *addr2);
+
+char *
+gf_backtrace_save (char *buf);
+
+void
+gf_backtrace_done (char *buf);
 #endif /* _COMMON_UTILS_H */
diff --git a/libglusterfs/src/ctx.c b/libglusterfs/src/ctx.c
index f273451a74b..bc704aceacb 100644
--- a/libglusterfs/src/ctx.c
+++ b/libglusterfs/src/ctx.c
@@ -44,6 +44,7 @@ glusterfs_ctx_new ()
 		free (ctx);
 		ctx = NULL;
 	}
+
 out:
 	return ctx;
 }
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 3f76f2d7d9c..5c22166d1b8 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -209,6 +209,9 @@
 #define GF_LOG_FLUSH_TIMEOUT_MIN_STR "30"
 #define GF_LOG_FLUSH_TIMEOUT_MAX_STR "300"
 
+#define GF_BACKTRACE_LEN        4096
+#define GF_BACKTRACE_FRAME_COUNT 7
+
 
 /* NOTE: add members ONLY at the end (just before _MAXVALUE) */
 typedef enum {
@@ -520,6 +523,8 @@ struct _glusterfs_ctx {
          * NFS.
          */
         mgmt_ssl_t          secure_srvr;
+        /* Buffer to 'save' backtrace even under OOM-kill like situations*/
+        char btbuf[GF_BACKTRACE_LEN];
 
 };
 typedef struct _glusterfs_ctx glusterfs_ctx_t;
diff --git a/libglusterfs/src/logging.c b/libglusterfs/src/logging.c
index e9734bcfca1..e82e0247234 100644
--- a/libglusterfs/src/logging.c
+++ b/libglusterfs/src/logging.c
@@ -774,7 +774,7 @@ _gf_log_callingfn (const char *domain, const char *file, const char *function,
         char           *str2            = NULL;
         char           *msg             = NULL;
         char            timestr[256]    = {0,};
-        char            callstr[4096]   = {0,};
+        char           *callstr         = NULL;
         struct timeval  tv              = {0,};
         size_t          len             = 0;
         int             ret             = 0;
@@ -816,28 +816,9 @@ _gf_log_callingfn (const char *domain, const char *file, const char *function,
         else
                 basename = file;
 
-        do {
-                void *array[5];
-                char **callingfn = NULL;
-                size_t size = 0;
-
-                size = backtrace (array, 5);
-                if (size)
-                        callingfn = backtrace_symbols (&array[2], size-2);
-                if (!callingfn)
-                        break;
-
-                if (size == 5)
-                        snprintf (callstr, 4096, "(-->%s (-->%s (-->%s)))",
-                                  callingfn[2], callingfn[1], callingfn[0]);
-                if (size == 4)
-                        snprintf (callstr, 4096, "(-->%s (-->%s))",
-                                  callingfn[1], callingfn[0]);
-                if (size == 3)
-                        snprintf (callstr, 4096, "(-->%s)", callingfn[0]);
-
-                free (callingfn);
-        } while (0);
+        /*Saving the backtrace to pre-allocated ctx->btbuf
+         * to avoid allocating memory from the heap*/
+        callstr = gf_backtrace_save (NULL);
 
         if (ctx->log.log_control_file_found)
         {
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c
index 28358aa555e..53c97bea533 100644
--- a/xlators/mgmt/glusterd/src/glusterd-locks.c
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.c
@@ -491,6 +491,7 @@ out:
         return ret;
 }
 
+
 int32_t
 glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, char *type)
 {
@@ -501,6 +502,7 @@ glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, char *type)
         gf_boolean_t                    is_valid        = _gf_true;
         uuid_t                          owner           = {0};
         xlator_t                       *this            = NULL;
+        char                           *bt              = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -568,6 +570,18 @@ glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, char *type)
                 goto out;
         }
 
+        /* Saving the backtrace into the pre-allocated buffer, ctx->btbuf*/
+        if ((bt = gf_backtrace_save (NULL))) {
+                snprintf (key, sizeof (key), "debug.last-success-bt-%s-%s",
+                          name, type);
+                ret = dict_set_dynstr_with_alloc (priv->mgmt_v3_lock, key, bt);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING, "Failed to save "
+                                "the back trace for lock %s-%s granted to %s",
+                                name, type, uuid_utoa (uuid));
+                ret = 0;
+        }
+
         gf_log (this->name, GF_LOG_DEBUG,
                 "Lock for %s %s successfully held by %s",
                 type, name, uuid_utoa (uuid));