inodelk-count: Add stats to count the number of lock objects

Summary:
We want to track the number of locks held by the locks xlator. One of the ways to do it would be to track the
total number of pl_lock objects in the system.

This patch tracks the total number of pl_lock object and exposes the stat via io-stats JSON dump.

Test Plan: WIP, haven't got a pass. Putting the diff to get a sense of this approach would yield what you guys are looking for?

Reviewers: kvigor, sshreyas, jdarcy

Reviewed By: jdarcy

Differential Revision: https://phabricator.intern.facebook.com/D5303071

Change-Id: I946debcbff61699ec28b4d6f243042440107a224
Signed-off-by: Jeff Darcy <jdarcy@fb.com>
Reviewed-on: https://review.gluster.org/18273
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>
Tested-by: Jeff Darcy <jeff@pl.atyp.us>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>

6 files changed, 194 insertions, 1 deletions

diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 408908cbd9b..f1970a41c3b 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -302,6 +302,7 @@ struct ios_conf {
         int32_t                   ios_dnscache_ttl_sec;
         gf_boolean_t              iamshd;
         gf_boolean_t              iamnfsd;
+        gf_boolean_t              iambrickd;
         gf_boolean_t              iamgfproxyd;
         gf_boolean_t              audit_creates_and_unlinks;
         gf_boolean_t              sample_hard_errors;
@@ -1022,6 +1023,9 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,
 {
         int                   i = 0;
         int                   j = 0;
+        int32_t               inodelk_count = -1;
+        int32_t               entrylk_count = -1;
+        dict_t                *xattr = NULL;
         struct ios_conf       *conf = NULL;
         char                  *key_prefix = NULL;
         char                  *str_prefix = NULL;
@@ -1064,6 +1068,46 @@ io_stats_dump_global_to_json_logfp (xlator_t *this,
         ios_log (this, logfp, "\"%s.%s.outstanding_req\": \"%u\",",
                  key_prefix, str_prefix, conf->outstanding_req);
 
+        if (conf->iambrickd) {
+                ret =  syncop_getxattr (FIRST_CHILD (this), &unused_loc,
+                                        &xattr, GLUSTERFS_INODELK_COUNT,
+                                        NULL, NULL);
+                if (ret == 0 && xattr) {
+                        if (dict_get_int32 (xattr, GLUSTERFS_INODELK_COUNT,
+                                            &inodelk_count) != 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "Error fetching %s",
+                                        GLUSTERFS_INODELK_COUNT);
+                        }
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Unable to get inode lock count counts "
+                                "from the posix-locks translator!");
+                }
+
+                ios_log (this, logfp, "\"%s.%s.inodelk_count\": \"%u\",",
+                         key_prefix, str_prefix, inodelk_count);
+
+                ret = syncop_getxattr (FIRST_CHILD (this), &unused_loc, &xattr,
+                                       GLUSTERFS_ENTRYLK_COUNT, NULL, NULL);
+                if (ret == 0 && xattr) {
+                        if (dict_get_int32 (xattr, GLUSTERFS_ENTRYLK_COUNT,
+                                            &entrylk_count) != 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "Error fetching %s",
+                                        GLUSTERFS_ENTRYLK_COUNT);
+                        }
+                        dict_unref (xattr);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Unable to get entry lock count counts "
+                                "from the posix-locks translator!");
+                }
+
+                ios_log (this, logfp, "\"%s.%s.entrylk_count\": \"%u\",",
+                         key_prefix, str_prefix, entrylk_count);
+        }
+
         for (i = 0; i < 31; i++) {
                 rw_size = (1 << i);
                 if (rw_size >= 1024 * 1024) {
@@ -4363,6 +4407,8 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("iam-nfs-daemon", conf->iamnfsd, bool, out);
 
+        GF_OPTION_INIT ("iam-brick-daemon", conf->iambrickd, bool, out);
+
         GF_OPTION_INIT ("iam-gfproxy-daemon", conf->iamgfproxyd, bool, out);
 
         GF_OPTION_INIT ("fop-sample-hard-errors", conf->sample_hard_errors,
@@ -4807,6 +4853,13 @@ struct volume_options options[] = {
                           "translator is running as part of an NFS daemon "
                           "or not."
         },
+        { .key = {"iam-brick-daemon"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option differentiates if the io-stats "
+                         "translator is running as part of brick daemon "
+                         "or not."
+        },
         { .key = {"iam-gfproxy-daemon"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index 626541237b3..fa1b33045dc 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -19,11 +19,40 @@
 #include "clear.h"
 #include "common.h"
 
+static inline void
+pl_private_inc_entrylk (posix_locks_private_t *private)
+{
+#ifdef HAVE_ATOMIC_BUILTINS
+        __sync_fetch_and_add (&private->entrylk_count, 1);
+#else
+        pthread_mutex_lock (&private->lock);
+        {
+                private->entrylk_count += 1;
+        }
+        pthread_mutex_unlock (&private->lock);
+#endif
+}
+
+static inline void
+pl_private_dec_entrylk (posix_locks_private_t *private)
+{
+#ifdef HAVE_ATOMIC_BUILTINS
+        __sync_fetch_and_sub (&private->entrylk_count, 1);
+#else
+        pthread_mutex_lock (&private->lock);
+        {
+                private->entrylk_count -= 1;
+        }
+        pthread_mutex_unlock (&private->lock);
+#endif
+}
+
 void
 __pl_entrylk_unref (pl_entry_lock_t *lock)
 {
         lock->ref--;
         if (!lock->ref) {
+                pl_private_dec_entrylk (lock->this->private);
 		GF_FREE ((char *)lock->basename);
                 GF_FREE (lock->connection_id);
                 GF_FREE (lock);
@@ -68,6 +97,8 @@ new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
 	INIT_LIST_HEAD (&newlock->client_list);
 
 	__pl_entrylk_ref (newlock);
+
+        pl_private_inc_entrylk (newlock->this->private);
 out:
         return newlock;
 }
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index 275fb9d20e4..7d9f123e48e 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -19,6 +19,34 @@
 #include "clear.h"
 #include "common.h"
 
+static inline void
+pl_private_inc_inodelk (posix_locks_private_t *private)
+{
+#ifdef HAVE_ATOMIC_BUILTINS
+        __sync_fetch_and_add (&private->inodelk_count, 1);
+#else
+        pthread_mutex_lock (&private->lock);
+        {
+                private->inodelk_count += 1;
+        }
+        pthread_mutex_unlock (&private->lock);
+#endif
+}
+
+static inline void
+pl_private_dec_inodelk (posix_locks_private_t *private)
+{
+#ifdef HAVE_ATOMIC_BUILTINS
+        __sync_fetch_and_sub (&private->inodelk_count, 1);
+#else
+        pthread_mutex_lock (&private->lock);
+        {
+                private->inodelk_count -= 1;
+        }
+        pthread_mutex_unlock (&private->lock);
+#endif
+}
+
 void
 __delete_inode_lock (pl_inode_lock_t *lock)
 {
@@ -36,6 +64,7 @@ __pl_inodelk_unref (pl_inode_lock_t *lock)
 {
         lock->ref--;
         if (!lock->ref) {
+                pl_private_dec_inodelk (lock->this->private);
                 GF_FREE (lock->connection_id);
                 GF_FREE (lock);
         }
@@ -744,6 +773,8 @@ new_inode_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
                 return NULL;
         }
 
+        pl_private_inc_inodelk (this->private);
+
         lock->fl_start = flock->l_start;
         lock->fl_type  = flock->l_type;
 
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index 8eb35da44be..12ab157a3a7 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -187,6 +187,9 @@ struct __pl_metalk {
 typedef struct __pl_metalk pl_meta_lock_t;
 
 typedef struct {
+#ifndef HAVE_ATOMIC_BUILTINS
+        pthread_mutex_t lock;       /* lock for the private structure */
+#endif
         mlk_mode_t      mandatory_mode; /* holds current mandatory locking mode */
         gf_boolean_t    trace;          /* trace lock requests in and out */
         char           *brickname;
@@ -194,6 +197,8 @@ typedef struct {
         uint32_t        revocation_secs;
         gf_boolean_t    revocation_clear_all;
         uint32_t        revocation_max_blocked;
+        uint32_t        inodelk_count;
+        uint32_t        entrylk_count;
 } posix_locks_private_t;
 
 
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index 616be0f7cff..59d05e81bec 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -1005,6 +1005,7 @@ int32_t
 pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
              const char *name, dict_t *xdata)
 {
+        posix_locks_private_t   *priv           = this->private;
         int32_t                 op_errno        = EINVAL;
         int                     op_ret          = -1;
         int32_t                 bcount          = 0;
@@ -1015,10 +1016,49 @@ pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         dict_t                  *dict           = NULL;
         clrlk_args              args            = {0,};
         char                    *brickname      = NULL;
+        int                     ret             = 0;
 
         if (!name)
                 goto usual;
 
+        if (strncmp (name, GLUSTERFS_INODELK_COUNT,
+                           strlen (GLUSTERFS_INODELK_COUNT)) == 0) {
+                dict = dict_new ();
+                if (!dict) {
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+
+                ret = dict_set_int32 (dict, GLUSTERFS_INODELK_COUNT,
+                                      priv->inodelk_count);
+                if (ret) {
+                        op_errno = EIO;
+                        goto out;
+                }
+
+                op_ret = 0;
+                goto out;
+        }
+
+        if (strncmp (name, GLUSTERFS_ENTRYLK_COUNT,
+                           strlen (GLUSTERFS_ENTRYLK_COUNT)) == 0) {
+                dict = dict_new ();
+                if (!dict) {
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+
+                ret = dict_set_int32 (dict, GLUSTERFS_ENTRYLK_COUNT,
+                                      priv->entrylk_count);
+                if (ret) {
+                        op_errno = EIO;
+                        goto out;
+                }
+
+                op_ret = 0;
+                goto out;
+        }
+
         if (strncmp (name, GF_XATTR_CLRLK_CMD, strlen (GF_XATTR_CLRLK_CMD)))
                 goto usual;
 
@@ -3679,6 +3719,10 @@ init (xlator_t *this)
         priv = GF_CALLOC (1, sizeof (*priv),
                           gf_locks_mt_posix_locks_private_t);
 
+#ifndef HAVE_ATOMIC_BUILTINS
+        pthread_mutex_init (&priv->lock, NULL);
+#endif
+
         GF_OPTION_INIT ("mandatory-locking", tmp_str, str, out);
         if (!strcmp (tmp_str, "forced"))
                 priv->mandatory_mode = MLK_FORCED;
@@ -3735,6 +3779,10 @@ fini (xlator_t *this)
         GF_FREE (priv->brickname);
         GF_FREE (priv);
 
+#ifndef HAVE_ATOMIC_BUILTINS
+        pthread_mutex_destroy (&priv->lock);
+#endif
+
         return 0;
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 98a2ab35801..6fcf6892d27 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -2573,6 +2573,25 @@ out:
 }
 
 static int
+volgen_graph_set_iam_brick_daemon (const volgen_graph_t *graph)
+{
+        xlator_t        *trav;
+        int             ret = 0;
+
+        for (trav = first_of ((volgen_graph_t *)graph); trav;
+                                                        trav = trav->next) {
+                if ((strcmp (trav->type, "debug/io-stats") == 0)) {
+                        ret = xlator_set_option (trav, "iam-brick-daemon",
+                                                 "yes");
+                        if (ret) {
+                                break;
+                        }
+                }
+        }
+        return ret;
+}
+
+static int
 server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                       dict_t *set_dict, void *param)
 {
@@ -2622,7 +2641,13 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                                                 (xlator && loglevel) ?  &server_spec_extended_option_handler :
                                                  &server_spec_option_handler);
 
- out:
+        if (ret) {
+                return -1;
+        }
+
+        ret = volgen_graph_set_iam_brick_daemon (graph);
+
+out:
         return ret;
 }