protocol/client: defer cleanup of private until RPC notifications are handled.

This fix is required for glfs_fini to be able to perform fini on client
xlators in a graph. We are deferring freeing of client xlator's private
until all RPC related resources are destroyed. This guarantees that
client xlator would free RPC related resources provided its private
structures are still accessible via its this pointer.

'Weak' property: If there are no epoll threads executing after calling
fini() on a client xlator, then all its RPC related resources are
guaranteed to be freed. We can now free the corresponding 'this'
pointer.

Change-Id: Ie00b14dda096ac128e1c37e0032f07d17fd701ce
BUG: 1093594
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/9680
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 36 insertions, 14 deletions

diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index b919e4c80a5..7fd1858cf50 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -35,6 +35,25 @@ int client_destroy_rpc (xlator_t *this);
 int client_mark_fd_bad (xlator_t *this);
 
 static int
+client_fini_complete (xlator_t *this)
+{
+        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+
+        clnt_conf_t *conf = this->private;
+
+        if (!conf->destroy)
+                return 0;
+
+        this->private = NULL;
+
+        pthread_mutex_destroy (&conf->lock);
+        GF_FREE (conf);
+
+out:
+        return 0;
+}
+
+static int
 client_notify_dispatch_uniq (xlator_t *this, int32_t event, void *data, ...)
 {
         clnt_conf_t     *conf = this->private;
@@ -2298,6 +2317,10 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
 
                 break;
 
+        case RPC_CLNT_DESTROY:
+                ret = client_fini_complete (this);
+                break;
+
         default:
                 gf_log (this->name, GF_LOG_TRACE,
                         "got some other RPC event %d", event);
@@ -2730,23 +2753,19 @@ fini (xlator_t *this)
         clnt_conf_t *conf = NULL;
 
         conf = this->private;
-        this->private = NULL;
-
-        if (conf) {
-                if (conf->rpc) {
-                        /* cleanup the saved-frames before last unref */
-                        rpc_clnt_connection_cleanup (&conf->rpc->conn);
-
-                        rpc_clnt_unref (conf->rpc);
-                }
+        if (!conf)
+                return;
 
-                /* Saved Fds */
-                /* TODO: */
+        conf->destroy = 1;
+        if (conf->rpc) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&conf->rpc->conn);
+                rpc_clnt_unref (conf->rpc);
+        }
 
-                pthread_mutex_destroy (&conf->lock);
+        /* Saved Fds */
+        /* TODO: */
 
-                GF_FREE (conf);
-        }
         return;
 }
 
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 1aea1353727..7157e120dda 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -128,6 +128,9 @@ typedef struct clnt_conf {
 
         int                     event_threads; /* # of event threads
                                                 * configured */
+
+        gf_boolean_t           destroy; /* if enabled implies fini was called
+                                         * on @this xlator instance */
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {