rpc-transport/socket: Add SSL support.

Based on OpenSSL.  Key/certificate management is still manual.  Enabling
SSL also enables multi-threading, though multi-threading can be forced on
or off using a separate option.

Change-Id: Icd9f256bb2fd8c6266a7abefdff16936b4f8922d
BUG: 764731
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: http://review.gluster.com/362
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

6 files changed, 688 insertions, 85 deletions

diff --git a/glusterfs.spec.in b/glusterfs.spec.in
index a969af8da..df88568ef 100644
--- a/glusterfs.spec.in
+++ b/glusterfs.spec.in
@@ -42,6 +42,7 @@ Packager: @PACKAGE_BUGREPORT@
 URL: http://www.gluster.org/docs/index.php/GlusterFS
 Source0: @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz
 BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
+Requires: openssl >= 1.0
 Requires(post): /sbin/chkconfig
 Requires(preun): /sbin/service, /sbin/chkconfig
 Requires(postun): /sbin/service
@@ -54,7 +55,7 @@ BuildRequires: python-ctypes
 %if 0%{?suse_version}
 BuildRequires: python-devel
 %endif
-Requires: openssl
+BuildRequires: openssl-devel >= 1.0
 
 Obsoletes:        %{name}-libs <= 2.0.0
 Obsoletes:        %{name}-common < %{version}-%{release}
diff --git a/rpc/rpc-transport/socket/src/Makefile.am b/rpc/rpc-transport/socket/src/Makefile.am
index 2c918c7e3..3d1631a34 100644
--- a/rpc/rpc-transport/socket/src/Makefile.am
+++ b/rpc/rpc-transport/socket/src/Makefile.am
@@ -3,7 +3,7 @@ noinst_HEADERS = socket.h name.h
 rpctransport_LTLIBRARIES = socket.la
 rpctransportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/rpc-transport
 
-socket_la_LDFLAGS = -module -avoidversion
+socket_la_LDFLAGS = -module -avoidversion -lssl
 
 socket_la_SOURCES = socket.c name.c
 socket_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index 9db09d77e..088ceb460 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -35,9 +35,33 @@
 #include <errno.h>
 #include <netinet/tcp.h>
 #include <rpc/xdr.h>
+#include <sys/ioctl.h>
 #define GF_LOG_ERRNO(errno) ((errno == ENOTCONN) ? GF_LOG_DEBUG : GF_LOG_ERROR)
 #define SA(ptr) ((struct sockaddr *)ptr)
 
+#define SSL_ENABLED_OPT     "transport.socket.ssl-enabled"
+#define SSL_OWN_CERT_OPT    "transport.socket.ssl-own-cert"
+#define SSL_PRIVATE_KEY_OPT "transport.socket.ssl-private-key"
+#define SSL_CA_LIST_OPT     "transport.socket.ssl-ca-list"
+#define OWN_THREAD_OPT      "transport.socket.own-thread"
+
+/* TBD: do automake substitutions etc. (ick) to set these. */
+#if !defined(DEFAULT_CERT_PATH)
+#define DEFAULT_CERT_PATH   "/etc/ssl/glusterfs.pem"
+#endif
+#if !defined(DEFAULT_KEY_PATH)
+#define DEFAULT_KEY_PATH    "/etc/ssl/glusterfs.key"
+#endif
+#if !defined(DEFAULT_CA_PATH)
+#define DEFAULT_CA_PATH     "/etc/ssl/glusterfs.ca"
+#endif
+
+#define POLL_MASK_INPUT  (POLLIN | POLLPRI)
+#define POLL_MASK_OUTPUT (POLLOUT)
+#define POLL_MASK_ERROR  (POLLERR | POLLHUP | POLLNVAL)
+
+typedef int SSL_unary_func (SSL *);
+typedef int SSL_trinary_func (SSL *, void *, int);
 
 #define __socket_proto_reset_pending(priv) do {                 \
                 memset (&priv->incoming.frag.vector, 0,         \
@@ -124,9 +148,156 @@
                 __socket_proto_update_priv_after_read (priv, ret, bytes_read); \
         }
 
-
 int socket_init (rpc_transport_t *this);
 
+void
+ssl_dump_error_stack (const char *caller)
+{
+	unsigned long  errnum = 0;
+	char           errbuf[120] = {0,};
+
+	/* OpenSSL docs explicitly give 120 as the error-string length. */
+
+	while ((errnum = ERR_get_error())) {
+		ERR_error_string(errnum,errbuf);
+		gf_log(caller,GF_LOG_ERROR,"  %s",errbuf);
+	}
+}
+
+int
+ssl_do (rpc_transport_t *this, void *buf, size_t len, SSL_trinary_func *func)
+{
+	int               r = (-1);
+	struct pollfd     pfd = {-1,};
+	socket_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO(this->name,this->private,out);
+	priv = this->private;
+
+	for (;;) {
+		if (buf) {
+                        if (priv->connected == -1) {
+                                /*
+                                 * Fields in the SSL structure (especially
+                                 * the BIO pointers) are not valid at this
+                                 * point, so we'll segfault if we pass them
+                                 * to SSL_read/SSL_write.
+                                 */
+                                gf_log(this->name,GF_LOG_INFO,
+                                       "lost connection in %s", __func__);
+                                break;
+                        }
+			r = func(priv->ssl_ssl,buf,len);
+		}
+		else {
+                        /*
+                         * We actually need these functions to get to
+                         * priv->connected == 1.
+                         */
+			r = ((SSL_unary_func *)func)(priv->ssl_ssl);
+		}
+		switch (SSL_get_error(priv->ssl_ssl,r)) {
+		case SSL_ERROR_NONE:
+			return r;
+		case SSL_ERROR_WANT_READ:
+			pfd.fd = priv->sock;
+			pfd.events = POLLIN;
+			if (poll(&pfd,1,-1) < 0) {
+				gf_log(this->name,GF_LOG_ERROR,"poll error %d",
+				       errno);
+			}
+			break;
+		case SSL_ERROR_WANT_WRITE:
+			pfd.fd = priv->sock;
+			pfd.events = POLLOUT;
+			if (poll(&pfd,1,-1) < 0) {
+				gf_log(this->name,GF_LOG_ERROR,"poll error %d",
+				       errno);
+			}
+			break;
+		case SSL_ERROR_SYSCALL:
+			/* This is what we get when remote disconnects. */
+			gf_log(this->name,GF_LOG_DEBUG,
+			       "syscall error (probably remote disconnect)");
+			errno = ENODATA;
+			goto out;
+		default:
+			errno = EIO;
+			goto out;	/* "break" would just loop again */
+		}
+	}
+out:
+	return -1;
+}
+
+#define ssl_connect_one(t)   ssl_do((t),NULL,0,(SSL_trinary_func *)SSL_connect)
+#define ssl_accept_one(t)    ssl_do((t),NULL,0,(SSL_trinary_func *)SSL_accept)
+#define ssl_read_one(t,b,l)  ssl_do((t),(b),(l),(SSL_trinary_func *)SSL_read)
+#define ssl_write_one(t,b,l) ssl_do((t),(b),(l),(SSL_trinary_func *)SSL_write)
+
+int
+ssl_setup_connection (rpc_transport_t *this, int server)
+{
+	X509             *peer = NULL;
+	char              peer_CN[256] = "";
+	int               ret = -1;
+	socket_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO(this->name,this->private,done);
+	priv = this->private;
+
+	priv->ssl_ssl = SSL_new(priv->ssl_ctx);
+	if (!priv->ssl_ssl) {
+		gf_log(this->name,GF_LOG_ERROR,"SSL_new failed");
+		ssl_dump_error_stack(this->name);
+		goto done;
+	}
+	priv->ssl_sbio = BIO_new_socket(priv->sock,BIO_NOCLOSE);
+	if (!priv->ssl_sbio) {
+		gf_log(this->name,GF_LOG_ERROR,"BIO_new_socket failed");
+		ssl_dump_error_stack(this->name);
+		goto free_ssl;
+	}
+	SSL_set_bio(priv->ssl_ssl,priv->ssl_sbio,priv->ssl_sbio);
+
+	if (server) {
+		ret = ssl_accept_one(this);
+	}
+	else {
+		ret = ssl_connect_one(this);
+	}
+
+	/* Make sure _the call_ succeeded. */
+	if (ret < 0) {
+		goto ssl_error;
+	}
+
+	/* Make sure _SSL verification_ succeeded, yielding an identity. */
+	if (SSL_get_verify_result(priv->ssl_ssl) != X509_V_OK) {
+		goto ssl_error;
+	}
+	peer = SSL_get_peer_certificate(priv->ssl_ssl);
+	if (!peer) {
+		goto ssl_error;
+	}
+
+	/* Finally, everything seems OK. */
+	X509_NAME_get_text_by_NID(X509_get_subject_name(peer),
+		NID_commonName, peer_CN, sizeof(peer_CN)-1);
+	peer_CN[sizeof(peer_CN)-1] = '\0';
+	gf_log(this->name,GF_LOG_INFO,"peer CN = %s", peer_CN);
+	return 0;
+
+	/* Error paths. */
+ssl_error:
+	gf_log(this->name,GF_LOG_ERROR,"SSL connect error");
+	ssl_dump_error_stack(this->name);
+free_ssl:
+	SSL_free(priv->ssl_ssl);
+done:
+	return ret;
+}
+
 /*
  * return value:
  *   0 = success (completed)
@@ -159,9 +330,22 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                 *bytes = 0;
         }
 
-        while (opcount) {
+        while (opcount > 0) {
+                if (opvector->iov_len == 0) {
+                        gf_log(this->name,GF_LOG_DEBUG,
+                               "would have passed zero length to read/write");
+                        ++opvector;
+                        --opcount;
+                        continue;
+                }
                 if (write) {
-                        ret = writev (sock, opvector, opcount);
+			if (priv->use_ssl) {
+				ret = ssl_write_one(this,
+					opvector->iov_base, opvector->iov_len);
+			}
+			else {
+				ret = writev (sock, opvector, opcount);
+			}
 
                         if (ret == 0 || (ret == -1 && errno == EAGAIN)) {
                                 /* done for now */
@@ -169,7 +353,18 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                         }
                         this->total_bytes_write += ret;
                 } else {
-                        ret = readv (sock, opvector, opcount);
+			if (priv->use_ssl) {
+				ret = ssl_read_one(this,
+					opvector->iov_base, opvector->iov_len);
+			}
+			else {
+				ret = readv (sock, opvector, opcount);
+			}
+			if (ret == 0) {
+				gf_log(this->name,GF_LOG_DEBUG,"EOF on socket");
+				errno = ENODATA;
+				ret = -1;
+			}
                         if (ret == -1 && errno == EAGAIN) {
                                 /* done for now */
                                 break;
@@ -193,6 +388,9 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s failed (%s)", write ? "writev" : "readv",
                                 strerror (errno));
+			if (priv->use_ssl) {
+				ssl_dump_error_stack(this->name);
+			}
                         opcount = -1;
                         break;
                 }
@@ -204,6 +402,17 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                 moved = 0;
 
                 while (moved < ret) {
+                        if (!opcount) {
+                                gf_log(this->name,GF_LOG_DEBUG,
+                                       "ran out of iov, moved %d/%d",
+                                       moved, ret);
+                                goto ran_out;
+                        }
+                        if (!opvector[0].iov_len) {
+                                opvector++;
+                                opcount--;
+                                continue;
+                        }
                         if ((ret - moved) >= opvector[0].iov_len) {
                                 moved += opvector[0].iov_len;
                                 opvector++;
@@ -213,13 +422,11 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                                 opvector[0].iov_base += (ret - moved);
                                 moved += (ret - moved);
                         }
-                        while (opcount && !opvector[0].iov_len) {
-                                opvector++;
-                                opcount--;
-                        }
                 }
         }
 
+ran_out:
+
         if (pending_vector)
                 *pending_vector = opvector;
 
@@ -279,6 +486,20 @@ __socket_disconnect (rpc_transport_t *this)
                                 "shutdown() returned %d. %s",
                                 ret, strerror (errno));
                 }
+		if (priv->use_ssl) {
+			SSL_shutdown(priv->ssl_ssl);
+			SSL_clear(priv->ssl_ssl);
+			SSL_free(priv->ssl_ssl);
+		}
+		if (priv->own_thread) {
+			/*
+			 * Without this, reconnect (= disconnect + connect)
+			 * won't work except by accident.
+			 */
+			close(priv->sock);
+			priv->sock = -1;
+			++(priv->socket_gen);
+		}
         }
 
 out:
@@ -356,7 +577,6 @@ __socket_nonblock (int fd)
         return ret;
 }
 
-
 int
 __socket_nodelay (int fd)
 {
@@ -610,9 +830,11 @@ out:
 
 
 int
-__socket_ioq_churn_entry (rpc_transport_t *this, struct ioq *entry)
+__socket_ioq_churn_entry (rpc_transport_t *this, struct ioq *entry, int direct)
 {
-        int ret = -1;
+        int               ret = -1;
+	socket_private_t *priv = NULL;
+	char              a_byte = 0;
 
         ret = __socket_writev (this, entry->pending_vector,
                                entry->pending_count,
@@ -623,6 +845,18 @@ __socket_ioq_churn_entry (rpc_transport_t *this, struct ioq *entry)
                 /* current entry was completely written */
                 GF_ASSERT (entry->pending_count == 0);
                 __socket_ioq_entry_free (entry);
+		priv = this->private;
+		if (priv->own_thread) {
+			/*
+			 * The pipe should only remain readable if there are
+			 * more entries after this, so drain the byte
+			 * representing this entry.
+			 */
+			if (!direct && read(priv->pipe[0],&a_byte,1) < 1) {
+				gf_log(this->name,GF_LOG_WARNING,
+				       "read error on pipe");
+			}
+		}
         }
 
         return ret;
@@ -645,13 +879,13 @@ __socket_ioq_churn (rpc_transport_t *this)
                 /* pick next entry */
                 entry = priv->ioq_next;
 
-                ret = __socket_ioq_churn_entry (this, entry);
+                ret = __socket_ioq_churn_entry (this, entry, 0);
 
                 if (ret != 0)
                         break;
         }
 
-        if (list_empty (&priv->ioq)) {
+        if (!priv->own_thread && list_empty (&priv->ioq)) {
                 /* all pending writes done, not interested in POLLOUT */
                 priv->idx = event_select_on (this->ctx->event_pool,
                                              priv->sock, priv->idx, -1, 0);
@@ -1729,7 +1963,6 @@ socket_event_poll_in (rpc_transport_t *this)
         if (pollin != NULL) {
                 ret = rpc_transport_notify (this, RPC_TRANSPORT_MSG_RECEIVED,
                                             pollin);
-
                 rpc_transport_pollin_destroy (pollin);
         }
 
@@ -1752,7 +1985,7 @@ socket_connect_finish (rpc_transport_t *this)
 
         pthread_mutex_lock (&priv->lock);
         {
-                if (priv->connected)
+                if (priv->connected != 0)
                         goto unlock;
 
                 ret = __socket_connect_finish (priv->sock);
@@ -1814,9 +2047,9 @@ int
 socket_event_handler (int fd, int idx, void *data,
                       int poll_in, int poll_out, int poll_err)
 {
-        rpc_transport_t      *this = NULL;
+        rpc_transport_t  *this = NULL;
         socket_private_t *priv = NULL;
-        int               ret = 0;
+	int               ret = -1;
 
         this = data;
         GF_VALIDATE_OR_GOTO ("socket", this, out);
@@ -1826,16 +2059,13 @@ socket_event_handler (int fd, int idx, void *data,
         THIS = this->xl;
         priv = this->private;
 
-
         pthread_mutex_lock (&priv->lock);
         {
                 priv->idx = idx;
         }
         pthread_mutex_unlock (&priv->lock);
 
-        if (!priv->connected) {
-                ret = socket_connect_finish (this);
-        }
+	ret = (priv->connected == 1) ? 0 : socket_connect_finish(this);
 
         if (!ret && poll_out) {
                 ret = socket_event_poll_out (this);
@@ -1851,13 +2081,112 @@ socket_event_handler (int fd, int idx, void *data,
                         "disconnecting now");
                 socket_event_poll_err (this);
                 rpc_transport_unref (this);
-        }
+	}
 
 out:
-        return 0;
+	return ret;
 }
 
 
+void *
+socket_poller (void *ctx)
+{
+        rpc_transport_t  *this = ctx;
+        socket_private_t *priv = this->private;
+	struct pollfd     pfd[2] = {{0,},};
+	gf_boolean_t      to_write = _gf_false;
+	int               ret = 0;
+	int               orig_gen;
+
+	orig_gen = ++(priv->socket_gen);
+
+        if (priv->connected == 0) {
+		THIS = this->xl;
+                ret = socket_connect_finish (this);
+        }
+
+	for (;;) {
+		if (priv->socket_gen != orig_gen) {
+			gf_log(this->name,GF_LOG_DEBUG,
+			       "redundant poller exiting");
+			return NULL;
+		}
+		pthread_mutex_lock(&priv->lock);
+		to_write = !list_empty(&priv->ioq);
+		pthread_mutex_unlock(&priv->lock);
+		pfd[0].fd = priv->pipe[0];
+		pfd[0].events = POLL_MASK_ERROR;
+		pfd[0].revents = 0;
+		pfd[1].fd = priv->sock;
+		pfd[1].events = POLL_MASK_INPUT | POLL_MASK_ERROR;
+		pfd[1].revents = 0;
+		if (to_write) {
+			pfd[1].events |= POLL_MASK_OUTPUT;
+		}
+		else {
+			pfd[0].events |= POLL_MASK_INPUT;
+		}
+		if (poll(pfd,2,-1) < 0) {
+			gf_log(this->name,GF_LOG_ERROR,"poll failed");
+			break;
+		}
+		if (pfd[0].revents & POLL_MASK_ERROR) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "poll error on pipe");
+			break;
+		}
+		/* Only glusterd actually seems to need this. */
+		THIS = this->xl;
+		if (pfd[1].revents & POLL_MASK_INPUT) {
+			ret = socket_event_poll_in(this);
+			if (ret >= 0) {
+				/* Suppress errors while making progress. */
+				pfd[1].revents &= ~POLL_MASK_ERROR;
+			}
+			else if (errno == ENOTCONN) {
+				ret = 0;
+			}
+		}
+		else if (pfd[1].revents & POLL_MASK_OUTPUT) {
+			ret = socket_event_poll_out(this);
+			if (ret >= 0) {
+				/* Suppress errors while making progress. */
+				pfd[1].revents &= ~POLL_MASK_ERROR;
+			}
+			else if (errno == ENOTCONN) {
+				ret = 0;
+			}
+		}
+		else {
+			/*
+			 * This usually means that we left poll() because
+			 * somebody pushed a byte onto our pipe.  That wakeup
+			 * is why the pipe is there, but once awake we can do
+			 * all the checking we need on the next iteration.
+			 */
+			ret = 0;
+		}
+		if (pfd[1].revents & POLL_MASK_ERROR) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "poll error on socket");
+			break;
+		}
+		if (ret < 0) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "error in polling loop");
+			break;
+		}
+	}
+
+	/* All (and only) I/O errors should come here. */
+	__socket_disconnect (this);
+        rpc_transport_notify (this, RPC_TRANSPORT_DISCONNECT, this);
+	rpc_transport_unref (this);
+	return NULL;
+}
+
+
+
 int
 socket_server_event_handler (int fd, int idx, void *data,
                              int poll_in, int poll_out, int poll_err)
@@ -1896,19 +2225,6 @@ socket_server_event_handler (int fd, int idx, void *data,
                                 goto unlock;
                         }
 
-                        if (!priv->bio) {
-                                ret = __socket_nonblock (new_sock);
-
-                                if (ret == -1) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "NBIO on %d failed (%s)",
-                                                new_sock, strerror (errno));
-
-                                        close (new_sock);
-                                        goto unlock;
-                                }
-                        }
-
                         if (priv->nodelay) {
                                 ret = __socket_nodelay (new_sock);
                                 if (ret == -1) {
@@ -1955,7 +2271,11 @@ socket_server_event_handler (int fd, int idx, void *data,
                         }
 
                         get_transport_identifiers (new_trans);
-                        socket_init (new_trans);
+                        ret = socket_init(new_trans);
+                        if (ret != 0) {
+                                close(new_sock);
+                                goto unlock;
+                        }
                         new_trans->ops = this->ops;
                         new_trans->init = this->init;
                         new_trans->fini = this->fini;
@@ -1966,20 +2286,61 @@ socket_server_event_handler (int fd, int idx, void *data,
                         new_trans->listener = this;
                         new_priv = new_trans->private;
 
+			new_priv->use_ssl = priv->use_ssl;
+			new_priv->sock = new_sock;
+			new_priv->own_thread = priv->own_thread;
+
+			if (priv->use_ssl) {
+				new_priv->ssl_ctx = priv->ssl_ctx;
+				if (ssl_setup_connection(new_trans,1) < 0) {
+					gf_log(this->name,GF_LOG_ERROR,
+					       "server setup failed");
+					close(new_sock);
+					goto unlock;
+				}
+			}
+
+                        if (!priv->bio) {
+                                ret = __socket_nonblock (new_sock);
+
+                                if (ret == -1) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "NBIO on %d failed (%s)",
+                                                new_sock, strerror (errno));
+
+                                        close (new_sock);
+                                        goto unlock;
+                                }
+                        }
+
                         pthread_mutex_lock (&new_priv->lock);
                         {
-                                new_priv->sock = new_sock;
                                 new_priv->connected = 1;
                                 rpc_transport_ref (new_trans);
 
-                                new_priv->idx =
-                                        event_register (ctx->event_pool,
-                                                        new_sock,
-                                                        socket_event_handler,
-                                                        new_trans, 1, 0);
+				if (new_priv->own_thread) {
+					if (pipe(new_priv->pipe) < 0) {
+						gf_log(this->name,GF_LOG_ERROR,
+						       "could not create pipe");
+					}
+					if (pthread_create(&new_priv->thread,
+							NULL, socket_poller,
+							new_trans) != 0) {
+						gf_log(this->name,GF_LOG_ERROR,
+						       "could not create poll thread");
+					}
+				}
+				else {
+					new_priv->idx =
+						event_register (ctx->event_pool,
+								new_sock,
+								socket_event_handler,
+								new_trans,
+								1, 0);
+					if (new_priv->idx == -1)
+						ret = -1;
+				}
 
-                                if (new_priv->idx == -1)
-                                        ret = -1;
                         }
                         pthread_mutex_unlock (&new_priv->lock);
                         if (ret == -1) {
@@ -2069,6 +2430,20 @@ socket_connect (rpc_transport_t *this, int port)
         if (port > 0) {
                 sock_union.sin.sin_port = htons (port);
         }
+        if (ntohs(sock_union.sin.sin_port) == GF_DEFAULT_SOCKET_LISTEN_PORT) {
+                if (priv->use_ssl) {
+                        gf_log(this->name,GF_LOG_DEBUG,
+                               "disabling SSL for portmapper connection");
+                        priv->use_ssl = _gf_false;
+                }
+        }
+        else {
+                if (priv->ssl_enabled && !priv->use_ssl) {
+                        gf_log(this->name,GF_LOG_DEBUG,
+                               "re-enabling SSL for I/O connection");
+                        priv->use_ssl = _gf_true;
+                }
+        }
         pthread_mutex_lock (&priv->lock);
         {
                 if (priv->sock != -1) {
@@ -2124,19 +2499,6 @@ socket_connect (rpc_transport_t *this, int port)
                         }
                 }
 
-                if (!priv->bio) {
-                        ret = __socket_nonblock (priv->sock);
-
-                        if (ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "NBIO on %d failed (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
-                        }
-                }
-
                 if (priv->keepalive) {
                         ret = __socket_keepalive (priv->sock,
                                                   priv->keepaliveintvl,
@@ -2172,17 +2534,55 @@ socket_connect (rpc_transport_t *this, int port)
                         goto unlock;
                 }
 
-                priv->connected = 0;
+		if (priv->use_ssl) {
+			ret = ssl_setup_connection(this,0);
+			if (ret < 0) {
+				gf_log(this->name,GF_LOG_ERROR,
+					"client setup failed");
+				close(priv->sock);
+				priv->sock = -1;
+				goto unlock;
+			}
+		}
 
-                rpc_transport_ref (this);
+                if (!priv->bio) {
+                        ret = __socket_nonblock (priv->sock);
 
-                priv->idx = event_register (ctx->event_pool, priv->sock,
-                                            socket_event_handler, this, 1, 1);
-                if (priv->idx == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to register the event");
-                        ret = -1;
+                        if (ret == -1) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "NBIO on %d failed (%s)",
+                                        priv->sock, strerror (errno));
+                                close (priv->sock);
+                                priv->sock = -1;
+                                goto unlock;
+                        }
                 }
+
+                priv->connected = 0;
+                rpc_transport_ref (this);
+
+		if (priv->own_thread) {
+			if (pipe(priv->pipe) < 0) {
+				gf_log(this->name,GF_LOG_ERROR,
+				       "could not create pipe");
+			}
+
+			if (pthread_create(&priv->thread,NULL,
+					socket_poller, this) != 0) {
+				gf_log(this->name,GF_LOG_ERROR,
+				       "could not create poll thread");
+			}
+		}
+		else {
+			priv->idx = event_register (ctx->event_pool, priv->sock,
+						    socket_event_handler,
+						    this, 1, 1);
+			if (priv->idx == -1) {
+				gf_log ("", GF_LOG_WARNING,
+					"failed to register the event");
+				ret = -1;
+			}
+		}
         }
 unlock:
         pthread_mutex_unlock (&priv->lock);
@@ -2352,6 +2752,7 @@ socket_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
         char              need_append = 1;
         struct ioq       *entry = NULL;
         glusterfs_ctx_t  *ctx = NULL;
+	char              a_byte = 'j';
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@@ -2377,21 +2778,31 @@ socket_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
                         goto unlock;
 
                 if (list_empty (&priv->ioq)) {
-                        ret = __socket_ioq_churn_entry (this, entry);
+                        ret = __socket_ioq_churn_entry (this, entry, 1);
 
-                        if (ret == 0)
+                        if (ret == 0) {
                                 need_append = 0;
-
-                        if (ret > 0)
+			}
+                        if (ret > 0) {
                                 need_poll_out = 1;
+			}
                 }
 
                 if (need_append) {
                         list_add_tail (&entry->list, &priv->ioq);
+			if (priv->own_thread) {
+				/*
+				 * Make sure the polling thread wakes up, by
+				 * writing a byte to represent this entry.
+				 */
+				if (write(priv->pipe[1],&a_byte,1) < 1) {
+					gf_log(this->name,GF_LOG_WARNING,
+					       "write error on pipe");
+				}
+			}
                         ret = 0;
                 }
-
-                if (need_poll_out) {
+                if (!priv->own_thread && need_poll_out) {
                         /* first entry to wait. continue writing on POLLOUT */
                         priv->idx = event_select_on (ctx->event_pool,
                                                      priv->sock,
@@ -2415,6 +2826,7 @@ socket_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
         char              need_append = 1;
         struct ioq       *entry = NULL;
         glusterfs_ctx_t  *ctx = NULL;
+	char              a_byte = 'd';
 
         GF_VALIDATE_OR_GOTO ("socket", this, out);
         GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@@ -2433,33 +2845,44 @@ socket_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
                         }
                         goto unlock;
                 }
+
                 priv->submit_log = 0;
                 entry = __socket_ioq_new (this, &reply->msg);
                 if (!entry)
                         goto unlock;
+
                 if (list_empty (&priv->ioq)) {
-                        ret = __socket_ioq_churn_entry (this, entry);
+                        ret = __socket_ioq_churn_entry (this, entry, 1);
 
-                        if (ret == 0)
+                        if (ret == 0) {
                                 need_append = 0;
-
-                        if (ret > 0)
+			}
+                        if (ret > 0) {
                                 need_poll_out = 1;
+			}
                 }
 
                 if (need_append) {
                         list_add_tail (&entry->list, &priv->ioq);
+			if (priv->own_thread) {
+				/*
+				 * Make sure the polling thread wakes up, by
+				 * writing a byte to represent this entry.
+				 */
+				if (write(priv->pipe[1],&a_byte,1) < 1) {
+					gf_log(this->name,GF_LOG_WARNING,
+					       "write error on pipe");
+				}
+			}
                         ret = 0;
                 }
-
-                if (need_poll_out) {
+                if (!priv->own_thread && need_poll_out) {
                         /* first entry to wait. continue writing on POLLOUT */
                         priv->idx = event_select_on (ctx->event_pool,
                                                      priv->sock,
                                                      priv->idx, -1, 1);
                 }
         }
-
 unlock:
         pthread_mutex_unlock (&priv->lock);
 
@@ -2622,6 +3045,7 @@ socket_init (rpc_transport_t *this)
         char             *optstr = NULL;
         uint32_t          keepalive = 0;
         uint32_t          backlog = 0;
+	int               session_id = 0;
 
         if (this->private) {
                 gf_log_callingfn (this->name, GF_LOG_ERROR,
@@ -2750,11 +3174,130 @@ socket_init (rpc_transport_t *this)
                 }
         }
 
-        optstr = NULL;
+        priv->windowsize = (int)windowsize;
+
+        priv->ssl_enabled = _gf_false;
+	if (dict_get_str(this->options,SSL_ENABLED_OPT,&optstr) == 0) {
+                if (gf_string2boolean (optstr, &priv->ssl_enabled) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+				"invalid value given for ssl-enabled boolean");
+		}
+	}
+
+        priv->ssl_own_cert = DEFAULT_CERT_PATH;
+	if (dict_get_str(this->options,SSL_OWN_CERT_OPT,&optstr) == 0) {
+                if (!priv->ssl_enabled) {
+                        gf_log(this->name,GF_LOG_WARNING,
+                               "%s specified without %s (ignored)",
+                               SSL_OWN_CERT_OPT, SSL_ENABLED_OPT);
+                }
+                priv->ssl_own_cert = optstr;
+	}
+        priv->ssl_own_cert = gf_strdup(priv->ssl_own_cert);
+
+        priv->ssl_private_key = DEFAULT_KEY_PATH;
+	if (dict_get_str(this->options,SSL_PRIVATE_KEY_OPT,&optstr) == 0) {
+                if (!priv->ssl_enabled) {
+                        gf_log(this->name,GF_LOG_WARNING,
+                               "%s specified without %s (ignored)",
+                               SSL_PRIVATE_KEY_OPT, SSL_ENABLED_OPT);
+                }
+                priv->ssl_private_key = optstr;
+	}
+        priv->ssl_private_key = gf_strdup(priv->ssl_private_key);
+
+        priv->ssl_ca_list = DEFAULT_CA_PATH;
+	if (dict_get_str(this->options,SSL_CA_LIST_OPT,&optstr) == 0) {
+                if (!priv->ssl_enabled) {
+                        gf_log(this->name,GF_LOG_WARNING,
+                               "%s specified without %s (ignored)",
+                               SSL_CA_LIST_OPT, SSL_ENABLED_OPT);
+                }
+                priv->ssl_ca_list = optstr;
+	}
+        priv->ssl_ca_list = gf_strdup(priv->ssl_ca_list);
+
+        gf_log(this->name,GF_LOG_INFO,"SSL support is %s",
+               priv->ssl_enabled ? "ENABLED" : "NOT enabled");
+        /*
+         * This might get overridden temporarily in socket_connect (q.v.)
+         * if we're using the glusterd portmapper.
+         */
+        priv->use_ssl = priv->ssl_enabled;
+
+	priv->own_thread = priv->use_ssl;
+	if (dict_get_str(this->options,OWN_THREAD_OPT,&optstr) == 0) {
+                if (gf_string2boolean (optstr, &priv->own_thread) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+				"invalid value given for own-thread boolean");
+		}
+	}
+	gf_log(this->name,GF_LOG_INFO,"using %s polling thread",
+	       priv->own_thread ? "private" : "system");
+
+	if (priv->use_ssl) {
+		SSL_library_init();
+		SSL_load_error_strings();
+		priv->ssl_meth = (SSL_METHOD *)TLSv1_method();
+		priv->ssl_ctx = SSL_CTX_new(priv->ssl_meth);
+
+                if (SSL_CTX_set_cipher_list(priv->ssl_ctx,
+                                            "HIGH:-SSLv2") == 0) {
+                        gf_log(this->name,GF_LOG_ERROR,
+                               "failed to find any valid ciphers");
+                        goto err;
+                }
+
+		if (!SSL_CTX_use_certificate_chain_file(priv->ssl_ctx,
+							priv->ssl_own_cert)) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "could not load our cert");
+			goto err;
+		}
+
+		if (!SSL_CTX_use_PrivateKey_file(priv->ssl_ctx,
+						 priv->ssl_private_key,
+						 SSL_FILETYPE_PEM)) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "could not load private key");
+			goto err;
+		}
+
+		if (!SSL_CTX_load_verify_locations(priv->ssl_ctx,
+						   priv->ssl_ca_list,0)) {
+			gf_log(this->name,GF_LOG_ERROR,
+			       "could not load CA list");
+			goto err;
+		}
+
+#if (OPENSSL_VERSION_NUMBER < 0x00905100L)
+		SSL_CTX_set_verify_depth(ctx,1);
+#endif
+
+		priv->ssl_session_id = ++session_id;
+		SSL_CTX_set_session_id_context(priv->ssl_ctx,
+					       (void *)&priv->ssl_session_id,
+					       sizeof(priv->ssl_session_id));
+
+		SSL_CTX_set_verify(priv->ssl_ctx,SSL_VERIFY_PEER,0);
+	}
+
 out:
         this->private = priv;
-
         return 0;
+
+err:
+        if (priv->ssl_own_cert) {
+                GF_FREE(priv->ssl_own_cert);
+        }
+        if (priv->ssl_private_key) {
+                GF_FREE(priv->ssl_private_key);
+        }
+        if (priv->ssl_ca_list) {
+                GF_FREE(priv->ssl_ca_list);
+        }
+        GF_FREE(priv);
+        return -1;
 }
 
 
@@ -2780,6 +3323,15 @@ fini (rpc_transport_t *this)
                         "transport %p destroyed", this);
 
                 pthread_mutex_destroy (&priv->lock);
+		if (priv->ssl_private_key) {
+			GF_FREE(priv->ssl_private_key);
+		}
+		if (priv->ssl_own_cert) {
+			GF_FREE(priv->ssl_own_cert);
+		}
+		if (priv->ssl_ca_list) {
+			GF_FREE(priv->ssl_ca_list);
+		}
                 GF_FREE (priv);
         }
 
@@ -2857,5 +3409,20 @@ struct volume_options options[] = {
         { .key   = {"transport.socket.read-fail-log"},
           .type  = GF_OPTION_TYPE_BOOL
         },
+        { .key   = {SSL_ENABLED_OPT},
+          .type  = GF_OPTION_TYPE_BOOL
+        },
+	{ .key   = {SSL_OWN_CERT_OPT},
+	  .type  = GF_OPTION_TYPE_STR
+	},
+	{ .key   = {SSL_PRIVATE_KEY_OPT},
+	  .type  = GF_OPTION_TYPE_STR
+	},
+	{ .key   = {SSL_CA_LIST_OPT},
+	  .type  = GF_OPTION_TYPE_STR
+	},
+	{ .key   = {OWN_THREAD_OPT},
+	  .type  = GF_OPTION_TYPE_BOOL
+	},
         { .key = {NULL} }
 };
diff --git a/rpc/rpc-transport/socket/src/socket.h b/rpc/rpc-transport/socket/src/socket.h
index 0304f1db1..0a407cc1a 100644
--- a/rpc/rpc-transport/socket/src/socket.h
+++ b/rpc/rpc-transport/socket/src/socket.h
@@ -11,6 +11,8 @@
 #ifndef _SOCKET_H
 #define _SOCKET_H
 
+#include <openssl/ssl.h>
+#include <openssl/err.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -144,7 +146,8 @@ typedef struct {
 typedef struct {
         int32_t                sock;
         int32_t                idx;
-        unsigned char          connected; // -1 = not connected. 0 = in progress. 1 = connected
+        /* -1 = not connected. 0 = in progress. 1 = connected */
+        char                   connected;
         char                   bio;
         char                   connect_finish_log;
         char                   submit_log;
@@ -195,6 +198,20 @@ typedef struct {
         int                    keepaliveintvl;
         uint32_t               backlog;
         gf_boolean_t           read_fail_log;
+        gf_boolean_t           ssl_enabled;
+	gf_boolean_t           use_ssl;
+	SSL_METHOD            *ssl_meth;
+	SSL_CTX               *ssl_ctx;
+	int                    ssl_session_id;
+	BIO                   *ssl_sbio;
+	SSL                   *ssl_ssl;
+	char                  *ssl_own_cert;
+	char                  *ssl_private_key;
+	char                  *ssl_ca_list;
+	pthread_t              thread;
+	int                    pipe[2];
+	gf_boolean_t           own_thread;
+	volatile int           socket_gen;
 } socket_private_t;
 
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index ff35b8b08..ae86eb18c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -175,6 +175,7 @@ static struct volopt_map_entry glusterd_volopt_map[] = {
         {"network.frame-timeout",                "protocol/client",           NULL, NULL, NO_DOC, 0},
         {"network.ping-timeout",                 "protocol/client",           NULL, NULL, NO_DOC, 0},
         {"network.tcp-window-size",              "protocol/client",           NULL, NULL, NO_DOC, 0},
+        { "client.ssl",                          "protocol/client",           "transport.socket.ssl-enabled", NULL, NO_DOC, 0},
 
         {"network.tcp-window-size",              "protocol/server",           NULL, NULL, NO_DOC, 0},
         {"network.inode-lru-limit",              "protocol/server",           NULL, NULL, NO_DOC, 0},
@@ -182,6 +183,7 @@ static struct volopt_map_entry glusterd_volopt_map[] = {
         {AUTH_REJECT_MAP_KEY,                    "protocol/server",           "!server-auth", NULL, DOC, 0},
         {"transport.keepalive",                  "protocol/server",           "transport.socket.keepalive", NULL, NO_DOC, 0},
         {"server.allow-insecure",                "protocol/server",           "rpc-auth-allow-insecure", NULL, NO_DOC, 0},
+        { "server.ssl",                          "protocol/server",           "transport.socket.ssl-enabled", NULL, NO_DOC, 0},
 
         {"performance.write-behind",             "performance/write-behind",  "!perf", "on", NO_DOC, 0},
         {"performance.read-ahead",               "performance/read-ahead",    "!perf", "on", NO_DOC, 0},
@@ -2157,6 +2159,8 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char                    *str                = NULL;
         glusterd_brickinfo_t    *brick              = NULL;
         xlator_t                *xl                 = NULL;
+        char                    *ssl_str            = NULL;
+        gf_boolean_t             ssl_bool;
 
         volname = volinfo->volname;
 
@@ -2222,6 +2226,19 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                         }
                 }
 
+                if (dict_get_str(set_dict,"client.ssl",&ssl_str) == 0) {
+                        if (gf_string2boolean(ssl_str,&ssl_bool) == 0) {
+                                if (ssl_bool) {
+                                        ret = xlator_set_option(xl,
+                                                "transport.socket.ssl-enabled",
+                                                "true");
+                                        if (ret) {
+                                                goto out;
+                                        }
+                                }
+                        }
+                }
+
                 i++;
         }
 
diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c
index e7e5fce53..da9028473 100644
--- a/xlators/protocol/server/src/server-rpc-fops.c
+++ b/xlators/protocol/server/src/server-rpc-fops.c
@@ -3462,7 +3462,8 @@ server3_3_release (rpcsvc_request_t *req)
 
         conn = req->trans->xl_private;
         if (!conn) {
-                req->rpc_err = GARBAGE_ARGS;
+                /* Handshake is not complete yet. */
+                req->rpc_err = SYSTEM_ERR;
                 goto out;
         }
         gf_fd_put (conn->fdtable, args.fd);