Reduce iothreads requests queue lock granularity.

iot_queue() and iot_dequeue() functions were using a io-threads
translator-wide lock which would be contended for by every worker
thread waiting for IO requests.

This patch reduces the granularity by turning the
lock into a per-worker lock.

Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

2 files changed, 22 insertions, 34 deletions

diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index 346e1234bb9..b3054ae6b86 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -1021,27 +1021,24 @@ iot_queue (iot_worker_t *worker,
            call_stub_t *stub)
 {
 	iot_queue_t *queue;
-	iot_conf_t *conf = worker->conf;
 
 	queue = CALLOC (1, sizeof (*queue));
 	ERR_ABORT (queue);
 	queue->stub = stub;
 
-	pthread_mutex_lock (&conf->lock);
+        pthread_mutex_lock (&worker->qlock);
+        queue->next = &worker->queue;
+        queue->prev = worker->queue.prev;
 
-	queue->next = &worker->queue;
-	queue->prev = worker->queue.prev;
+        queue->next->prev = queue;
+        queue->prev->next = queue;
 
-	queue->next->prev = queue;
-	queue->prev->next = queue;
+        /* dq_cond */
+        worker->queue_size++;
+        worker->q++;
 
-	/* dq_cond */
-	worker->queue_size++;
-	worker->q++;
-
-	pthread_cond_broadcast (&worker->dq_cond);
-
-	pthread_mutex_unlock (&conf->lock);
+        pthread_cond_broadcast (&worker->dq_cond);
+	pthread_mutex_unlock (&worker->qlock);
 }
 
 static call_stub_t *
@@ -1049,27 +1046,20 @@ iot_dequeue (iot_worker_t *worker)
 {
 	call_stub_t *stub = NULL;
 	iot_queue_t *queue = NULL;
-	iot_conf_t *conf = worker->conf;
-
-
-	pthread_mutex_lock (&conf->lock);
 
-	while (!worker->queue_size)
-		pthread_cond_wait (&worker->dq_cond, &conf->lock);
+	pthread_mutex_lock (&worker->qlock);
+        while (!worker->queue_size)
+	       pthread_cond_wait (&worker->dq_cond, &worker->qlock);
 
-	queue = worker->queue.next;
+        queue = worker->queue.next;
+        queue->next->prev = queue->prev;
+        queue->prev->next = queue->next;
 
-	queue->next->prev = queue->prev;
-	queue->prev->next = queue->next;
+        stub = queue->stub;
 
-	stub = queue->stub;
-
-	worker->queue_size--;
-	worker->dq++;
-
-	pthread_cond_broadcast (&conf->q_cond);
-
-	pthread_mutex_unlock (&conf->lock);
+        worker->queue_size--;
+        worker->dq++;
+	pthread_mutex_unlock (&worker->qlock);
 
 	FREE (queue);
 
@@ -1110,6 +1100,7 @@ workers_init (iot_conf_t *conf)
 		worker->queue.next = &worker->queue;
 		worker->queue.prev = &worker->queue;
 
+		pthread_mutex_init (&worker->qlock, NULL);
 		pthread_cond_init (&worker->dq_cond, NULL);
 		worker->conf = conf;
 
@@ -1149,9 +1140,6 @@ init (xlator_t *this)
 			conf->thread_count);
 	}
 
-	pthread_mutex_init (&conf->lock, NULL);
-	pthread_cond_init (&conf->q_cond, NULL);
-
 	conf->files.next = &conf->files;
 	conf->files.prev = &conf->files;
 	pthread_mutex_init (&conf->files_lock, NULL);
diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h
index 419f6fdb8e6..401d0d86dc9 100644
--- a/xlators/performance/io-threads/src/io-threads.h
+++ b/xlators/performance/io-threads/src/io-threads.h
@@ -58,6 +58,7 @@ struct iot_worker {
   struct iot_conf *conf;
   int64_t q,dq;
   pthread_cond_t dq_cond;
+  pthread_mutex_t qlock;
   int32_t fd_count;
   int32_t queue_size;
   pthread_t thread;
@@ -78,7 +79,6 @@ struct iot_conf {
   pthread_mutex_t files_lock;
 
   pthread_cond_t q_cond;
-  pthread_mutex_t lock;
 };
 
 typedef struct iot_file iot_file_t;