store: Add (un)locking functionality

Some configuration/cache files (like the NFS rmtab) can be stored on
a GlusterFS volume and be used by multiple storage servers. This
requires suitable locking for the gf_store_handle_t structure. Introduce
gf_store_lock() and gf_store_unlock() for this purpose. The
gf_store_locked_local() function can be used to check if the
gf_store_handle_t has been locked by the current process.

This change also includes an unrelated correction where a FILE* was
getting leaked. Krishnan Parthasarathi identified this while reviewing
the new locking functionality.

Change-Id: I431b7510801841d4bad64480b4bb99d87e2ad347
BUG: 904065
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: http://review.gluster.org/4677
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 73 insertions, 4 deletions

diff --git a/libglusterfs/src/store.c b/libglusterfs/src/store.c
index 8642538cefa..621146bea67 100644
--- a/libglusterfs/src/store.c
+++ b/libglusterfs/src/store.c
@@ -221,7 +221,12 @@ gf_store_retrieve_value (gf_store_handle_t *handle, char *key, char **value)
 
         GF_ASSERT (handle);
 
-        handle->fd = open (handle->path, O_RDWR);
+        if (handle->locked == F_ULOCK)
+                /* no locking is used handle->fd gets closed() after usage */
+                handle->fd = open (handle->path, O_RDWR);
+        else
+                /* handle->fd is valid already, kept open for lockf() */
+                lseek (handle->fd, 0, SEEK_SET);
 
         if (handle->fd == -1) {
                 gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %s",
@@ -229,7 +234,9 @@ gf_store_retrieve_value (gf_store_handle_t *handle, char *key, char **value)
                 goto out;
         }
         if (!handle->read)
-                handle->read = fdopen (handle->fd, "r");
+                handle->read = fdopen (dup(handle->fd), "r");
+        else
+                fseek (handle->read, 0, SEEK_SET);
 
         if (!handle->read) {
                 gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %s",
@@ -278,11 +285,16 @@ gf_store_retrieve_value (gf_store_handle_t *handle, char *key, char **value)
                 }
         } while (1);
 out:
-        if (handle->fd > 0) {
-                close (handle->fd);
+        if (handle->read) {
+                fclose (handle->read);
                 handle->read = NULL;
         }
 
+        if (handle->fd > 0 && handle->locked == F_ULOCK) {
+                /* only invalidate handle->fd if not locked */
+                close (handle->fd);
+        }
+
         GF_FREE (free_str);
 
         return ret;
@@ -366,6 +378,7 @@ gf_store_handle_new (char *path, gf_store_handle_t **handle)
                 goto out;
 
         shandle->path = spath;
+        shandle->locked = F_ULOCK;
         *handle = shandle;
 
         ret = 0;
@@ -630,3 +643,49 @@ gf_store_strerror (gf_store_op_errno_t op_errno)
         }
         return "Invalid errno";
 }
+
+int
+gf_store_lock (gf_store_handle_t *sh)
+{
+        int                     ret;
+
+        GF_ASSERT (sh);
+        GF_ASSERT (sh->path);
+        GF_ASSERT (sh->locked == F_ULOCK);
+
+        sh->fd = open (sh->path, O_RDWR);
+        if (sh->fd == -1) {
+                gf_log ("", GF_LOG_ERROR, "Failed to open '%s': %s", sh->path,
+                        strerror (errno));
+                return -1;
+        }
+
+        ret = lockf (sh->fd, F_LOCK, 0);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Failed to gain lock on '%s': %s",
+                        sh->path, strerror (errno));
+        else
+                /* sh->locked is protected by the lockf(sh->fd) above */
+                sh->locked = F_LOCK;
+
+        return ret;
+}
+
+void
+gf_store_unlock (gf_store_handle_t *sh)
+{
+        GF_ASSERT (sh);
+        GF_ASSERT (sh->locked == F_LOCK);
+
+        sh->locked = F_ULOCK;
+        lockf (sh->fd, F_ULOCK, 0);
+        close (sh->fd);
+}
+
+int
+gf_store_locked_local (gf_store_handle_t *sh)
+{
+        GF_ASSERT (sh);
+
+        return (sh->locked == F_LOCK);
+}
diff --git a/libglusterfs/src/store.h b/libglusterfs/src/store.h
index 4fe432e56de..337103ff73e 100644
--- a/libglusterfs/src/store.h
+++ b/libglusterfs/src/store.h
@@ -21,6 +21,7 @@ struct gf_store_handle_ {
         char    *path;
         int     fd;
         FILE    *read;
+        int     locked;   /* state of lockf() */
 };
 
 typedef struct gf_store_handle_ gf_store_handle_t;
@@ -99,4 +100,13 @@ gf_store_iter_destroy (gf_store_iter_t *iter);
 char*
 gf_store_strerror (gf_store_op_errno_t op_errno);
 
+int
+gf_store_lock (gf_store_handle_t *sh);
+
+void
+gf_store_unlock (gf_store_handle_t *sh);
+
+int
+gf_store_locked_local (gf_store_handle_t *sh);
+
 #endif