glusterd: Ensured 'store' data reaches disk.

- Opened temporary file(s) with O_SYNC flag to avoid explicit fsync'ing.
- Sync'd directory entry after creation and rename of 'store' files.
- Thanks to Jeff Moyer's article on http://lwn.net/Articles/457667/

Change-Id: I68a8672dc6a0b24d128de53f3b60c74dd08d8ab8
BUG: 765434
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.com/3726
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

1 files changed, 55 insertions, 15 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 2f392f5760c..fba681e093a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -96,7 +96,7 @@ glusterd_store_mkstemp (glusterd_store_handle_t *shandle)
         GF_ASSERT (shandle->path);
 
         snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
-        fd = open (tmppath, O_RDWR | O_CREAT | O_TRUNC, 0600);
+        fd = open (tmppath, O_RDWR | O_CREAT | O_TRUNC | O_SYNC, 0600);
         if (fd <= 0) {
                 gf_log ("glusterd", GF_LOG_ERROR, "Failed to open %s, "
                         "error: %s", tmppath, strerror (errno));
@@ -105,6 +105,52 @@ glusterd_store_mkstemp (glusterd_store_handle_t *shandle)
         return fd;
 }
 
+int
+glusterd_store_sync_direntry (char *path)
+{
+        int             ret     = -1;
+        int             dirfd   = -1;
+        char            *dir    = NULL;
+        char            *pdir   = NULL;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+
+        dir = gf_strdup (path);
+        if (!dir)
+                goto out;
+
+        pdir = dirname (dir);
+        dirfd = open (pdir, O_RDONLY);
+        if (dirfd == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open directory "
+                        "%s, due to %s", pdir, strerror (errno));
+                goto out;
+        }
+
+        ret = fsync (dirfd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to fsync %s, due to "
+                        "%s", pdir, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (dirfd >= 0) {
+                ret = close (dirfd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to close "
+                                "%s, due to %s", pdir, strerror (errno));
+                }
+        }
+
+        if (dir)
+                GF_FREE (dir);
+
+        return ret;
+}
+
 int32_t
 glusterd_store_rename_tmppath (glusterd_store_handle_t *shandle)
 {
@@ -117,10 +163,13 @@ glusterd_store_rename_tmppath (glusterd_store_handle_t *shandle)
         snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
         ret = rename (tmppath, shandle->path);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to mv %s to %s, "
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to rename %s to %s, "
                         "error: %s", tmppath, shandle->path, strerror (errno));
+                goto out;
         }
 
+        ret = glusterd_store_sync_direntry (tmppath);
+out:
         return ret;
 }
 
@@ -283,8 +332,6 @@ glusterd_store_volinfo_brick_fname_write (int vol_fd,
         if (ret)
                 goto out;
 
-        ret = fsync (vol_fd);
-
 out:
         return ret;
 }
@@ -339,7 +386,6 @@ glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo)
         if (ret)
                 goto out;
 
-        ret = fsync (fd);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -655,8 +701,6 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
                 if (ret)
                         goto out;
         }
-        ret = fsync (fd);
-
 out:
         if (ret)
                 gf_log ("", GF_LOG_ERROR, "Unable to write volume values"
@@ -712,7 +756,6 @@ glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
 
         dict_foreach (volinfo->gsync_slaves, _storeslaves, shandle);
         shandle->fd = 0;
-        ret = fsync (fd);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -862,7 +905,6 @@ glusterd_store_rbstate_write (int fd, glusterd_volinfo_t *volinfo)
                         goto out;
         }
 
-        ret = fsync (fd);
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -918,10 +960,6 @@ glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        ret = fsync (fd);
-        if (ret)
-                goto out;
-
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -1404,11 +1442,14 @@ glusterd_store_handle_new (char *path, glusterd_store_handle_t **handle)
                 goto out;
         }
 
+        ret = glusterd_store_sync_direntry (spath);
+        if (ret)
+                goto out;
+
         shandle->path = spath;
         *handle = shandle;
 
         ret = 0;
-
 out:
         if (fd > 0)
                 close (fd);
@@ -2541,7 +2582,6 @@ glusterd_store_peer_write (int fd, glusterd_peerinfo_t *peerinfo)
         if (ret)
                 goto out;
 
-        ret = fsync (fd);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;