volumefile modification awareness to make sure there are no inconsistencies.

Complete (including feature to properly umount) in my sense.

Signed-off-by: Anand V. Avati <avati@amp.gluster.com>

6 files changed, 274 insertions, 96 deletions

diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index e876799a2..4affd40d5 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -53,8 +53,9 @@
 /* TODO: when supporting posix acl, remove this definition */
 #define DISABLE_POSIX_ACL
 
-#define ZR_MOUNTPOINT_OPT   "mountpoint"
-#define ZR_DIRECT_IO_OPT    "direct-io-mode"
+#define ZR_MOUNTPOINT_OPT       "mountpoint"
+#define ZR_DIRECT_IO_OPT        "direct-io-mode"
+#define ZR_STRICT_VOLFILE_CHECK "strict-volfile-check"
 
 #define BIG_FUSE_CHANNEL_SIZE 1048576
 
@@ -72,7 +73,7 @@ struct fuse_private {
         uint32_t             direct_io_mode;
         double               entry_timeout;
         double               attribute_timeout;
-
+        gf_boolean_t         strict_volfile_check;
 };
 typedef struct fuse_private fuse_private_t;
 
@@ -2553,6 +2554,10 @@ int32_t
 notify (xlator_t *this, int32_t event,
         void *data, ...)
 {
+        int32_t         ret     = 0;
+        fuse_private_t *private = NULL;
+
+        private = this->private;
   
         switch (event)
         {
@@ -2577,8 +2582,6 @@ notify (xlator_t *this, int32_t event,
 #endif /* DARWIN */
 
         {
-                fuse_private_t *private = this->private;
-                int32_t ret = 0;
 
                 if (!private->fuse_thread_started)
                 {
@@ -2597,6 +2600,21 @@ notify (xlator_t *this, int32_t event,
         case GF_EVENT_PARENT_UP:
         {
                 default_notify (this, GF_EVENT_PARENT_UP, data);
+                break;
+        }
+        case GF_EVENT_VOLFILE_MODIFIED:
+        {
+                gf_log ("fuse", GF_LOG_CRITICAL, 
+                        "remote volume file changed, try re-mounting");
+                if (private->strict_volfile_check) {
+                        //fuse_session_remove_chan (private->ch);
+                        //fuse_session_destroy (private->se);
+                        //fuse_unmount (private->mount_point, private->ch);
+                        /* TODO: Above code if works, will be a cleaner way, 
+                           but for now, lets just achieve what we want */
+                        raise (SIGTERM);
+                }
+                break;
         }
         default:
                 break;
@@ -2736,7 +2754,14 @@ init (xlator_t *this_xl)
 	if (value_string) {
 		ret = gf_string2boolean (value_string, &priv->direct_io_mode);
 	}
-	
+
+        priv->strict_volfile_check = 0;
+	ret = dict_get_str (options, ZR_STRICT_VOLFILE_CHECK, &value_string);
+	if (value_string) {
+		ret = gf_string2boolean (value_string, 
+                                         &priv->strict_volfile_check);
+	}
+
         priv->ch = fuse_mount (priv->mount_point, &args);
         if (priv->ch == NULL) {
                 if (errno == ENOTCONN) {
@@ -2851,5 +2876,8 @@ struct volume_options options[] = {
 	{ .key  = {"entry-timeout"}, 
 	  .type = GF_OPTION_TYPE_DOUBLE
 	},
+	{ .key  = {"strict-volfile-check"}, 
+	  .type = GF_OPTION_TYPE_BOOL
+	},
 	{ .key = {NULL} },
 };
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 481fd265f..58da509f1 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -66,6 +66,10 @@ start_glusterfs ()
 	cmd_line=$(echo "$cmd_line --log-file=$log_file");
     fi
 
+    if [ -n "$volfile_check" ]; then
+	cmd_line=$(echo "$cmd_line --volfile-check");
+    fi
+
     if [ -n "$direct_io_mode" ]; then
 	cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
     fi
@@ -115,7 +119,9 @@ main ()
     volume_name=$(echo "$options" | sed -n 's/.*volume-name=\([^,]*\).*/\1/p');
 
     volume_id=$(echo "$options" | sed -n 's/.*volume-id=\([^,]*\).*/\1/p');
-    
+
+    volfile_check=$(echo "$options" | sed -n 's/.*volfile-check=\([^,]*\).*/\1/p');
+
     volfile_loc="$1";
     
     [ -r "$volfile_loc" ] || {
@@ -131,6 +137,7 @@ main ()
 	                                   -e 's/[,]*log-level=[^,]*//' \
 	                                   -e 's/[,]*volume-name=[^,]*//' \
 	                                   -e 's/[,]*direct-io-mode=[^,]*//' \
+	                                   -e 's/[,]*volfile-check=[^,]*//' \
 	                                   -e 's/[,]*transport=[^,]*//' \
 	                                   -e 's/[,]*volume-id=[^,]*//');
     # following line is product of love towards sed
diff --git a/xlators/mount/fuse/utils/mount_glusterfs.in b/xlators/mount/fuse/utils/mount_glusterfs.in
index 1376a8897..b064e1ead 100755
--- a/xlators/mount/fuse/utils/mount_glusterfs.in
+++ b/xlators/mount/fuse/utils/mount_glusterfs.in
@@ -66,8 +66,12 @@ start_glusterfs ()
 	cmd_line=$(echo "$cmd_line --log-file=$log_file");
     fi
 
+    if [ -n "$volfile_check" ]; then
+	cmd_line=$(echo "$cmd_line --volfile-check");
+    fi
+
     if [ -n "$direct_io_mode" ]; then
-	cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
+	cmd_line=$(echo "$cmd_line --disable-direct-io-mode");
     fi
     
     if [ -z "$volfile_loc" ]; then
@@ -107,6 +111,7 @@ main ()
     direct_io_mode=""
     volume_name=""
     new_fs_options=""
+    volfile_check=""
 
     while getopts o: opt; do
 	case "$opt" in
@@ -128,6 +133,10 @@ main ()
 		    direct_io_mode=$(echo "$options" | sed -n 's/.*direct-io-mode=\([^,]*\).*/\1/p');
 		}
 		
+		[ -z $volfile_check ] && {
+		    volfile_check=$(echo "$options" | sed -n 's/.*volfile-check=\([^,]*\).*/\1/p');
+		}
+		
 		[ -z $volume_name ] && {
 		    volume_name=$(echo "$options" | sed -n 's/.*volume-name=\([^,]*\).*/\1/p');
 		}
@@ -139,6 +148,7 @@ main ()
 		this_option=$(echo "$options" | sed -e 's/[,]*log-file=[^,]*//' \
 		    -e 's/[,]*log-level=[^,]*//' \
 		    -e 's/[,]*volume-name=[^,]*//' \
+		    -e 's/[,]*volfile-check=[^,]*//' \
 		    -e 's/[,]*direct-io-mode=[^,]*//' \
 		    -e 's/[,]*transport=[^,]*//' \
 		    -e 's/[,]*volume-id=[^,]*//');
diff --git a/xlators/protocol/client/src/client-protocol.c b/xlators/protocol/client/src/client-protocol.c
index da2242ad6..11e669833 100644
--- a/xlators/protocol/client/src/client-protocol.c
+++ b/xlators/protocol/client/src/client-protocol.c
@@ -5875,7 +5875,7 @@ client_setvolume_cbk (call_frame_t *frame,
 	op_ret   = ntoh32 (hdr->rsp.op_ret);
 	op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
 
-	if (op_ret < 0 && op_errno == ENOTCONN) {
+	if ((op_ret < 0) && (op_errno == ENOTCONN)) {
 		gf_log (this->name, GF_LOG_ERROR,
 			"setvolume failed (%s)",
 			strerror (op_errno));
@@ -5911,8 +5911,16 @@ client_setvolume_cbk (call_frame_t *frame,
 			"SETVOLUME on remote-host failed: %s",
 			remote_error ? remote_error : strerror (op_errno));
 		errno = op_errno;
-		if (op_errno == ENOTCONN)
-			goto out;
+                if (op_errno == ESTALE) {
+                        parent = trans->xl->parents;
+                        while (parent) {
+                                parent->xlator->notify (parent->xlator,
+                                                        GF_EVENT_VOLFILE_MODIFIED,
+                                                        trans->xl);
+                                parent = parent->next;
+                        }
+                }
+
 	} else {
 		ctx = get_global_ctx_ptr ();
 		if (process_uuid && !strcmp (ctx->process_uuid,process_uuid)) {
@@ -6439,6 +6447,14 @@ protocol_client_handshake (xlator_t *this, transport_t *trans)
 			PACKAGE_VERSION);
 	}
 
+        if (this->ctx->cmd_args.volfile_server) {
+                if (this->ctx->cmd_args.volfile_id)
+                        ret = dict_set_str (options, "volfile-key", 
+                                            this->ctx->cmd_args.volfile_id);
+                ret = dict_set_uint32 (options, "volfile-checksum", 
+                                       this->ctx->volfile_checksum);
+        }
+
 	dict_len = dict_serialized_length (options);
 	if (dict_len < 0) {
 		gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/protocol/server/src/server-protocol.c b/xlators/protocol/server/src/server-protocol.c
index 68e1cc06c..4e06aaa5e 100644
--- a/xlators/protocol/server/src/server-protocol.c
+++ b/xlators/protocol/server/src/server-protocol.c
@@ -6777,6 +6777,150 @@ out:
 
 /* xxx_MOPS */
 
+
+int
+_volfile_update_checksum (xlator_t *this, char *key, uint32_t checksum)
+{
+        server_conf_t       *conf         = NULL;
+        struct _volfile_ctx *temp_volfile = NULL;
+
+        conf         = this->private;
+        temp_volfile = conf->volfile;
+
+        while (temp_volfile) {
+                if ((NULL == key) && (NULL == temp_volfile->key))
+                        break;
+                if ((NULL == key) || (NULL == temp_volfile->key)) {
+                        temp_volfile = temp_volfile->next;
+                        continue;
+                }
+                if (strcmp (temp_volfile->key, key) == 0)
+                        break;
+                temp_volfile = temp_volfile->next;
+        }
+
+        if (!temp_volfile) {
+                temp_volfile = CALLOC (1, sizeof (struct _volfile_ctx));
+
+                temp_volfile->next  = conf->volfile;
+                temp_volfile->key   = (key)? strdup (key): NULL;
+                temp_volfile->checksum = checksum;
+                
+                conf->volfile = temp_volfile;
+                goto out;
+        }
+
+        if (temp_volfile->checksum != checksum) {
+                gf_log (this->name, GF_LOG_CRITICAL, 
+                        "the volume file got modified between earlier access "
+                        "and now, this may lead to inconsistency between "
+                        "clients, advised to remount client");
+                temp_volfile->checksum  = checksum;
+        }
+
+ out:
+        return 0;
+}
+
+
+char *
+build_volfile_path (xlator_t *this, char *key)
+{
+        int   ret = -1;
+        char *filename = NULL;
+	char  data_key[256] = {0,};
+
+	/* Inform users that this option is changed now */
+	ret = dict_get_str (this->options, "client-volume-filename", 
+			    &filename);
+	if (ret == 0) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"option 'client-volume-filename' is changed to "
+			"'volume-filename.<key>' which now takes 'key' as an "
+			"option to choose/fetch different files from server. "
+			"Refer documentation or contact developers for more "
+			"info. Currently defaulting to given file '%s'", 
+			filename);
+	}
+	
+	if (key && !filename) {
+		sprintf (data_key, "volume-filename.%s", key);
+		ret = dict_get_str (this->options, data_key, &filename);
+		if (ret < 0) {
+			gf_log (this->name, GF_LOG_ERROR,
+				"failed to get corresponding volume file "
+				"for the key '%s'.", key);
+		} 
+	}
+        
+	if (!filename) {
+		ret = dict_get_str (this->options, 
+                                    "volume-filename.default", &filename);
+		if (ret < 0) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"no default volume filename given, "
+                                "defaulting to %s", DEFAULT_VOLUME_FILE_PATH);
+
+                        filename = DEFAULT_VOLUME_FILE_PATH;
+                }
+	}
+
+        return filename;
+}
+
+int 
+_validate_volfile_checksum (xlator_t *this, char *key,
+                            uint32_t checksum)
+{        
+	char                *filename     = NULL;
+        server_conf_t       *conf         = NULL;
+        struct _volfile_ctx *temp_volfile = NULL;
+        int                  ret          = 0;
+        uint32_t             local_checksum = 0;
+
+        conf         = this->private;
+        temp_volfile = conf->volfile;
+        
+        if (!checksum) 
+                goto out;
+        
+        if (!temp_volfile) {
+                filename = build_volfile_path (this, key);
+                if (NULL == filename)
+                        goto out;
+                ret = open (filename, O_RDONLY);
+                if (-1 == ret) {
+                        ret = 0;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "failed to open volume file (%s) : %s",
+                                filename, strerror (errno));
+                        goto out;
+                }
+                get_checksum_for_file (ret, &local_checksum);
+                _volfile_update_checksum (this, key, local_checksum);
+                close (ret);
+        }
+
+        temp_volfile = conf->volfile;
+        while (temp_volfile) {
+                if ((NULL == key) && (NULL == temp_volfile->key))
+                        break;
+                if (strcmp (temp_volfile->key, key) == 0)
+                        break;
+                temp_volfile = temp_volfile->next;
+        }
+
+        if (!temp_volfile)
+                goto out;
+
+        if ((temp_volfile->checksum) && 
+            (checksum != temp_volfile->checksum)) 
+                ret = -1;
+
+ out:
+        return ret;
+}
+
 /* Management Calls */
 /*
  * mop_getspec - getspec function for server protocol
@@ -6799,17 +6943,14 @@ mop_getspec (call_frame_t *frame,
 	int32_t spec_fd = -1;
 	size_t  file_len = 0;
 	size_t _hdrlen = 0;
-	char  tmp_filename[ZR_FILENAME_MAX] = {0,};
-	char  data_key[256] = {0,};
 	char *filename = NULL;
 	struct stat stbuf = {0,};
-	peer_info_t *peerinfo = NULL;
-	transport_t *trans = NULL;
 
 	gf_mop_getspec_req_t *req = NULL;
+        uint32_t checksum = 0;
 	uint32_t flags  = 0;
 	uint32_t keylen = 0;
-	char *key = NULL;
+	char    *key    = NULL;
 
 	req   = gf_param (hdr);
 	flags = ntoh32 (req->flags);
@@ -6818,85 +6959,32 @@ mop_getspec (call_frame_t *frame,
 		key = req->key;
 	}
 
-	trans = TRANSPORT_FROM_FRAME(frame);
-
-	peerinfo = &(trans->peerinfo);
-	/* Inform users that this option is changed now */
-	ret = dict_get_str (frame->this->options, "client-volume-filename", 
-			    &filename);
-	if (ret == 0) {
-		gf_log (trans->xl->name, GF_LOG_WARNING,
-			"option 'client-volume-filename' is changed to "
-			"'volume-filename.<key>' which now takes 'key' as an "
-			"option to choose/fetch different files from server. "
-			"Refer documentation or contact developers for more "
-			"info. Currently defaulting to given file '%s'", 
-			filename);
-	}
-	
-	if (key && !filename) {
-		sprintf (data_key, "volume-filename.%s", key);
-		ret = dict_get_str (frame->this->options, data_key, &filename);
-		if (ret < 0) {
-			gf_log (trans->xl->name, GF_LOG_ERROR,
-				"failed to get corresponding volume file "
-				"for the key '%s'. using default file %s", 
-				key, GLUSTERFSD_SPEC_PATH);
-		} 
-	}
-        
-	if (!filename) {
-		ret = dict_get_str (frame->this->options, 
-                                    "volume-filename.default", &filename);
-		if (ret < 0) {
-			gf_log (trans->xl->name, GF_LOG_DEBUG,
-				"no default volume filename given, "
-                                "defaulting to %s", GLUSTERFSD_SPEC_PATH);
-
-                        filename = GLUSTERFSD_SPEC_PATH;
-                }
-	}
-
-	{
-		sprintf (tmp_filename, "%s.%s", 
-			 filename, peerinfo->identifier);
-
-		/* Try for ip specific client volfile.
-		 * If not found, then go for, regular client file.
-		 */
-		ret = open (tmp_filename, O_RDONLY);
-		spec_fd = ret;
-		if (spec_fd < 0) {
-			gf_log (trans->xl->name, GF_LOG_DEBUG,
-				"Unable to open %s (%s)", 
-				tmp_filename, strerror (errno));
-			/* fall back */
-			ret = open (filename, O_RDONLY);
-			spec_fd = ret;
-			if (spec_fd < 0) {
-				gf_log (trans->xl->name, GF_LOG_ERROR,
-					"Unable to open %s (%s)", 
-					filename, strerror (errno));
-				goto fail;
-			}
-		} else {
-			/* Successful */
-			filename = tmp_filename;
-		}
-	}
-
-	/* to allocate the proper buffer to hold the file data */
-	{
+        filename = build_volfile_path (frame->this, key);
+        if (filename) {
+                /* to allocate the proper buffer to hold the file data */
 		ret = stat (filename, &stbuf);
 		if (ret < 0){
-			gf_log (trans->xl->name, GF_LOG_ERROR,
+			gf_log (frame->this->name, GF_LOG_ERROR,
 				"Unable to stat %s (%s)", 
 				filename, strerror (errno));
 			goto fail;
 		}
-
-		file_len = stbuf.st_size;
-	}
+                
+                ret = open (filename, O_RDONLY);
+                spec_fd = ret;
+                if (spec_fd < 0) {
+                        gf_log (frame->this->name, GF_LOG_ERROR,
+                                "Unable to open %s (%s)", 
+                                filename, strerror (errno));
+                        goto fail;
+                }
+                ret = 0;
+                file_len = stbuf.st_size;
+                get_checksum_for_file (spec_fd, &checksum);
+                _volfile_update_checksum (frame->this, key, checksum);
+	} else {
+                errno = ENOENT;
+        }
 
 fail:
 	op_errno = errno;
@@ -6910,7 +6998,7 @@ fail:
 	_hdr->rsp.op_errno = hton32 (gf_errno);
 
 	if (file_len) {
-		read (spec_fd, rsp->spec, file_len);
+		ret = read (spec_fd, rsp->spec, file_len);
 		close (spec_fd);
 	}
 	protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_GETSPEC,
@@ -7043,7 +7131,7 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
                gf_hdr_common_t *req_hdr, size_t req_hdrlen,
                char *req_buf, size_t req_buflen)
 {
-	server_connection_t *conn = NULL;
+	server_connection_t         *conn = NULL;
 	server_conf_t               *conf = NULL;
 	gf_hdr_common_t             *rsp_hdr = NULL;
 	gf_mop_setvolume_req_t      *req = NULL;
@@ -7064,6 +7152,9 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
 	size_t                       rsp_hdrlen = -1;
 	size_t                       dict_len = -1;
 	size_t                       req_dictlen = -1;
+        char                        *msg = NULL;
+        char                        *volfile_key = NULL;
+        uint32_t                     checksum = 0;
 
 	params = dict_new ();
 	reply  = dict_new ();
@@ -7124,7 +7215,6 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
 	
 	ret = strcmp (version, PACKAGE_VERSION);
 	if (ret != 0) {
-		char *msg = NULL;
 		asprintf (&msg,
 			  "Version mismatch: client(%s) Vs server (%s)",
 			  version, PACKAGE_VERSION);
@@ -7138,7 +7228,6 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
 		goto fail;
 	}
 
-
 	ret = dict_get_str (params,
 			    "remote-subvolume", &name);
 	if (ret < 0) {
@@ -7155,7 +7244,6 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
 
 	xl = get_xlator_by_name (frame->this, name);
 	if (xl == NULL) {
-		char *msg = NULL;
 		asprintf (&msg, "remote-subvolume \"%s\" is not found", name);
 		ret = dict_set_dynstr (reply, "ERROR", msg);
 		if (ret < 0)
@@ -7167,6 +7255,27 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
 		goto fail;
 	}
 
+	ret = dict_get_uint32 (params, "volfile-checksum", &checksum);
+	if (ret == 0) {
+                ret = dict_get_str (params, "volfile-key", &volfile_key);
+                
+                ret = _validate_volfile_checksum (trans->xl, volfile_key, 
+                                                  checksum);
+                if (-1 == ret) {
+                        ret = dict_set_str (reply, "ERROR",
+                                            "volume-file checksum varies from "
+                                            "earlier access");
+                        if (ret < 0)
+                                gf_log (trans->xl->name, GF_LOG_ERROR, 
+                                        "failed to set error msg");
+                        
+                        op_ret   = -1;
+                        op_errno = ESTALE;
+                        goto fail;
+                }
+	}
+
+
 	peerinfo = &trans->peerinfo;
 	ret = dict_set_static_ptr (params, "peer-info", peerinfo);
 	if (ret < 0)
diff --git a/xlators/protocol/server/src/server-protocol.h b/xlators/protocol/server/src/server-protocol.h
index 3b28a3e5e..30ccb91e7 100644
--- a/xlators/protocol/server/src/server-protocol.h
+++ b/xlators/protocol/server/src/server-protocol.h
@@ -35,8 +35,8 @@
 #include "fd.h"
 #include "byte-order.h"
 
-#define DEFAULT_BLOCK_SIZE     4194304   /* 4MB */
-#define GLUSTERFSD_SPEC_PATH   CONFDIR "/glusterfs-client.vol"
+#define DEFAULT_BLOCK_SIZE         4194304   /* 4MB */
+#define DEFAULT_VOLUME_FILE_PATH   CONFDIR "/glusterfs.vol"
 
 typedef struct _server_state server_state_t;
 
@@ -86,7 +86,15 @@ server_nop_cbk (call_frame_t *frame, void *cookie,
 		xlator_t *this, int32_t op_ret, int32_t op_errno);
 
 
+struct _volfile_ctx {
+        struct _volfile_ctx *next;
+        char                *key;
+        uint32_t             checksum;
+};
+
 typedef struct {
+        struct _volfile_ctx *volfile;
+
 	dict_t           *auth_modules;
 	transport_t      *trans;
 	int32_t           max_block_size;