From 11f6c56f83b977a08f9d74563249cef59e22a05d Mon Sep 17 00:00:00 2001 From: Avra Sengupta Date: Tue, 4 Jun 2013 14:20:58 +0530 Subject: features/changelog: changelog translator This is the initial version of the Changelog Translator. What is it ----------- Goal is to capture changes performed on a GlusterFS volume. The translator needs to be loaded on the server (bricks) and captures changes in a plain text file inside a configured directory path (controlled by "changelog-dir", should be somewhere in /.glusterfs/changelog by default). Changes are classified into 3 types: - Data: : TYPE-I - Metadata : TYPE-II - Entry : TYPE-III Changelog file is rolled over after a certain time interval (defauls to 60 seconds) after which a changelog is started. The thing to be noted here is that for a time interval (time slice) multiple changes for an inode are recorded only once (ie. say for 100+ writes on an inode that happens within the time slice has only a single corresponding entry in the changelog file). That way we do not bloat up the changelog and also save lots of writes. Changelog Format ----------------- TYPE-I and TYPE-II changes have the gfid on the entity on which the operation happened. TYPE-III being a entry op requires the parent gfid and the basename. Changelog format has been kept to a minimal and it's upto the consumers to do the heavy loading of figuring out deletes, renames etc.. A single changelog file records all three types of changes, with each change starting with an identifier ("D": DATA, "M": METADATA and "E": ENTRY). Option is provided for the encoding type (See TUNABLES). Consumers ---------- The only consumer as of today would be geo-replication, although backup utilities, self-heal, bit-rot detection could be possible consumers in the future. CLI ---- By default, change-logging is disabled (the translator is present in the server graph but does nothing). When enabled (via cli) each brick starts to log the changes. There are a set of tunable that can be used to change the translators behaviour: - enable/disable changelog (disabled by default) gluster volume set changelog {on|off} - set the logging directory (/.glusterfs/changelogs is the default) gluster volume set changelog-dir /path/to/dir - select encoding type (binary (default) or ascii) gluster volume set encoding {binary|ascii} - change the rollover time for the logs (60 secs by default) gluster volume set rollover-time - when secs > 0, changelog file is not open()'d with O_SYNC flag - and fsync is trigerred periodically every seconds. gluster volume set fsync-interval features/changelog: changelog consumer library (libgfchangelog) A shared library is provided for the consumer of the changelogs for easy acess via APIs. Application can link against this library and request for changelog updates. Conversion of binary logs to human-readable ascii format is also taken care by the library which keeps a copy of the changelog in application provided working directory. Change-Id: I75575fb7f1c53d2bec3dba1a329ea7bb3c628497 BUG: 847839 Original Author: Venky Shankar Signed-off-by: Avra Sengupta Reviewed-on: http://review.gluster.org/5127 Tested-by: Gluster Build System Reviewed-by: Vijay Bellur --- libglusterfs/src/common-utils.c | 10 ++++++++++ libglusterfs/src/common-utils.h | 2 ++ libglusterfs/src/glusterfs.h | 2 ++ 3 files changed, 14 insertions(+) (limited to 'libglusterfs') diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c index 034e3da25..6eb886b38 100644 --- a/libglusterfs/src/common-utils.c +++ b/libglusterfs/src/common-utils.c @@ -59,6 +59,16 @@ struct dnscache6 { struct addrinfo *next; }; +void +md5_wrapper(const unsigned char *data, size_t len, char *md5) +{ + unsigned short i = 0; + unsigned short lim = MD5_DIGEST_LENGTH*2+1; + unsigned char scratch[MD5_DIGEST_LENGTH] = {0,}; + MD5(data, len, scratch); + for (; i < MD5_DIGEST_LENGTH; i++) + snprintf(md5 + i * 2, lim-i*2, "%02x", scratch[i]); +} /* works similar to mkdir(1) -p. */ diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h index b0d9c18b3..bf41444d1 100644 --- a/libglusterfs/src/common-utils.h +++ b/libglusterfs/src/common-utils.h @@ -23,6 +23,7 @@ #include #include #include +#include #ifndef GF_BSD_HOST_OS #include #endif @@ -584,5 +585,6 @@ gf_boolean_t gf_ports_reserved (char *blocked_port, gf_boolean_t *ports); int gf_get_hostname_from_ip (char *client_ip, char **hostname); gf_boolean_t gf_is_local_addr (char *hostname); gf_boolean_t gf_is_same_address (char *host1, char *host2); +void md5_wrapper(const unsigned char *data, size_t len, char *md5); #endif /* _COMMON_UTILS_H */ diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h index 6c09e89b2..dbb2b58d9 100644 --- a/libglusterfs/src/glusterfs.h +++ b/libglusterfs/src/glusterfs.h @@ -156,6 +156,8 @@ #define GF_REMOVE_BRICK_TID_KEY "remove-brick-id" #define GF_REPLACE_BRICK_TID_KEY "replace-brick-id" +#define UUID_CANONICAL_FORM_LEN 36 + /* NOTE: add members ONLY at the end (just before _MAXVALUE) */ typedef enum { GF_FOP_NULL = 0, -- cgit