diff options
-rw-r--r-- | common.h | 2 | ||||
-rw-r--r-- | glfs-operations.c | 189 | ||||
-rw-r--r-- | glfs-operations.h | 55 | ||||
-rw-r--r-- | gluster-block.c | 28 | ||||
-rw-r--r-- | gluster-blockd.c | 171 | ||||
-rw-r--r-- | rpc/block.h | 1 | ||||
-rw-r--r-- | rpc/block.x | 1 | ||||
-rw-r--r-- | rpc/block_xdr.c | 3 | ||||
-rw-r--r-- | utils.h | 24 |
9 files changed, 422 insertions, 52 deletions
@@ -14,6 +14,8 @@ # include "utils.h" +# define LOG_FILE "/var/log/gluster-block/block.log" +# define LOG_LEVEL 7 size_t glusterBlockCreateParseSize(char *value); diff --git a/glfs-operations.c b/glfs-operations.c index 7f76b23..5b99045 100644 --- a/glfs-operations.c +++ b/glfs-operations.c @@ -9,28 +9,24 @@ */ -# include "utils.h" +# include "common.h" # include "glfs-operations.h" -# define LOG_FILE "/var/log/gluster-block/block.log" -# define LOG_LEVEL 7 - -int -glusterBlockCreateEntry(blockCreateCli *blk, char *gbid) +struct glfs * +glusterBlockVolumeInit(char *volume, char *volfileserver) { struct glfs *glfs; - struct glfs_fd *fd; int ret = 0; - glfs = glfs_new(blk->volume); + glfs = glfs_new(volume); if (!glfs) { ERROR("%s", "glfs_new: returned NULL"); - return -1; + return NULL; } - ret = glfs_set_volfile_server(glfs, "tcp", blk->volfileserver, 24007); + ret = glfs_set_volfile_server(glfs, "tcp", volfileserver, 24007); if (ret) { ERROR("%s", "glfs_set_volfile_server: failed"); goto out; @@ -48,6 +44,27 @@ glusterBlockCreateEntry(blockCreateCli *blk, char *gbid) goto out; } + return glfs; + + out: + glfs_fini(glfs); + return NULL; +} + + +int +glusterBlockCreateEntry(blockCreateCli *blk, char *gbid) +{ + struct glfs *glfs; + struct glfs_fd *fd; + int ret = -1; + + glfs = glusterBlockVolumeInit(blk->volume, blk->volfileserver); + if (!glfs) { + ERROR("%s", "glusterBlockVolumeInit: failed"); + goto out; + } + fd = glfs_creat(glfs, gbid, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); @@ -77,39 +94,159 @@ int glusterBlockDeleteEntry(blockCreate *blk) { struct glfs *glfs; - int ret = 0; + int ret = -1; - glfs = glfs_new(blk->volume); + glfs = glusterBlockVolumeInit(blk->volume, blk->volfileserver); if (!glfs) { - ERROR("%s", "glfs_new: returned NULL"); - return -1; + ERROR("%s", "glusterBlockVolumeInit: failed"); + goto out; } - ret = glfs_set_volfile_server(glfs, "tcp", blk->volfileserver, 24007); + ret = glfs_unlink(glfs, blk->gbid); if (ret) { - ERROR("%s", "glfs_set_volfile_server: failed"); + ERROR("%s", "glfs_unlink: failed"); goto out; } - ret = glfs_set_logging(glfs, LOG_FILE, LOG_LEVEL); - if (ret) { - ERROR("%s", "glfs_set_logging: failed"); + out: + glfs_fini(glfs); + return ret; +} + + +struct glfs_fd * +glusterBlockCreateMetaLockFile(struct glfs *glfs) +{ + struct glfs_fd *lkfd; + int ret; + + ret = glfs_mkdir (glfs, "/block-meta", 0); + if (ret && errno != EEXIST) { + ERROR("%s", "glfs_mkdir: failed"); goto out; } - ret = glfs_init(glfs); + ret = glfs_chdir (glfs, "/block-meta"); if (ret) { - ERROR("%s", "glfs_init: failed"); + ERROR("%s", "glfs_chdir: failed"); goto out; } - ret = glfs_unlink(glfs, blk->gbid); - if (ret) { - ERROR("%s", "glfs_unlink: failed"); + lkfd = glfs_creat(glfs, "meta.lock", O_RDWR, S_IRUSR | S_IWUSR); + if (!lkfd) { + ERROR("%s", "glfs_creat: failed"); goto out; } + return lkfd; + out: - glfs_fini(glfs); - return ret; + return NULL; +} + + +static int +blockEnumParse(const char *opt) +{ + int i; + + if (!opt) { + return METAKEY__MAX; + } + + for (i = 0; i < METAKEY__MAX; i++) { + if (!strcmp(opt, MetakeyLookup[i])) { + return i; + } + } + + return i; +} + +void +blockFreeMetaInfo(MetaInfo *info) +{ + int i; + + for (i = 0; i< info->nhosts; i++) + GB_FREE(info->list[i]); + + GB_FREE(info->list); + GB_FREE(info); +} + +static void +blockStuffMetaInfo(MetaInfo *info, char *line) +{ + char* tmp = strdup(line); + char* opt = strtok(tmp,":"); + int Flag = 0; + size_t i; + + switch (blockEnumParse(opt)) { + case GBID: + strcpy(info->gbid, strchr(line, ' ')+1); + break; + case SIZE: + sscanf(strchr(line, ' ')+1, "%zu", &info->size); + break; + case HA: + sscanf(strchr(line, ' ')+1, "%zu", &info->mpath); + break; + case ENTRYCREATE: + strcpy(info->entry, strchr(line, ' ')+1); + break; + + default: + if(!info->list) { + if(GB_ALLOC(info->list) < 0) + return; + if(GB_ALLOC(info->list[0]) < 0) + return; + strcpy(info->list[0]->addr, opt); + strcpy(info->list[0]->status, strchr(line, ' ')+1); + info->nhosts = 1; + } else { + for (i = 0; i < info->nhosts; i++) { + if(!strcmp(info->list[i]->addr, opt)) { + strcpy(info->list[i]->status, strchr(line, ' ')+1); + Flag = 1; + break; + } + } + if (!Flag) { + if(GB_ALLOC(info->list[info->nhosts]) < 0) + return; + strcpy(info->list[info->nhosts]->addr, opt); + strcpy(info->list[info->nhosts]->status, strchr(line, ' ')+1); + info->nhosts++; + } + } + break; + } + + GB_FREE(tmp); +} + +void +blockGetMetaInfo(struct glfs* glfs, char* metafile, MetaInfo *info) +{ + size_t count = 0; + struct glfs_fd *tgfd; + char line[48]; + char *tmp; + + tgfd = glfs_open(glfs, metafile, O_RDWR); + if (!tgfd) { + ERROR("%s", "glfs_open failed"); + } + + while (glfs_read (tgfd, line, 48, 0) > 0) { + tmp = strtok(line,"\n"); + count += strlen(tmp) + 1; + blockStuffMetaInfo(info, tmp); + glfs_lseek(tgfd, count, SEEK_SET); + } + + glfs_close(tgfd); } diff --git a/glfs-operations.h b/glfs-operations.h index ceadcd7..8fe1bd6 100644 --- a/glfs-operations.h +++ b/glfs-operations.h @@ -22,19 +22,56 @@ # include "rpc/block.h" +typedef enum Metakey { + GBID = 0, + SIZE = 1, + HA = 2, + ENTRYCREATE = 3, -typedef struct glusterBlockDef { - char *volume; - char *host; /* TODO: use proper Transport Object */ - char *filename; + METAKEY__MAX = 4 /* Updata this when add new Key */ +} Metakey; + + +typedef struct NodeInfo { + char addr[255]; + char status[16]; +} NodeInfo; + +typedef struct MetaInfo { + char gbid[38]; size_t size; - bool status; -} glusterBlockDef; -typedef glusterBlockDef *glusterBlockDefPtr; + size_t mpath; + char entry[16]; + + size_t nhosts; + NodeInfo **list; +} MetaInfo; + +static const char *const MetakeyLookup[] = { + [GBID] = "GBID", + [SIZE] = "SIZE", + [HA] = "HA", + [ENTRYCREATE] = "ENTRYCREATE", + [METAKEY__MAX] = NULL, +}; + + +struct glfs * +glusterBlockVolumeInit(char *volume, char *volfileserver); + +int +glusterBlockCreateEntry(blockCreateCli *blk, char *gbid); + +int +glusterBlockDeleteEntry(blockCreate *blk); +struct glfs_fd * +glusterBlockCreateMetaLockFile(struct glfs *glfs); -int glusterBlockCreateEntry(blockCreateCli *blk, char *gbid); +void +blockGetMetaInfo(struct glfs *glfs, char *metafile, MetaInfo *info); -int glusterBlockDeleteEntry(blockCreate *blk); +void +blockFreeMetaInfo(MetaInfo *info); #endif /* _GLFS_OPERATIONS_H */ diff --git a/gluster-block.c b/gluster-block.c index ef2ba1b..b8e64b9 100644 --- a/gluster-block.c +++ b/gluster-block.c @@ -23,6 +23,7 @@ # define INFO "info" # define MODIFY "modify" # define BLOCKHOST "block-host" +# define VOLUME "volume" # define HELP "help" @@ -112,7 +113,6 @@ glusterBlockHelp(void) MSG("%s", "gluster-block (Version 0.1) \n" " -c, --create <name> Create the gluster block\n" - " -v, --volume <vol> gluster volume name\n" " -h, --host <gluster-node> node addr from gluster pool\n" " -s, --size <size> block storage size in KiB|MiB|GiB|TiB..\n" "\n" @@ -124,7 +124,8 @@ glusterBlockHelp(void) "\n" " -d, --delete <name> Delete the gluster block\n" "\n" - " [-b, --block-host <IP1,IP2,IP3...>] block servers, clubbed with any option\n"); + " -v, --volume <vol> gluster volume name\n" + " [-b, --block-host <IP1,IP2,IP3...>] block servers, clubbed with any option\n"); } @@ -230,12 +231,13 @@ glusterBlockCreate(int count, char **options, char *name) static int -glusterBlockList(char *blkServers) +glusterBlockList(char *volume, char *blkServers) { static blockListCli cobj; char *out = NULL; int ret = -1; + strcpy(cobj.volume, volume); if (GB_STRDUP(cobj.block_hosts, blkServers) < 0) { return -1; } @@ -251,13 +253,14 @@ glusterBlockList(char *blkServers) static int -glusterBlockDelete(char* name, char *blkServers) +glusterBlockDelete(char* name, char* volume, char *blkServers) { static blockDeleteCli cobj; char *out = NULL; int ret = -1; strcpy(cobj.block_name, name); + strcpy(cobj.volume, volume); if (GB_STRDUP(cobj.block_hosts, blkServers) < 0) { return -1; } @@ -273,13 +276,14 @@ glusterBlockDelete(char* name, char *blkServers) static int -glusterBlockInfo(char* name, char *blkServers) +glusterBlockInfo(char* name, char* volume, char *blkServers) { static blockInfoCli cobj; char *out = NULL; int ret = -1; strcpy(cobj.block_name, name); + strcpy(cobj.volume, volume); if (GB_STRDUP(cobj.block_hosts, blkServers) < 0) { return -1; } @@ -302,6 +306,7 @@ glusterBlockParseArgs(int count, char **options) int optFlag = 0; char *block = NULL; char *blkServers = NULL; + char *volume = NULL; while (1) { static const struct option long_options[] = { @@ -311,6 +316,7 @@ glusterBlockParseArgs(int count, char **options) {LIST, no_argument, 0, 'l'}, {INFO, required_argument, 0, 'i'}, {MODIFY, required_argument, 0, 'm'}, + {VOLUME, required_argument, 0, 'v'}, {BLOCKHOST, required_argument, 0, 'b'}, {0, 0, 0, 0} }; @@ -332,9 +338,13 @@ glusterBlockParseArgs(int count, char **options) goto opt; break; + case 'v': + volume = optarg; + break; + case 'c': ret = glusterBlockCreate(count, options, optarg); - if (ret) { + if (ret && ret != EEXIST) { ERROR("%s", FAILED_CREATE); goto out; } @@ -368,17 +378,17 @@ glusterBlockParseArgs(int count, char **options) opt: switch (optFlag) { case 'l': - ret = glusterBlockList(blkServers); + ret = glusterBlockList(volume, blkServers); if (ret) ERROR("%s", FAILED_LIST); break; case 'i': - ret = glusterBlockInfo(block, blkServers); + ret = glusterBlockInfo(block, volume, blkServers); if (ret) ERROR("%s", FAILED_INFO); break; case 'd': - ret = glusterBlockDelete(block, blkServers); + ret = glusterBlockDelete(block, volume, blkServers); if (ret) ERROR("%s", FAILED_DELETE); break; diff --git a/gluster-blockd.c b/gluster-blockd.c index 0cef7da..db45b9c 100644 --- a/gluster-blockd.c +++ b/gluster-blockd.c @@ -20,7 +20,7 @@ # include "glfs-operations.h" -# define UUID_BUF_SIZE 256 +# define UUID_BUF_SIZE 38 # define CREATE "create" # define LIST "list" @@ -39,6 +39,7 @@ # define MSERVER_DELIMITER "," + typedef struct blockServerDef { size_t nhosts; char **hosts; @@ -212,7 +213,7 @@ getCfgstring(char* name, char *blkServer) blockResponse * block_create_cli_1_svc(blockCreateCli *blk, struct svc_req *rqstp) { - int ret; + int ret = -1; size_t i = 0; char *out = NULL; char savereply[8096] = {0,}; @@ -221,10 +222,49 @@ block_create_cli_1_svc(blockCreateCli *blk, struct svc_req *rqstp) static blockResponse *reply = NULL; blockServerDefPtr list = NULL; char *gbid = CALLOC(UUID_BUF_SIZE); + struct glfs *glfs = NULL; + struct glfs_fd *lkfd; + struct glfs_fd *tgfd = NULL; + struct flock lock = {0, }; + char *write = NULL; + + glfs = glusterBlockVolumeInit(blk->volume, blk->volfileserver); + if (!glfs) { + ERROR("%s", "glusterBlockVolumeInit failed"); + goto out; + } + + lkfd = glusterBlockCreateMetaLockFile(glfs); + if (!lkfd) { + ERROR("%s", "glusterBlockCreateMetaLockFile failed"); + goto out; + } + + METALOCK(lock, lkfd); uuid_generate(uuid); uuid_unparse(uuid, gbid); + if(GB_ALLOC(reply) < 0) + goto out; + + + if (!glfs_access(glfs, blk->block_name, F_OK)) { + GB_STRDUP(reply->out, "BLOCK Already EXIST"); + reply->exit = EEXIST; + goto out; + } + + tgfd = glfs_creat(glfs, blk->block_name, O_RDWR, S_IRUSR | S_IWUSR); + if (!tgfd) { + ERROR("%s", "glfs_creat: failed"); + goto out; + } + + METAUPDATE(tgfd, write, + "GBID: %s\nSIZE: %zu\nHA: %d\nENTRYCREATE: INPROGRESS\n", + gbid, blk->size, 1); + ret = glusterBlockCreateEntry(blk, gbid); if (ret) { ERROR("%s volume: %s host: %s", @@ -232,6 +272,8 @@ block_create_cli_1_svc(blockCreateCli *blk, struct svc_req *rqstp) goto out; } + METAUPDATE(tgfd, write, "ENTRYCREATE: SUCCESS\n"); + if(GB_ALLOC(cobj) < 0) goto out; @@ -244,24 +286,35 @@ block_create_cli_1_svc(blockCreateCli *blk, struct svc_req *rqstp) list = blockServerParse(blk->block_hosts); for (i = 0; i < list->nhosts; i++) { + METAUPDATE(tgfd, write, "%s: INPROGRESS\n", list->hosts[i]); + ret = gluster_block_1(list->hosts[i], cobj, CREATE_SRV, &out); if (ret) { + METAUPDATE(tgfd, write, "%s: FAIL\n", list->hosts[i]); ERROR("%s on host: %s", FAILED_CREATE, list->hosts[i]); - goto out; } + + METAUPDATE(tgfd, write, "%s: SUCCESS\n", list->hosts[i]); + strcpy(savereply, out); GB_FREE(out); } - if(GB_ALLOC(reply) < 0) - goto out; - if (GB_STRDUP(reply->out, savereply) < 0) goto out; reply->exit = ret; out: + if (glfs_close(tgfd) != 0) + ERROR("%s", "glfs_close: failed"); + + METAUNLOCK(lock, lkfd); + + if (glfs_close(lkfd) != 0) + ERROR("%s", "glfs_close: failed"); + + glfs_fini(glfs); blockServerDefFree(list); GB_FREE(cobj); @@ -400,11 +453,38 @@ block_delete_cli_1_svc(blockDeleteCli *blk, struct svc_req *rqstp) static blockCreate *blkcfg; static blockDelete *cobj; static blockResponse *reply = NULL; + struct glfs *glfs = NULL; + struct glfs_fd *lkfd; + struct glfs_fd *tgfd; + struct flock lock = {0, }; + char *write; + + glfs = glusterBlockVolumeInit(blk->volume, "localhost"); + if (!glfs) { + ERROR("%s", "glusterBlockVolumeInit failed"); + goto out; + } - if(GB_ALLOC(cobj) < 0) + lkfd = glusterBlockCreateMetaLockFile(glfs); + if (!lkfd) { + ERROR("%s", "glusterBlockCreateMetaLockFile failed"); goto out; + } - strcpy(cobj->block_name, blk->block_name); + METALOCK(lock, lkfd); + + if (glfs_access(glfs, blk->block_name, F_OK)) { + GB_STRDUP(reply->out, "BLOCK Doesn't EXIST"); + reply->exit = ENOENT; + goto out; + } + + tgfd = glfs_open(glfs, blk->block_name, O_RDWR); + if (!tgfd) { + ERROR("%s", "glfs_open: failed"); + goto out; + } + glfs_lseek (tgfd, 0, SEEK_END); //for cfgstring = getCfgstring(blk->block_name, blk->block_hosts); @@ -421,16 +501,25 @@ block_delete_cli_1_svc(blockDeleteCli *blk, struct svc_req *rqstp) goto out; } + if(GB_ALLOC(cobj) < 0) + goto out; + + strcpy(cobj->block_name, blk->block_name); + + strcpy(cobj->gbid, blkcfg->gbid); list = blockServerParse(blk->block_hosts); for (i = 0; i < list->nhosts; i++) { + METAUPDATE(tgfd, write, "%s: CLEANUPINPROGRES\n", list->hosts[i]); ret = gluster_block_1(list->hosts[i], cobj, DELETE_SRV, &out); if (ret) { + METAUPDATE(tgfd, write, "%s: CLEANUPFAIL\n", list->hosts[i]); ERROR("%s on host: %s", FAILED_GATHERING_INFO, list->hosts[i]); goto out; } + METAUPDATE(tgfd, write, "%s: CLEANUPSUCCESS\n", list->hosts[i]); /* TODO: aggrigate the result */ strcpy(savereply, out); GB_FREE(out); @@ -449,6 +538,22 @@ block_delete_cli_1_svc(blockDeleteCli *blk, struct svc_req *rqstp) } out: + if (glfs_close(tgfd) != 0) + ERROR("%s", "glfs_close: failed"); + + ret = glfs_unlink(glfs, blk->block_name); + if (ret && errno != ENOENT) { + ERROR("%s", "glfs_unlink: failed"); + goto out; + } + + METAUNLOCK(lock, lkfd); + + if (glfs_close(lkfd) != 0) + ERROR("%s", "glfs_close: failed"); + + glfs_fini(glfs); + blockServerDefFree(list); GB_FREE(cfgstring); GB_FREE(blkcfg); @@ -541,6 +646,23 @@ block_list_cli_1_svc(blockListCli *blk, struct svc_req *rqstp) int ret = -1; char savereply[8096] = {0,}; blockServerDefPtr list = NULL; + struct glfs *glfs; + struct glfs_fd *lkfd; + struct flock lock = {0, }; + + glfs = glusterBlockVolumeInit(blk->volume, "localhost"); + if (!glfs) { + ERROR("%s", "glusterBlockVolumeInit failed"); + goto out; + } + + lkfd = glusterBlockCreateMetaLockFile(glfs); + if (!lkfd) { + ERROR("%s", "glusterBlockCreateMetaLockFile failed"); + goto out; + } + + METALOCK(lock, lkfd); asprintf(&cmd, "%s %s", TARGETCLI_GLFS, LUNS_LIST); @@ -566,6 +688,14 @@ block_list_cli_1_svc(blockListCli *blk, struct svc_req *rqstp) reply->exit = ret; out: + + METAUNLOCK(lock, lkfd); + + if (glfs_close(lkfd) != 0) + ERROR("%s", "glfs_close: failed"); + + glfs_fini(glfs); + blockServerDefFree(list); GB_FREE(cmd); @@ -584,6 +714,24 @@ block_info_cli_1_svc(blockInfoCli *blk, struct svc_req *rqstp) char savereply[8096] = {0,}; blockServerDefPtr list = NULL; + struct glfs *glfs; + struct glfs_fd *lkfd; + struct flock lock = {0, }; + + glfs = glusterBlockVolumeInit(blk->volume, "localhost"); + if (!glfs) { + ERROR("%s", "glusterBlockVolumeInit failed"); + goto out; + } + + lkfd = glusterBlockCreateMetaLockFile(glfs); + if (!lkfd) { + ERROR("%s", "glusterBlockCreateMetaLockFile failed"); + goto out; + } + + METALOCK(lock, lkfd); + asprintf(&cmd, "%s/%s %s", TARGETCLI_GLFS, blk->block_name, INFO); //for @@ -609,6 +757,13 @@ block_info_cli_1_svc(blockInfoCli *blk, struct svc_req *rqstp) reply->exit = ret; out: + METAUNLOCK(lock, lkfd); + + if (glfs_close(lkfd) != 0) + ERROR("%s", "glfs_close: failed"); + + glfs_fini(glfs); + blockServerDefFree(list); return reply; diff --git a/rpc/block.h b/rpc/block.h index 8cd38fb..132cf63 100644 --- a/rpc/block.h +++ b/rpc/block.h @@ -35,6 +35,7 @@ typedef struct blockCreateCli blockCreateCli; struct blockDeleteCli { char block_name[255]; + char volume[255]; char *block_hosts; }; typedef struct blockDeleteCli blockDeleteCli; diff --git a/rpc/block.x b/rpc/block.x index b90aa45..e5c1984 100644 --- a/rpc/block.x +++ b/rpc/block.x @@ -16,6 +16,7 @@ struct blockCreateCli { struct blockDeleteCli { char block_name[255]; + char volume[255]; string block_hosts<>; }; diff --git a/rpc/block_xdr.c b/rpc/block_xdr.c index c0a5666..3bc0afc 100644 --- a/rpc/block_xdr.c +++ b/rpc/block_xdr.c @@ -50,6 +50,9 @@ xdr_blockDeleteCli (XDR *xdrs, blockDeleteCli *objp) if (!xdr_vector (xdrs, (char *)objp->block_name, 255, sizeof (char), (xdrproc_t) xdr_char)) return FALSE; + if (!xdr_vector (xdrs, (char *)objp->volume, 255, + sizeof (char), (xdrproc_t) xdr_char)) + return FALSE; if (!xdr_string (xdrs, &objp->block_hosts, ~0)) return FALSE; return TRUE; @@ -54,6 +54,30 @@ fprintf(stdout, fmt "\n", __VA_ARGS__) +# define METALOCK(a, b) {\ + memset (&a, 0, sizeof(a)); \ + a.l_type = F_WRLCK; \ + if (glfs_posix_lock (b, F_SETLKW, &a)) {\ + ERROR("%s", "glfs_posix_lock: failed");\ + goto out;\ + }\ + } + +# define METAUPDATE(a, b, ...) {\ + asprintf(&b, __VA_ARGS__);\ + if(glfs_write (a, b, strlen(b), 0) < 0) {\ + ERROR("%s", "glfs_write: failed");\ + goto out;\ + }\ + GB_FREE(b); \ + } + +# define METAUNLOCK(a, b) {\ + a.l_type = F_UNLCK; \ + glfs_posix_lock(b, F_SETLKW, &a); \ + } + + # define CALLOC(x) calloc(1, x) # define GB_ALLOC_N(ptr, count) gbAllocN(&(ptr), sizeof(*(ptr)), (count), \ |