diff options
author | Prasanna Kumar Kalever <prasanna.kalever@redhat.com> | 2017-01-20 12:13:37 +0530 |
---|---|---|
committer | Prasanna Kumar Kalever <prasanna.kalever@redhat.com> | 2017-01-30 19:31:50 +0530 |
commit | 683b6ba3f17a9bbf876c66f5d4b7a9d573d8853f (patch) | |
tree | a2bf7cbcda416818d663613284b29e16b2b533c3 /rpc | |
parent | 4ef14866295c1328a11f9a06cfc3bc1db9b1e7b6 (diff) |
gluster-block: listen on unix and inet
from now We basically have 2 RPC connections,
1. Between gluster block CLI and local gluster-blockd
This connection is basically UNIX/local netid ,listening on
/var/run/gluster-blockd.socket file.
The CLI always Send/Receive the commands to/from the local
gluster-blockd via local rpc.
2. Between gluster-blockd's, i.e local (to cli) gluster-blockd and the
gluster-blockd's running on remote(blockhost)
This is the tcp connection. The rpc requests are listening on 24006
Also from now gluster-blockd is multi threaded (As of now 2 threads)
Lets consider the Create Request to understand what each thread solves
Thread1 (THE CLI THREAD)
* Listening on local RPC
* Generate the GBID (UUID) and create the entry with name GBID in the
given volume with a requested size.
* And Send the Configuration requests to remote hosts,
waits for the replies
(HINt: after this point Read Thread2 and come back)
* Return to CLI.
Thread 2 (THE SERVER THREAD)
* Listens on 24006
* On Receiving an event, read the structure.
* Executes the required "targetcli bla bla bla" command locally
* Fills the command exitcode and the output in the RPC reply structure
and send reply
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Diffstat (limited to 'rpc')
-rw-r--r-- | rpc/block.h | 64 | ||||
-rw-r--r-- | rpc/block.x | 31 | ||||
-rw-r--r-- | rpc/block_clnt.c | 27 | ||||
-rw-r--r-- | rpc/block_svc.c | 159 | ||||
-rw-r--r-- | rpc/block_xdr.c | 45 |
5 files changed, 275 insertions, 51 deletions
diff --git a/rpc/block.h b/rpc/block.h index 54315b7..d939057 100644 --- a/rpc/block.h +++ b/rpc/block.h @@ -8,41 +8,85 @@ #include <rpc/rpc.h> +#define ADDRESS "/var/run/gluster-block.socket" #ifdef __cplusplus extern "C" { #endif -struct blockTrans { +struct blockCreate { + char volume[255]; + char volfileserver[255]; + char gbid[127]; + u_quad_t size; + char block_name[255]; +}; +typedef struct blockCreate blockCreate; + +struct blockCreateCli { + char volume[255]; + char volfileserver[255]; + u_quad_t size; + char block_name[255]; + char *block_hosts; +}; +typedef struct blockCreateCli blockCreateCli; + +struct blockResponse { int exit; char *out; + u_quad_t offset; + struct { + u_int xdata_len; + char *xdata_val; + } xdata; }; -typedef struct blockTrans blockTrans; +typedef struct blockResponse blockResponse; + +#define GLUSTER_BLOCK_CLI 212153113 +#define GLUSTER_BLOCK_CLI_VERS 1 + +#if defined(__STDC__) || defined(__cplusplus) +#define BLOCK_CREATE_CLI 1 +extern blockResponse * block_create_cli_1(blockCreateCli *, CLIENT *); +extern blockResponse * block_create_cli_1_svc(blockCreateCli *, struct svc_req *); +extern int gluster_block_cli_1_freeresult (SVCXPRT *, xdrproc_t, caddr_t); + +#else /* K&R C */ +#define BLOCK_CREATE_CLI 1 +extern blockResponse * block_create_cli_1(); +extern blockResponse * block_create_cli_1_svc(); +extern int gluster_block_cli_1_freeresult (); +#endif /* K&R C */ #define GLUSTER_BLOCK 21215311 #define GLUSTER_BLOCK_VERS 1 #if defined(__STDC__) || defined(__cplusplus) -#define BLOCK_EXEC 1 -extern blockTrans * block_exec_1(char **, CLIENT *); -extern blockTrans * block_exec_1_svc(char **, struct svc_req *); +#define BLOCK_CREATE 1 +extern blockResponse * block_create_1(blockCreate *, CLIENT *); +extern blockResponse * block_create_1_svc(blockCreate *, struct svc_req *); extern int gluster_block_1_freeresult (SVCXPRT *, xdrproc_t, caddr_t); #else /* K&R C */ -#define BLOCK_EXEC 1 -extern blockTrans * block_exec_1(); -extern blockTrans * block_exec_1_svc(); +#define BLOCK_CREATE 1 +extern blockResponse * block_create_1(); +extern blockResponse * block_create_1_svc(); extern int gluster_block_1_freeresult (); #endif /* K&R C */ /* the xdr functions */ #if defined(__STDC__) || defined(__cplusplus) -extern bool_t xdr_blockTrans (XDR *, blockTrans*); +extern bool_t xdr_blockCreate (XDR *, blockCreate*); +extern bool_t xdr_blockCreateCli (XDR *, blockCreateCli*); +extern bool_t xdr_blockResponse (XDR *, blockResponse*); #else /* K&R C */ -extern bool_t xdr_blockTrans (); +extern bool_t xdr_blockCreate (); +extern bool_t xdr_blockCreateCli (); +extern bool_t xdr_blockResponse (); #endif /* K&R C */ diff --git a/rpc/block.x b/rpc/block.x index c88fdf2..b563c9e 100644 --- a/rpc/block.x +++ b/rpc/block.x @@ -1,12 +1,35 @@ +struct blockCreate { + char volume[255]; + char volfileserver[255]; + char gbid[127]; /* uuid */ + u_quad_t size; + char block_name[255]; +}; + +struct blockCreateCli { + char volume[255]; + char volfileserver[255]; + u_quad_t size; + char block_name[255]; + string block_hosts<>; +}; -struct blockTrans { - int exit; /* exit code of the command */ - string out<>; /* stdout of the command */ +struct blockResponse { + int exit; /* exit code of the command */ + string out<>; /* json output */ + u_quad_t offset; /* dentry d_name offset */ + opaque xdata<>; /* future reserve */ }; +program GLUSTER_BLOCK_CLI { + version GLUSTER_BLOCK_CLI_VERS { + blockResponse BLOCK_CREATE_CLI(blockCreateCli) = 1; + } = 1; +} = 212153113; /* B2 L12 O15 C3 K11 C3 */ + program GLUSTER_BLOCK { version GLUSTER_BLOCK_VERS { - blockTrans BLOCK_EXEC(string) = 1; + blockResponse BLOCK_CREATE(blockCreate) = 1; } = 1; } = 21215311; /* B2 L12 O15 C3 K11 */ diff --git a/rpc/block_clnt.c b/rpc/block_clnt.c index 9e5b585..ae95d9a 100644 --- a/rpc/block_clnt.c +++ b/rpc/block_clnt.c @@ -9,15 +9,30 @@ /* Default timeout can be changed using clnt_control() */ static struct timeval TIMEOUT = { 25, 0 }; -blockTrans * -block_exec_1(char **argp, CLIENT *clnt) +blockResponse * +block_create_cli_1(blockCreateCli *argp, CLIENT *clnt) { - static blockTrans clnt_res; + static blockResponse clnt_res; memset((char *)&clnt_res, 0, sizeof(clnt_res)); - if (clnt_call (clnt, BLOCK_EXEC, - (xdrproc_t) xdr_wrapstring, (caddr_t) argp, - (xdrproc_t) xdr_blockTrans, (caddr_t) &clnt_res, + if (clnt_call (clnt, BLOCK_CREATE_CLI, + (xdrproc_t) xdr_blockCreateCli, (caddr_t) argp, + (xdrproc_t) xdr_blockResponse, (caddr_t) &clnt_res, + TIMEOUT) != RPC_SUCCESS) { + return (NULL); + } + return (&clnt_res); +} + +blockResponse * +block_create_1(blockCreate *argp, CLIENT *clnt) +{ + static blockResponse clnt_res; + + memset((char *)&clnt_res, 0, sizeof(clnt_res)); + if (clnt_call (clnt, BLOCK_CREATE, + (xdrproc_t) xdr_blockCreate, (caddr_t) argp, + (xdrproc_t) xdr_blockResponse, (caddr_t) &clnt_res, TIMEOUT) != RPC_SUCCESS) { return (NULL); } diff --git a/rpc/block_svc.c b/rpc/block_svc.c index 144b7e1..4c054f5 100644 --- a/rpc/block_svc.c +++ b/rpc/block_svc.c @@ -6,6 +6,8 @@ #include "block.h" #include <stdio.h> #include <stdlib.h> +#include <unistd.h> +#include <pthread.h> #include <rpc/pmap_clnt.h> #include <string.h> #include <memory.h> @@ -16,11 +18,16 @@ #define SIG_PF void(*)(int) #endif + + +pthread_t p_thread1, p_thread2; + + static void -gluster_block_1(struct svc_req *rqstp, register SVCXPRT *transp) +gluster_block_cli_1(struct svc_req *rqstp, register SVCXPRT *transp) { union { - char *block_exec_1_arg; + blockCreateCli block_create_cli_1_arg; } argument; char *result; xdrproc_t _xdr_argument, _xdr_result; @@ -31,10 +38,10 @@ gluster_block_1(struct svc_req *rqstp, register SVCXPRT *transp) (void) svc_sendreply (transp, (xdrproc_t) xdr_void, (char *)NULL); return; - case BLOCK_EXEC: - _xdr_argument = (xdrproc_t) xdr_wrapstring; - _xdr_result = (xdrproc_t) xdr_blockTrans; - local = (char *(*)(char *, struct svc_req *)) block_exec_1_svc; + case BLOCK_CREATE_CLI: + _xdr_argument = (xdrproc_t) xdr_blockCreateCli; + _xdr_result = (xdrproc_t) xdr_blockResponse; + local = (char *(*)(char *, struct svc_req *)) block_create_cli_1_svc; break; default: @@ -57,46 +64,138 @@ gluster_block_1(struct svc_req *rqstp, register SVCXPRT *transp) return; } -int -main (int argc, char **argv) +static void +gluster_block_1(struct svc_req *rqstp, register SVCXPRT *transp) { - register SVCXPRT *transp; - struct sockaddr_in addr; - int sockfd; + union { + blockCreate block_create_1_arg; + } argument; + char *result; + xdrproc_t _xdr_argument, _xdr_result; + char *(*local)(char *, struct svc_req *); - pmap_unset (GLUSTER_BLOCK, GLUSTER_BLOCK_VERS); + switch (rqstp->rq_proc) { + case NULLPROC: + (void) svc_sendreply (transp, (xdrproc_t) xdr_void, (char *)NULL); + return; - if ((sockfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { - fprintf (stderr, "%s", "socket failed"); - } + case BLOCK_CREATE: + _xdr_argument = (xdrproc_t) xdr_blockCreate; + _xdr_result = (xdrproc_t) xdr_blockResponse; + local = (char *(*)(char *, struct svc_req *)) block_create_1_svc; + break; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = htons(24009); + default: + svcerr_noproc (transp); + return; + } + memset ((char *)&argument, 0, sizeof (argument)); + if (!svc_getargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) { + svcerr_decode (transp); + return; + } + result = (*local)((char *)&argument, rqstp); + if (result != NULL && !svc_sendreply(transp, (xdrproc_t) _xdr_result, result)) { + svcerr_systemerr (transp); + } + if (!svc_freeargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) { + fprintf (stderr, "%s", "unable to free arguments"); + exit (1); + } + return; +} + +void * +cli_thread(void *vargp) +{ + register SVCXPRT *transp; + struct sockaddr_un saun; + int sockfd, len; - if (bind(sockfd, (struct sockaddr *)&addr, sizeof (addr)) < 0) { - fprintf (stderr, "%s", "bind failed"); + if ((sockfd = socket(AF_UNIX, SOCK_STREAM, IPPROTO_IP)) < 0) { + perror("server: socket"); + exit(1); } - if (listen(sockfd, 10) < 0) { - fprintf (stderr, "%s", "server: listen failed"); + saun.sun_family = AF_UNIX; + strcpy(saun.sun_path, ADDRESS); + + unlink(ADDRESS); + len = sizeof(saun.sun_family) + strlen(saun.sun_path); + + if (bind(sockfd, (struct sockaddr *) &saun, len) < 0) { + perror("server: bind"); exit(1); } - transp = svctcp_create(sockfd, 0, 0); - if (transp == NULL) { - fprintf (stderr, "%s", "cannot create tcp service"); + transp = svcunix_create(sockfd, 0, 0, ADDRESS); + if (transp == NULL) { + fprintf (stderr, "%s", "cannot create tcp service"); + exit(1); + } + + if (!svc_register(transp, GLUSTER_BLOCK_CLI, GLUSTER_BLOCK_CLI_VERS, gluster_block_cli_1, IPPROTO_IP)) { + fprintf (stderr, "%s", "unable to register (GLUSTER_BLOCK_CLI, GLUSTER_BLOCK_CLI_VERS, unix|local)."); exit(1); } + svc_run (); + + return NULL; +} + +void * +server_thread(void *vargp) +{ + register SVCXPRT *transp; + struct sockaddr_in sain; + int sockfd; + + if ((sockfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + perror("server: socket"); + exit(1); + } + + sain.sin_family = AF_INET; + sain.sin_addr.s_addr = INADDR_ANY; + sain.sin_port = htons(24006); + + if (bind(sockfd, (struct sockaddr *) &sain, sizeof (sain)) < 0) { + perror("server: bind"); + exit(1); + } + + transp = svctcp_create(sockfd, 0, 0); + if (transp == NULL) { + fprintf (stderr, "%s", "cannot create tcp service"); + exit(1); + } + if (!svc_register(transp, GLUSTER_BLOCK, GLUSTER_BLOCK_VERS, gluster_block_1, IPPROTO_TCP)) { - fprintf (stderr, "%s", "unable to register (BLOCK_PROG, GLUSTER_BLOCK_VERS, tcp)."); + fprintf (stderr, "%s", "unable to register (GLUSTER_BLOCK, GLUSTER_BLOCK_VERS, tcp)."); exit(1); } - svc_run (); + svc_run (); + + return NULL; +} + + +int +main (int argc, char **argv) +{ + pmap_unset (GLUSTER_BLOCK_CLI, GLUSTER_BLOCK_CLI_VERS); + pmap_unset (GLUSTER_BLOCK, GLUSTER_BLOCK_VERS); + + pthread_create(&p_thread1, NULL, cli_thread, NULL); + pthread_create(&p_thread2, NULL, server_thread, NULL); + + pthread_join(p_thread1, NULL); + pthread_join(p_thread2, NULL); + - fprintf (stderr, "%s", "svc_run returned"); - exit (1); - /* NOTREACHED */ + fprintf (stderr, "%s", "svc_run returned"); + exit (0); + /* NOTREACHED */ } diff --git a/rpc/block_xdr.c b/rpc/block_xdr.c index cd8400e..91775fa 100644 --- a/rpc/block_xdr.c +++ b/rpc/block_xdr.c @@ -6,11 +6,54 @@ #include "block.h" bool_t -xdr_blockTrans (XDR *xdrs, blockTrans *objp) +xdr_blockCreate (XDR *xdrs, blockCreate *objp) +{ + if (!xdr_vector (xdrs, (char *)objp->volume, 255, + sizeof (char), (xdrproc_t) xdr_char)) + return FALSE; + if (!xdr_vector (xdrs, (char *)objp->volfileserver, 255, + sizeof (char), (xdrproc_t) xdr_char)) + return FALSE; + if (!xdr_vector (xdrs, (char *)objp->gbid, 127, + sizeof (char), (xdrproc_t) xdr_char)) + return FALSE; + if (!xdr_u_quad_t (xdrs, &objp->size)) + return FALSE; + if (!xdr_vector (xdrs, (char *)objp->block_name, 255, + sizeof (char), (xdrproc_t) xdr_char)) + return FALSE; + return TRUE; +} + +bool_t +xdr_blockCreateCli (XDR *xdrs, blockCreateCli *objp) +{ + if (!xdr_vector (xdrs, (char *)objp->volume, 255, + sizeof (char), (xdrproc_t) xdr_char)) + return FALSE; + if (!xdr_vector (xdrs, (char *)objp->volfileserver, 255, + sizeof (char), (xdrproc_t) xdr_char)) + return FALSE; + if (!xdr_u_quad_t (xdrs, &objp->size)) + return FALSE; + if (!xdr_vector (xdrs, (char *)objp->block_name, 255, + sizeof (char), (xdrproc_t) xdr_char)) + return FALSE; + if (!xdr_string (xdrs, &objp->block_hosts, ~0)) + return FALSE; + return TRUE; +} + +bool_t +xdr_blockResponse (XDR *xdrs, blockResponse *objp) { if (!xdr_int (xdrs, &objp->exit)) return FALSE; if (!xdr_string (xdrs, &objp->out, ~0)) return FALSE; + if (!xdr_u_quad_t (xdrs, &objp->offset)) + return FALSE; + if (!xdr_bytes (xdrs, (char **)&objp->xdata.xdata_val, (u_int *) &objp->xdata.xdata_len, ~0)) + return FALSE; return TRUE; } |