cli,glusterd: more volume status improvements

The major changes are,
* "volume status" now supports getting details of the self-heal daemon processes
for replica volumes. A new cli options "shd", similar to "nfs", has been
introduced for this. "detail", "fd" and "clients" status ops are not supported
for self-heal daemons.
* The default/normal ouput of "volume status" has been enhanced to contain
information about nfs-server and self-heal daemon processes as well. Some tweaks
have been done to the cli output to show appropriate output.

Also, changes have been done to rebalance/remove-brick status, so that hostnames
are displayed instead of uuids.

Change-Id: I3972396dcf72d45e14837fa5f9c7d62410901df8
BUG: 803676
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.com/3016
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kp@gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

4 files changed, 187 insertions, 76 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 4c86ab488c5..ba9ca63d849 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1905,6 +1905,8 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
                         if (cmd == GF_CLI_STATUS_NONE) {
                                 if (!strcmp (words[3], "nfs")) {
                                         cmd |= GF_CLI_STATUS_NFS;
+                                } else if (!strcmp (words[3], "shd")) {
+                                        cmd |= GF_CLI_STATUS_SHD;
                                 } else {
                                         cmd = GF_CLI_STATUS_BRICK;
                                         ret = dict_set_str (dict, "brick",
@@ -1940,12 +1942,24 @@ cli_cmd_volume_status_parse (const char **words, int wordcount,
                         goto out;
 
                 if (!strcmp (words[3], "nfs")) {
-                        if (cmd == GF_CLI_STATUS_FD) {
-                                cli_out ("FD status not available for NFS");
+                        if (cmd == GF_CLI_STATUS_FD ||
+                            cmd == GF_CLI_STATUS_DETAIL) {
+                                cli_out ("Detail/FD status not available"
+                                         " for NFS Servers");
                                 ret = -1;
                                 goto out;
                         }
                         cmd |= GF_CLI_STATUS_NFS;
+                } else if (!strcmp (words[3], "shd")){
+                        if (cmd == GF_CLI_STATUS_FD ||
+                            cmd == GF_CLI_STATUS_CLIENTS ||
+                            cmd == GF_CLI_STATUS_DETAIL) {
+                                cli_out ("Detail/FD/Clients status not "
+                                         "available for Self-heal Daemons");
+                                ret = -1;
+                                goto out;
+                        }
+                        cmd |= GF_CLI_STATUS_SHD;
                 } else {
                         cmd |= GF_CLI_STATUS_BRICK;
                         ret = dict_set_str (dict, "brick", (char *)words[3]);
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index aca5b1b3210..e045964a7b9 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1559,9 +1559,14 @@ cli_print_brick_status (cli_volume_status_t *status)
                         printf ("%s", p);
                         while (num_tabs-- != 0)
                                 printf ("\t");
-                        cli_out ("%d\t%c\t%s",
-                                 status->port, status->online?'Y':'N',
-                                 status->pid_str);
+                        if (status->port)
+                                cli_out ("%d\t%c\t%s",
+                                         status->port, status->online?'Y':'N',
+                                         status->pid_str);
+                        else
+                                cli_out ("%s\t%c\t%s",
+                                         "N/A", status->online?'Y':'N',
+                                         status->pid_str);
                         bricklen = 0;
                 }
         }
@@ -1838,7 +1843,7 @@ struct cli_cmd volume_cmds[] = {
            cli_cmd_volume_top_cbk,
            "volume top operations"},
 
-        { "volume status [all | <VOLNAME> [nfs|<BRICK>]]"
+        { "volume status [all | <VOLNAME> [nfs|shd|<BRICK>]]"
           " [detail|clients|mem|inode|fd|callpool]",
           cli_cmd_volume_status_cbk,
           "display status of all or specified volume(s)/brick"},
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index d75d095b555..6deda9c0007 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -4308,7 +4308,7 @@ out:
 }
 
 void
-cli_print_volume_status_mem (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_mem (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
@@ -4316,7 +4316,9 @@ cli_print_volume_status_mem (dict_t *dict, gf_boolean_t nfs)
         char            *path = NULL;
         int             online = -1;
         char            key[1024] = {0,};
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         int             val = 0;
         int             i = 0;
 
@@ -4327,24 +4329,29 @@ cli_print_volume_status_mem (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("Memory status for volume : %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.hostname", i);
                 ret = dict_get_str (dict, key, &hostname);
                 if (ret)
-                        goto out;
+                        continue;
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.path", i);
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
-                        goto out;
-                if (nfs)
+                        continue;
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -4355,7 +4362,10 @@ cli_print_volume_status_mem (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4442,11 +4452,13 @@ out:
 }
 
 void
-cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_clients (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4465,11 +4477,16 @@ cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("Client connections for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for ( i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4483,7 +4500,7 @@ cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
 
-                if (nfs)
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -4494,7 +4511,10 @@ cli_print_volume_status_clients (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4690,11 +4710,13 @@ out:
 }
 
 void
-cli_print_volume_status_inode (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_inode (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4710,11 +4732,16 @@ cli_print_volume_status_inode (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("Inode tables for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
         if (ret)
                 goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
+
+        index_max = brick_index_max + other_count;
 
-        for (i = 0; i < brick_count; i++) {
+        for ( i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4727,7 +4754,7 @@ cli_print_volume_status_inode (dict_t *dict, gf_boolean_t nfs)
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
                         goto out;
-                if (nfs)
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -4738,7 +4765,10 @@ cli_print_volume_status_inode (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -4846,11 +4876,13 @@ out:
 }
 
 void
-cli_print_volume_status_fd (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_fd (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -4866,11 +4898,16 @@ cli_print_volume_status_fd (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("FD tables for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -4884,7 +4921,7 @@ cli_print_volume_status_fd (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
 
-                if (nfs)
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -4895,7 +4932,10 @@ cli_print_volume_status_fd (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -5064,11 +5104,13 @@ cli_print_volume_status_call_stack (dict_t *dict, char *prefix)
 }
 
 void
-cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t nfs)
+cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t notbrick)
 {
         int             ret = -1;
         char            *volname = NULL;
-        int             brick_count = 0;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             index_max = 0;
         char            *hostname = NULL;
         char            *path = NULL;
         int             online = -1;
@@ -5084,11 +5126,16 @@ cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t nfs)
                 goto out;
         cli_out ("Pending calls for volume %s", volname);
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
                 cli_out ("----------------------------------------------");
 
                 memset (key, 0, sizeof (key));
@@ -5102,7 +5149,7 @@ cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
 
-                if (nfs)
+                if (notbrick)
                         cli_out ("%s : %s", hostname, path);
                 else
                         cli_out ("Brick : %s:%s", hostname, path);
@@ -5113,7 +5160,10 @@ cli_print_volume_status_callpool (dict_t *dict, gf_boolean_t nfs)
                 if (ret)
                         goto out;
                 if (!online) {
-                        cli_out ("Brick is offline");
+                        if (notbrick)
+                                cli_out ("%s is offline", hostname);
+                        else
+                                cli_out ("Brick is offline");
                         continue;
                 }
 
@@ -5146,18 +5196,21 @@ static int
 gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
                       int count, void *myframe)
 {
-        int                             ret            = -1;
-        int                             i              = 0;
-        int                             pid            = -1;
-        uint32_t                        cmd            = 0;
-        gf_boolean_t                    nfs            = _gf_false;
-        char                            key[1024]      = {0,};
-        char                           *hostname       = NULL;
-        char                           *path           = NULL;
-        char                           *volname        = NULL;
-        dict_t                         *dict           = NULL;
-        gf_cli_rsp                      rsp            = {0,};
-        cli_volume_status_t             status         = {0};
+        int                             ret             = -1;
+        int                             brick_index_max = -1;
+        int                             other_count     = 0;
+        int                             index_max       = 0;
+        int                             i               = 0;
+        int                             pid             = -1;
+        uint32_t                        cmd             = 0;
+        gf_boolean_t                    notbrick        = _gf_false;
+        char                            key[1024]       = {0,};
+        char                           *hostname        = NULL;
+        char                           *path            = NULL;
+        char                           *volname         = NULL;
+        dict_t                         *dict            = NULL;
+        gf_cli_rsp                      rsp             = {0,};
+        cli_volume_status_t             status          = {0};
 
         if (req->rpc_status == -1)
                 goto out;
@@ -5195,8 +5248,8 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-        if (cmd & GF_CLI_STATUS_NFS)
-                nfs = _gf_true;
+        if ((cmd & GF_CLI_STATUS_NFS) || (cmd & GF_CLI_STATUS_SHD))
+                notbrick = _gf_true;
 
         ret = dict_get_int32 (dict, "count", &count);
         if (ret)
@@ -5206,6 +5259,15 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
+
+        index_max = brick_index_max + other_count;
+
 #if (HAVE_LIB_XML)
         if (global_state->mode & GLUSTER_MODE_XML) {
                 ret = cli_xml_output_vol_status (dict, rsp.op_ret,
@@ -5222,23 +5284,23 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
 
         switch (cmd & GF_CLI_STATUS_MASK) {
                 case GF_CLI_STATUS_MEM:
-                        cli_print_volume_status_mem (dict, nfs);
+                        cli_print_volume_status_mem (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_CLIENTS:
-                        cli_print_volume_status_clients (dict, nfs);
+                        cli_print_volume_status_clients (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_INODE:
-                        cli_print_volume_status_inode (dict, nfs);
+                        cli_print_volume_status_inode (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_FD:
-                        cli_print_volume_status_fd (dict, nfs);
+                        cli_print_volume_status_fd (dict, notbrick);
                         goto cont;
                         break;
                 case GF_CLI_STATUS_CALLPOOL:
-                        cli_print_volume_status_callpool (dict, nfs);
+                        cli_print_volume_status_callpool (dict, notbrick);
                         goto cont;
                         break;
                 default:
@@ -5252,46 +5314,54 @@ gf_cli3_1_status_cbk (struct rpc_req *req, struct iovec *iov,
         cli_out ("\nStatus of volume: %s", volname);
 
         if ((cmd & GF_CLI_STATUS_DETAIL) == 0) {
-                cli_out ("Brick\t\t\t\t\t\t\tPort\tOnline\tPid");
+                cli_out ("Gluster process\t\t\t\t\t\tPort\tOnline\tPid");
                 cli_print_line (CLI_BRICK_STATUS_LINE_LEN);
         }
 
-        for (i = 0; i < count; i++) {
+        for (i = 0; i <= index_max; i++) {
 
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.hostname", i);
                 ret = dict_get_str (dict, key, &hostname);
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.path", i);
                 ret = dict_get_str (dict, key, &path);
                 if (ret)
-                        goto out;
+                        continue;
 
+                /* Brick/not-brick is handled seperately here as all
+                 * types of nodes are contained in the default ouput
+                 */
                 memset (status.brick, 0, PATH_MAX + 255);
-                snprintf (status.brick, PATH_MAX + 255, "%s:%s",
-                          hostname, path);
+                if (!strcmp (hostname, "NFS Server") ||
+                    !strcmp (hostname, "Self-heal Daemon"))
+                        snprintf (status.brick, PATH_MAX + 255, "%s on %s",
+                                  hostname, path);
+                else
+                        snprintf (status.brick, PATH_MAX + 255, "Brick %s:%s",
+                                  hostname, path);
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.port", i);
                 ret = dict_get_int32 (dict, key, &(status.port));
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.status", i);
                 ret = dict_get_int32 (dict, key, &(status.online));
                 if (ret)
-                        goto out;
+                        continue;
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "brick%d.pid", i);
                 ret = dict_get_int32 (dict, key, &pid);
                 if (ret)
-                        goto out;
+                        continue;
                 if (pid == -1)
                         ret = gf_asprintf (&(status.pid_str), "%s", "N/A");
                 else
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
index 7f8f9cfc45e..702a7f7feb1 100644
--- a/cli/src/cli-xml-output.c
+++ b/cli/src/cli-xml-output.c
@@ -189,7 +189,8 @@ out:
 
 int
 cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
-                                  int   brick_index, int *online)
+                                  int   brick_index, int *online,
+                                  gf_boolean_t *node_present)
 {
         int             ret = -1;
         char            *hostname = NULL;
@@ -201,8 +202,12 @@ cli_xml_output_vol_status_common (xmlTextWriterPtr writer, dict_t *dict,
 
         snprintf (key, sizeof (key), "brick%d.hostname", brick_index);
         ret = dict_get_str (dict, key, &hostname);
-        if (ret)
+        if (ret) {
+                *node_present = _gf_false;
                 goto out;
+        }
+        *node_present = _gf_true;
+
         ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"hostname",
                                                "%s", hostname);
         XML_RET_CHECK_AND_GOTO (ret, out);
@@ -1282,8 +1287,12 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
         xmlBufferPtr            buf = NULL;
         char                    *volname = NULL;
         int                     brick_count = 0;
+        int                     brick_index_max = -1;
+        int                     other_count = 0;
+        int                     index_max = 0;
         uint32_t                cmd = GF_CLI_STATUS_NONE;
         int                     online = 0;
+        gf_boolean_t            node_present = _gf_true;
         int                     i;
 
         ret = cli_begin_xml_output (&writer, &buf);
@@ -1308,7 +1317,7 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret)
                 goto out;
-        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"brickCount",
+        ret = xmlTextWriterWriteFormatElement (writer, (xmlChar *)"nodeCount",
                                                "%d", brick_count);
         if (ret)
                 goto out;
@@ -1317,15 +1326,28 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
         if (ret)
                 goto out;
 
-        for (i = 0; i < brick_count; i++) {
-                /* <brick> */
-                ret = xmlTextWriterStartElement (writer, (xmlChar *)"brick");
+        ret = dict_get_int32 (dict, "brick-index-max", &brick_index_max);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "other-count", &other_count);
+        if (ret)
+                goto out;
+
+        index_max = brick_index_max + other_count;
+
+        for (i = 0; i <= index_max; i++) {
+                /* <node> */
+                ret = xmlTextWriterStartElement (writer, (xmlChar *)"node");
                 XML_RET_CHECK_AND_GOTO (ret, out);
 
                 ret = cli_xml_output_vol_status_common (writer, dict, i,
-                                                        &online);
-                if (ret)
-                        goto out;
+                                                        &online, &node_present);
+                if (ret) {
+                        if (node_present)
+                                goto out;
+                        else
+                                continue;
+                }
 
                 switch (cmd & GF_CLI_STATUS_MASK) {
                 case GF_CLI_STATUS_DETAIL:
@@ -1384,7 +1406,7 @@ cli_xml_output_vol_status (dict_t *dict, int op_ret, int op_errno,
                         break;
 
                 }
-                /* </brick> */
+                /* </node> */
                 ret = xmlTextWriterEndElement (writer);
                 XML_RET_CHECK_AND_GOTO (ret, out);
         }