From d269087b4f04ef23635581b4b5789ad486c9120c Mon Sep 17 00:00:00 2001 From: Shireesh Anjal Date: Tue, 3 May 2011 21:45:10 +0530 Subject: Story #42 - Volume logs download --- .../build/glusterserver.ant | 3 +- .../server/resources/VolumesResource.java | 107 ++++++++++++++++++--- .../management/server/utils/ServerUtil.java | 84 +++++++++++----- 3 files changed, 158 insertions(+), 36 deletions(-) (limited to 'src/com.gluster.storage.management.server') diff --git a/src/com.gluster.storage.management.server/build/glusterserver.ant b/src/com.gluster.storage.management.server/build/glusterserver.ant index 88602fb6..98012515 100644 --- a/src/com.gluster.storage.management.server/build/glusterserver.ant +++ b/src/com.gluster.storage.management.server/build/glusterserver.ant @@ -39,7 +39,8 @@ - + + diff --git a/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/resources/VolumesResource.java b/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/resources/VolumesResource.java index fd4643a6..e635510a 100644 --- a/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/resources/VolumesResource.java +++ b/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/resources/VolumesResource.java @@ -31,6 +31,7 @@ import static com.gluster.storage.management.core.constants.RESTConstants.PATH_P import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DELETE_OPTION; import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DISKS; import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DISK_NAME; +import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DOWNLOAD; import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_FROM_TIMESTAMP; import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_LINE_COUNT; import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_LOG_SEVERITY; @@ -39,15 +40,22 @@ import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_ import static com.gluster.storage.management.core.constants.RESTConstants.RESOURCE_PATH_VOLUMES; import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_DEFAULT_OPTIONS; import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_DISKS; +import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_DOWNLOAD; import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_LOGS; import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_OPTIONS; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.List; +import java.util.zip.GZIPOutputStream; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; @@ -59,7 +67,10 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; import com.gluster.storage.management.core.constants.CoreConstants; import com.gluster.storage.management.core.constants.RESTConstants; @@ -73,6 +84,8 @@ import com.gluster.storage.management.core.response.LogMessageListResponse; import com.gluster.storage.management.core.response.VolumeListResponse; import com.gluster.storage.management.core.response.VolumeOptionInfoListResponse; import com.gluster.storage.management.core.utils.DateUtil; +import com.gluster.storage.management.core.utils.FileUtil; +import com.gluster.storage.management.core.utils.ProcessUtil; import com.gluster.storage.management.server.constants.VolumeOptionsDefaults; import com.gluster.storage.management.server.utils.GlusterUtil; import com.gluster.storage.management.server.utils.ServerUtil; @@ -219,8 +232,14 @@ public class VolumesResource { @SuppressWarnings("rawtypes") private Status prepareBrick(String serverName, String diskName, String volumeName) { - return (Status) ((GenericResponse) serverUtil.executeOnServer(true, serverName, PREPARE_BRICK_SCRIPT + " " - + diskName + " " + volumeName, GenericResponse.class)).getStatus(); + Object response = serverUtil.executeOnServer(true, serverName, PREPARE_BRICK_SCRIPT + " " + + diskName + " " + volumeName, GenericResponse.class); + if(response instanceof GenericResponse) { + return ((GenericResponse)response).getStatus(); + } else { + // in case of script failure on server, a Status object will be returned + return (Status) response; + } } private Status createDirectories(List disks, String volumeName) { @@ -232,11 +251,8 @@ public class VolumesResource { String[] diskParts = disk.split(":"); String serverName = diskParts[0]; String diskName = diskParts[1]; - try { - status = prepareBrick(serverName, diskName, volumeName); - } catch (Exception e) { - status = new Status(e); - } + + status = prepareBrick(serverName, diskName, volumeName); if (status.isSuccess()) { String brickDir = status.getMessage().trim(); bricks.add(serverName + ":" + brickDir); @@ -271,15 +287,24 @@ public class VolumesResource { diskInfo = disks.get(i).split(":"); serverName = diskInfo[0]; diskName = diskInfo[1]; - result = ((GenericResponse) serverUtil.executeOnServer(true, serverName, VOLUME_DIRECTORY_CLEANUP_SCRIPT - + " " + diskName + " " + volumeName + " " + deleteFlag, GenericResponse.class)).getStatus(); - if (!result.isSuccess()) { - return result; + + Object response = serverUtil.executeOnServer(true, serverName, VOLUME_DIRECTORY_CLEANUP_SCRIPT + + " " + diskName + " " + volumeName + " " + deleteFlag, GenericResponse.class); + if(response instanceof GenericResponse) { + result = ((GenericResponse)response).getStatus(); + if (!result.isSuccess()) { + // TODO: append error and continue with cleaning up of other directories + return result; + } + } else { + // TODO: append error and continue with cleaning up of other directories + // In case of script execution failure, a Status object will be returned. + return (Status)response; } } return new Status(Status.STATUS_CODE_SUCCESS, "Directories cleaned up successfully!"); } - + private List getBrickLogs(Volume volume, String brickName, Integer lineCount) throws GlusterRuntimeException { // brick name format is : @@ -316,6 +341,61 @@ public class VolumesResource { } return logMessages; } + + @GET + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @Path("{" + PATH_PARAM_VOLUME_NAME + "}/" + SUBRESOURCE_LOGS + "/" + SUBRESOURCE_DOWNLOAD) + public StreamingOutput getLogs(@PathParam(PATH_PARAM_VOLUME_NAME) final String volumeName) { + return new StreamingOutput() { + + @Override + public void write(OutputStream output) throws IOException, WebApplicationException { + Volume volume = getVolume(volumeName); + try { + String archiveFileName = downloadLogs(volume); + FileInputStream inputStream = new FileInputStream(archiveFileName); + int size = inputStream.available(); + byte[] data = new byte[size]; + inputStream.read(data); + inputStream.close(); + output.write(data); + } catch (Exception e) { + e.printStackTrace(); + throw new GlusterRuntimeException("Exception while downloading/archiving volume log files!", e); + } + } + }; + } + + private String downloadLogs(Volume volume) { + FileUtil fileUtil = new FileUtil(); + + // create temporary directory + File tempDir = fileUtil.createTempDir(); + String tempDirPath = tempDir.getPath(); + + for(String brickName : volume.getBricks()) { + // brick name format is : + String[] brickParts = brickName.split(":"); + String serverName = brickParts[0]; + String brickDir = brickParts[1]; + + String logDir = glusterUtil.getLogLocation(volume.getName(), brickName); + String logFileName = glusterUtil.getLogFileNameForBrickDir(brickDir); + String logFilePath = logDir + CoreConstants.FILE_SEPARATOR + logFileName; + + String logContents = serverUtil.getFileFromServer(serverName, logFilePath); + fileUtil.createTextFile(tempDirPath + CoreConstants.FILE_SEPARATOR + logFileName, logContents); + } + + String gzipPath = fileUtil.getTempDirName() + CoreConstants.FILE_SEPARATOR + volume.getName() + "-logs.tar.gz"; + new ProcessUtil().executeCommand("tar", "czvf", gzipPath, tempDirPath); + + // delete the temp directory + fileUtil.recursiveDelete(tempDir); + + return gzipPath; + } @GET @Path("{" + PATH_PARAM_VOLUME_NAME + "}/" + SUBRESOURCE_LOGS) @@ -323,7 +403,8 @@ public class VolumesResource { @QueryParam(QUERY_PARAM_DISK_NAME) String diskName, @QueryParam(QUERY_PARAM_LOG_SEVERITY) String severity, @QueryParam(QUERY_PARAM_FROM_TIMESTAMP) String fromTimestamp, @QueryParam(QUERY_PARAM_TO_TIMESTAMP) String toTimestamp, - @QueryParam(QUERY_PARAM_LINE_COUNT) Integer lineCount) { + @QueryParam(QUERY_PARAM_LINE_COUNT) Integer lineCount, + @QueryParam(QUERY_PARAM_DOWNLOAD) Boolean download) { List logMessages = null; try { diff --git a/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/utils/ServerUtil.java b/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/utils/ServerUtil.java index 5e423f1c..627882b7 100644 --- a/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/utils/ServerUtil.java +++ b/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/utils/ServerUtil.java @@ -22,6 +22,7 @@ package com.gluster.storage.management.server.utils; import java.io.BufferedReader; import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.InetAddress; @@ -38,7 +39,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.gluster.storage.management.core.constants.CoreConstants; +import com.gluster.storage.management.core.exceptions.GlusterRuntimeException; import com.gluster.storage.management.core.model.Status; +import com.gluster.storage.management.core.response.GenericResponse; import com.gluster.storage.management.core.utils.ProcessResult; import com.gluster.storage.management.core.utils.ProcessUtil; @@ -71,34 +74,70 @@ public class ServerUtil { * @param runInForeground * @param serverName * @param commandWithArgs - * @param expectedClass Class of the object expected from script execution - * @return Response from remote execution of the command + * @param expectedClass + * Class of the object expected from script execution + * @return Object of the expected class from remote execution of the command. In case the remote execution fails + * ungracefully, an object of class {@link Status} will be returned. */ @SuppressWarnings("rawtypes") - public Object executeOnServer(boolean runInForeground, String serverName, String commandWithArgs, Class expectedClass) { - StringBuffer output = new StringBuffer(); + public Object executeOnServer(boolean runInForeground, String serverName, String commandWithArgs, + Class expectedClass) { + try { + String output = executeOnServer(serverName, commandWithArgs); + + // In case the script execution exits ungracefully, the agent would return a GenericResponse. + // hence pass last argument as true to try GenericResponse unmarshalling in such cases. + Object response = unmarshal(expectedClass, output, expectedClass != GenericResponse.class); + if (expectedClass != GenericResponse.class && response instanceof GenericResponse) { + // expected class was not GenericResponse, but that's what we got. This means the + // script failed ungracefully. Extract and return the status object from the response + return ((GenericResponse) response).getStatus(); + } + return response; + } catch (Exception e) { + // any other exception means unexpected error. return status with error from exception. + return new Status(e); + } + } + + private String executeOnServer(String serverName, String commandWithArgs) { try { InetAddress address = InetAddress.getByName(serverName); Socket connection = new Socket(address, 50000); PrintWriter writer = new PrintWriter(connection.getOutputStream(), true); - BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8")); - writer.println(commandWithArgs); writer.println(); // empty line means end of request - String line; - while (!(line = reader.readLine()).trim().isEmpty()) { - output.append(line + CoreConstants.NEWLINE); + InputStream inputStream = connection.getInputStream(); + int available = inputStream.available(); + + StringBuffer output = new StringBuffer(); + if( available > 0 ) { + // This happens when PeerAgent sends complete file + byte[] responseData = new byte[available]; + inputStream.read(responseData); + output.append(new String(responseData, "UTF-8")); + } else { + // This happens in case of normal XML response from PeerAgent + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); + + String line; + while (!(line = reader.readLine()).trim().isEmpty()) { + output.append(line + CoreConstants.NEWLINE); + } } - connection.close(); - return unmarshal(expectedClass, output.toString(), expectedClass != Status.class); - } catch(Exception e) { - // any other exception means unexpected error. return status with error from exception. - return new Status(Status.STATUS_CODE_FAILURE, "Error during remote execution: [" + e.getMessage() + "]"); + + return output.toString(); + } catch (Exception e) { + throw new GlusterRuntimeException("Error during remote execution: [" + e.getMessage() + "]"); } } + + public String getFileFromServer(String serverName, String fileName) { + return executeOnServer(serverName, "get_file " + fileName); + } /** * Unmarshals given input string into object of given class @@ -107,22 +146,23 @@ public class ServerUtil { * Class whose object is expected * @param input * Input string - * @param tryStatusOnFailure + * @param tryGenericResponseOnFailure * If true, and if the unmarshalling fails for given class, another unmarshalling will be attempted with - * class Status. If that also fails, a status object with exception message is created and returned. + * class {@link GenericResponse}. If this also fails, a status object with exception message is created + * and returned. * @return Object of given expected class, or a status object in case first unmarshalling fails. */ @SuppressWarnings("rawtypes") - private Object unmarshal(Class expectedClass, String input, boolean tryStatusOnFailure) { + private Object unmarshal(Class expectedClass, String input, boolean tryGenericResponseOnFailure) { try { // create JAXB context and instantiate marshaller JAXBContext context = JAXBContext.newInstance(expectedClass); Unmarshaller um = context.createUnmarshaller(); return um.unmarshal(new ByteArrayInputStream(input.getBytes())); } catch (JAXBException e) { - if(tryStatusOnFailure) { - // unmarshalling failed. try to unmarshal a Status object - return unmarshal(Status.class, input, false); + if(tryGenericResponseOnFailure) { + // unmarshalling failed. try to unmarshal a GenericResponse object + return unmarshal(GenericResponse.class, input, false); } return new Status(Status.STATUS_CODE_FAILURE, "Error during unmarshalling string [" + input @@ -130,9 +170,9 @@ public class ServerUtil { } } - public static void main(String args[]) { + public static void main(String args[]) throws Exception { // CreateVolumeExportDirectory.py md0 testvol - System.out.println(new ServerUtil().executeOnServer(true, "localhost", "python CreateVolumeExportDirectory.py md0 testvol", Status.class)); + System.out.println(new ServerUtil().getFileFromServer("localhost", "/tmp/python/PeerAgent.py")); } /** -- cgit