Story #42 - Volume logs download

3 files changed, 145 insertions, 36 deletions

diff --git a/src/com.gluster.storage.management.server/build/glusterserver.ant b/src/com.gluster.storage.management.server/build/glusterserver.ant
index 88602fb6..98012515 100644
--- a/src/com.gluster.storage.management.server/build/glusterserver.ant
+++ b/src/com.gluster.storage.management.server/build/glusterserver.ant
@@ -39,7 +39,8 @@
 				<include name="**/*.sql" />
 			</fileset>
 		</copy>
-		<javac srcdir="${basedir}/src" destdir="${WEB-INF}/classes" classpathref="libs" />
+		<!-- TODO: make debug option configurable in jenkins -->
+		<javac srcdir="${basedir}/src" destdir="${WEB-INF}/classes" classpathref="libs" debug="true" debuglevel="lines,vars,source"/>
 	</target>
 
 	<target name="archive" depends="compile">
diff --git a/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/resources/VolumesResource.java b/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/resources/VolumesResource.java
index fd4643a6..e635510a 100644
--- a/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/resources/VolumesResource.java
+++ b/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/resources/VolumesResource.java
@@ -31,6 +31,7 @@ import static com.gluster.storage.management.core.constants.RESTConstants.PATH_P
 import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DELETE_OPTION;
 import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DISKS;
 import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DISK_NAME;
+import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_DOWNLOAD;
 import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_FROM_TIMESTAMP;
 import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_LINE_COUNT;
 import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_PARAM_LOG_SEVERITY;
@@ -39,15 +40,22 @@ import static com.gluster.storage.management.core.constants.RESTConstants.QUERY_
 import static com.gluster.storage.management.core.constants.RESTConstants.RESOURCE_PATH_VOLUMES;
 import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_DEFAULT_OPTIONS;
 import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_DISKS;
+import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_DOWNLOAD;
 import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_LOGS;
 import static com.gluster.storage.management.core.constants.RESTConstants.SUBRESOURCE_OPTIONS;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.List;
+import java.util.zip.GZIPOutputStream;
 
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
@@ -59,7 +67,10 @@ import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
 
 import com.gluster.storage.management.core.constants.CoreConstants;
 import com.gluster.storage.management.core.constants.RESTConstants;
@@ -73,6 +84,8 @@ import com.gluster.storage.management.core.response.LogMessageListResponse;
 import com.gluster.storage.management.core.response.VolumeListResponse;
 import com.gluster.storage.management.core.response.VolumeOptionInfoListResponse;
 import com.gluster.storage.management.core.utils.DateUtil;
+import com.gluster.storage.management.core.utils.FileUtil;
+import com.gluster.storage.management.core.utils.ProcessUtil;
 import com.gluster.storage.management.server.constants.VolumeOptionsDefaults;
 import com.gluster.storage.management.server.utils.GlusterUtil;
 import com.gluster.storage.management.server.utils.ServerUtil;
@@ -219,8 +232,14 @@ public class VolumesResource {
 
 	@SuppressWarnings("rawtypes")
 	private Status prepareBrick(String serverName, String diskName, String volumeName) {
-		return (Status) ((GenericResponse) serverUtil.executeOnServer(true, serverName, PREPARE_BRICK_SCRIPT + " "
-				+ diskName + " " + volumeName, GenericResponse.class)).getStatus();
+		Object response = serverUtil.executeOnServer(true, serverName, PREPARE_BRICK_SCRIPT + " "
+				+ diskName + " " + volumeName, GenericResponse.class);
+		if(response instanceof GenericResponse) {
+			return ((GenericResponse)response).getStatus();
+		} else {
+			// in case of script failure on server, a Status object will be returned
+			return (Status) response;
+		}		
 	}
 
 	private Status createDirectories(List<String> disks, String volumeName) {
@@ -232,11 +251,8 @@ public class VolumesResource {
 			String[] diskParts = disk.split(":");
 			String serverName = diskParts[0];
 			String diskName = diskParts[1];
-			try {
-				status = prepareBrick(serverName, diskName, volumeName);
-			} catch (Exception e) {
-				status = new Status(e);
-			}
+
+			status = prepareBrick(serverName, diskName, volumeName);
 			if (status.isSuccess()) {
 				String brickDir = status.getMessage().trim();
 				bricks.add(serverName + ":" + brickDir);
@@ -271,15 +287,24 @@ public class VolumesResource {
 			diskInfo = disks.get(i).split(":");
 			serverName = diskInfo[0];
 			diskName = diskInfo[1];
-			result = ((GenericResponse) serverUtil.executeOnServer(true, serverName, VOLUME_DIRECTORY_CLEANUP_SCRIPT
-					+ " " + diskName + " " + volumeName + " " + deleteFlag, GenericResponse.class)).getStatus();
-			if (!result.isSuccess()) {
-				return result;
+			
+			Object response = serverUtil.executeOnServer(true, serverName, VOLUME_DIRECTORY_CLEANUP_SCRIPT
+					+ " " + diskName + " " + volumeName + " " + deleteFlag, GenericResponse.class);
+			if(response instanceof GenericResponse) {
+				result = ((GenericResponse)response).getStatus();
+				if (!result.isSuccess()) {
+					// TODO: append error and continue with cleaning up of other directories
+					return result;
+				}
+			} else {
+				// TODO: append error and continue with cleaning up of other directories
+				// In case of script execution failure, a Status object will be returned.
+				return (Status)response;
 			}
 		}
 		return new Status(Status.STATUS_CODE_SUCCESS, "Directories cleaned up successfully!");
 	}
-
+	
 	private List<LogMessage> getBrickLogs(Volume volume, String brickName, Integer lineCount)
 			throws GlusterRuntimeException {
 		// brick name format is <serverName>:<brickDirectory>
@@ -316,6 +341,61 @@ public class VolumesResource {
 		}
 		return logMessages;
 	}
+	
+	@GET
+	@Produces(MediaType.APPLICATION_OCTET_STREAM)
+	@Path("{" + PATH_PARAM_VOLUME_NAME + "}/" + SUBRESOURCE_LOGS + "/" + SUBRESOURCE_DOWNLOAD)
+	public StreamingOutput getLogs(@PathParam(PATH_PARAM_VOLUME_NAME) final String volumeName) {
+		return new StreamingOutput() {
+			
+			@Override
+			public void write(OutputStream output) throws IOException, WebApplicationException {
+				Volume volume = getVolume(volumeName);
+				try {
+					String archiveFileName = downloadLogs(volume);
+					FileInputStream inputStream = new FileInputStream(archiveFileName);
+					int size = inputStream.available();
+					byte[] data = new byte[size];
+					inputStream.read(data);
+					inputStream.close();
+					output.write(data);
+				} catch (Exception e) {
+					e.printStackTrace();
+					throw new GlusterRuntimeException("Exception while downloading/archiving volume log files!", e);
+				}
+			}
+		};
+	}
+
+	private String downloadLogs(Volume volume) {
+		FileUtil fileUtil = new FileUtil();
+		
+		// create temporary directory
+		File tempDir = fileUtil.createTempDir();
+		String tempDirPath = tempDir.getPath();
+		
+		for(String brickName : volume.getBricks()) {
+			// brick name format is <serverName>:<brickDirectory>
+			String[] brickParts = brickName.split(":");
+			String serverName = brickParts[0];
+			String brickDir = brickParts[1];
+			
+			String logDir = glusterUtil.getLogLocation(volume.getName(), brickName);
+			String logFileName = glusterUtil.getLogFileNameForBrickDir(brickDir);
+			String logFilePath = logDir + CoreConstants.FILE_SEPARATOR + logFileName;
+
+			String logContents = serverUtil.getFileFromServer(serverName, logFilePath);
+			fileUtil.createTextFile(tempDirPath + CoreConstants.FILE_SEPARATOR + logFileName, logContents);
+		}
+		
+		String gzipPath = fileUtil.getTempDirName() + CoreConstants.FILE_SEPARATOR + volume.getName() + "-logs.tar.gz";
+		new ProcessUtil().executeCommand("tar", "czvf", gzipPath, tempDirPath);
+		
+		// delete the temp directory
+		fileUtil.recursiveDelete(tempDir);
+		
+		return gzipPath;
+	}
 
 	@GET
 	@Path("{" + PATH_PARAM_VOLUME_NAME + "}/" + SUBRESOURCE_LOGS)
@@ -323,7 +403,8 @@ public class VolumesResource {
 			@QueryParam(QUERY_PARAM_DISK_NAME) String diskName, @QueryParam(QUERY_PARAM_LOG_SEVERITY) String severity,
 			@QueryParam(QUERY_PARAM_FROM_TIMESTAMP) String fromTimestamp,
 			@QueryParam(QUERY_PARAM_TO_TIMESTAMP) String toTimestamp,
-			@QueryParam(QUERY_PARAM_LINE_COUNT) Integer lineCount) {
+			@QueryParam(QUERY_PARAM_LINE_COUNT) Integer lineCount,
+			@QueryParam(QUERY_PARAM_DOWNLOAD) Boolean download) {
 		List<LogMessage> logMessages = null;
 
 		try {
diff --git a/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/utils/ServerUtil.java b/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/utils/ServerUtil.java
index 5e423f1c..59dc36c2 100644
--- a/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/utils/ServerUtil.java
+++ b/src/com.gluster.storage.management.server/src/com/gluster/storage/management/server/utils/ServerUtil.java
@@ -22,6 +22,7 @@ package com.gluster.storage.management.server.utils;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.net.InetAddress;
@@ -38,7 +39,9 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import com.gluster.storage.management.core.constants.CoreConstants;
+import com.gluster.storage.management.core.exceptions.GlusterRuntimeException;
 import com.gluster.storage.management.core.model.Status;
+import com.gluster.storage.management.core.response.GenericResponse;
 import com.gluster.storage.management.core.utils.ProcessResult;
 import com.gluster.storage.management.core.utils.ProcessUtil;
 
@@ -71,34 +74,57 @@ public class ServerUtil {
 	 * @param runInForeground
 	 * @param serverName
 	 * @param commandWithArgs
-	 * @param expectedClass Class of the object expected from script execution 
-	 * @return Response from remote execution of the command
+	 * @param expectedClass
+	 *            Class of the object expected from script execution
+	 * @return Object of the expected class from remote execution of the command. In case the remote execution fails
+	 *         ungracefully, an object of class {@link Status} will be returned.
 	 */
 	@SuppressWarnings("rawtypes")
-	public Object executeOnServer(boolean runInForeground, String serverName, String commandWithArgs, Class expectedClass) {
-		StringBuffer output = new StringBuffer();
+	public Object executeOnServer(boolean runInForeground, String serverName, String commandWithArgs,
+			Class expectedClass) {
+		try {
+			String output = executeOnServer(serverName, commandWithArgs);
+
+			// In case the script execution exits ungracefully, the agent would return a GenericResponse.
+			// hence pass last argument as true to try GenericResponse unmarshalling in such cases.
+			Object response = unmarshal(expectedClass, output, expectedClass != GenericResponse.class);
+			if (expectedClass != GenericResponse.class && response instanceof GenericResponse) {
+				// expected class was not GenericResponse, but that's what we got. This means the
+				// script failed ungracefully. Extract and return the status object from the response
+				return ((GenericResponse) response).getStatus();
+			}
+			return response;
+		} catch (Exception e) {
+			// any other exception means unexpected error. return status with error from exception.
+			return new Status(e);
+		}
+	}
+
+	private String executeOnServer(String serverName, String commandWithArgs) {
 		try {
 			InetAddress address = InetAddress.getByName(serverName);
 			Socket connection = new Socket(address, 50000);
 
 			PrintWriter writer = new PrintWriter(connection.getOutputStream(), true);
-			BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
+			InputStream inputStream = connection.getInputStream();
 
 			writer.println(commandWithArgs);
 			writer.println(); // empty line means end of request
 
-			String line;
-			while (!(line = reader.readLine()).trim().isEmpty()) {
-				output.append(line + CoreConstants.NEWLINE);
-			}
-
+			int available = inputStream.available();
+			byte[] responseData = new byte[available];
+			inputStream.read(responseData);
 			connection.close();
-			return unmarshal(expectedClass, output.toString(), expectedClass != Status.class);
-		} catch(Exception e) {
-			// any other exception means unexpected error. return status with error from exception.
-			return new Status(Status.STATUS_CODE_FAILURE, "Error during remote execution: [" + e.getMessage() + "]");
+
+			return new String(responseData, "UTF-8");
+		} catch (Exception e) {
+			throw new GlusterRuntimeException("Error during remote execution: [" + e.getMessage() + "]");
 		}
 	}
+	
+	public String getFileFromServer(String serverName, String fileName) {
+		return executeOnServer(serverName, "get_file " + fileName); 
+	}
 
 	/**
 	 * Unmarshals given input string into object of given class
@@ -107,22 +133,23 @@ public class ServerUtil {
 	 *            Class whose object is expected
 	 * @param input
 	 *            Input string
-	 * @param tryStatusOnFailure
+	 * @param tryGenericResponseOnFailure
 	 *            If true, and if the unmarshalling fails for given class, another unmarshalling will be attempted with
-	 *            class Status. If that also fails, a status object with exception message is created and returned.
+	 *            class {@link GenericResponse}. If this also fails, a status object with exception message is created
+	 *            and returned.
 	 * @return Object of given expected class, or a status object in case first unmarshalling fails.
 	 */
 	@SuppressWarnings("rawtypes")
-	private Object unmarshal(Class expectedClass, String input, boolean tryStatusOnFailure) {
+	private Object unmarshal(Class expectedClass, String input, boolean tryGenericResponseOnFailure) {
 		try {
 			// create JAXB context and instantiate marshaller
 			JAXBContext context = JAXBContext.newInstance(expectedClass);
 			Unmarshaller um = context.createUnmarshaller();
 			return um.unmarshal(new ByteArrayInputStream(input.getBytes()));
 		} catch (JAXBException e) {
-			if(tryStatusOnFailure) {
-				// unmarshalling failed. try to unmarshal a Status object
-				return unmarshal(Status.class, input, false);
+			if(tryGenericResponseOnFailure) {
+				// unmarshalling failed. try to unmarshal a GenericResponse object
+				return unmarshal(GenericResponse.class, input, false);
 			}
 			
 			return new Status(Status.STATUS_CODE_FAILURE, "Error during unmarshalling string [" + input
@@ -130,9 +157,9 @@ public class ServerUtil {
 		}
 	}
 
-	public static void main(String args[]) {
+	public static void main(String args[]) throws Exception {
 		// CreateVolumeExportDirectory.py md0 testvol
-		System.out.println(new ServerUtil().executeOnServer(true, "localhost", "python CreateVolumeExportDirectory.py md0 testvol", Status.class));
+		System.out.println(new ServerUtil().getFileFromServer("localhost", "/tmp/python/PeerAgent.py"));
 	}
 
 	/**