Bug 3279 - Not able to switch online server if glusterd on current server is down

8 files changed, 164 insertions, 87 deletions

diff --git a/src/com.gluster.storage.management.core/src/com/gluster/storage/management/core/constants/RESTConstants.java b/src/com.gluster.storage.management.core/src/com/gluster/storage/management/core/constants/RESTConstants.java
index b4bd7c15..faddb1c6 100644
--- a/src/com.gluster.storage.management.core/src/com/gluster/storage/management/core/constants/RESTConstants.java
+++ b/src/com.gluster.storage.management.core/src/com/gluster/storage/management/core/constants/RESTConstants.java
@@ -82,8 +82,8 @@ public class RESTConstants {
 	public static final String FORM_PARAM_FIX_LAYOUT = "fix-layout";
 	public static final String FORM_PARAM_MIGRATE_DATA = "migrate-data";
 	public static final String FORM_PARAM_FORCED_DATA_MIGRATE = "forced-data-migrate";
-	public static final String FORM_PARAM_OLD_PASSWORD = "oldpassword";
-	public static final String FORM_PARAM_NEW_PASSWORD = "newpassword";
+	public static final String FORM_PARAM_OLD_PASSWORD = "oldPassword";
+	public static final String FORM_PARAM_NEW_PASSWORD = "newPassword";
 	
 	public static final String PATH_PARAM_FORMAT = "format";
 	public static final String PATH_PARAM_VOLUME_NAME = "volumeName";
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/constants/VolumeOptionsDefaults.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/constants/VolumeOptionsDefaults.java
index 0e008b32..892ebf11 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/constants/VolumeOptionsDefaults.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/constants/VolumeOptionsDefaults.java
@@ -137,23 +137,27 @@ public class VolumeOptionsDefaults {
 	
 	public List<VolumeOptionInfo> getVolumeOptionsInfo(String clusterName) {
 		String command = "gluster volume set help-xml";
-		String output = "";
-		VolumeOptionInfoListResponse options = new VolumeOptionInfoListResponse();
 		GlusterServer onlineServer = clusterService.getOnlineServer(clusterName);
 
 		try {
-			output = getVolumeOptionsInfo(onlineServer.getName(), command);
-			options = parseXML(output); 
-			return options.getOptions();
-		} catch (ConnectionException e) {
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
-			output =  getVolumeOptionsInfo(onlineServer.getName(), command);
-			options = parseXML(output);
-			return options.getOptions();
+			return getVolumeOptionsInfo(command, onlineServer);
 		} catch (Exception e) {
-			throw new GlusterRuntimeException("Fetching volume options default failed! [" + e.getMessage() + "]"); 
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				return getVolumeOptionsInfo(command, clusterService.getNewOnlineServer(clusterName));
+			} else {
+				throw new GlusterRuntimeException("Fetching volume options default failed! [" + e.getMessage() + "]");
+			}
 		}
 	}
+
+	private List<VolumeOptionInfo> getVolumeOptionsInfo(String command, GlusterServer onlineServer) {
+		String output;
+		VolumeOptionInfoListResponse options;
+		output =  getVolumeOptionsInfo(onlineServer.getName(), command);
+		options = parseXML(output);
+		return options.getOptions();
+	}
 	
 	private String getVolumeOptionsInfo(String serverName, String command) {
 		return serverUtil.executeOnServer(true, serverName, command,
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/GlusterServersResource.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/GlusterServersResource.java
index 90442f4a..dc8fa3dd 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/GlusterServersResource.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/resources/v1_0/GlusterServersResource.java
@@ -210,9 +210,12 @@ public class GlusterServersResource extends AbstractResource {
 
 		try {
 			glusterUtil.addServer(onlineServer.getName(), serverName);
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+			}
 			if (onlineServer == null) {
 				throw new GlusterRuntimeException("No online server found in cluster [" + clusterName + "]");
 			}
@@ -327,9 +330,12 @@ public class GlusterServersResource extends AbstractResource {
 
 		try {
 			glusterUtil.removeServer(onlineServer.getName(), serverName);
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName, serverName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName, serverName);
+			}
 			if (onlineServer == null) {
 				throw new GlusterRuntimeException("No online server found in cluster [" + clusterName + "]");
 			}
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/GlusterServerService.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/GlusterServerService.java
index c4cf2850..8080df3e 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/GlusterServerService.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/GlusterServerService.java
@@ -68,9 +68,12 @@ public class GlusterServerService {
 
 		try {
 			glusterServers = getGlusterServers(clusterName, onlineServer, fetchDetails, maxCount, previousServerName);
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);	
+			}
 			if (onlineServer == null) {
 				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
 			}
@@ -84,9 +87,12 @@ public class GlusterServerService {
 		List<GlusterServer> glusterServers;
 		try {
 			glusterServers = glusterUtil.getGlusterServers(onlineServer);
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+			}
 			if (onlineServer == null) {
 				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
 			}
@@ -146,9 +152,12 @@ public class GlusterServerService {
 		GlusterServer server = null;
 		try {
 			server = glusterUtil.getGlusterServer(onlineServer, serverName);
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+			}
 			if (onlineServer == null) {
 				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
 			}
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/VolumeService.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/VolumeService.java
index 69fe01c1..944b29c6 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/VolumeService.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/services/VolumeService.java
@@ -118,14 +118,17 @@ public class VolumeService {
 		List<String> brickList = Arrays.asList(bricks.split(",")); 
 		try {
 			glusterUtil.addBricks(volumeName, brickList, onlineServer.getName());
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
-			if (onlineServer == null) {
-				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+				if (onlineServer == null) {
+					throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
+				}
+
+				glusterUtil.addBricks(volumeName, brickList, onlineServer.getName());
 			}
-			
-			glusterUtil.addBricks(volumeName, brickList, onlineServer.getName());
 		}
 	}
 
@@ -148,11 +151,14 @@ public class VolumeService {
 			volume = glusterUtil.getVolume(volumeName, onlineServer.getName());
 			// Collect the CIFS users if CIFS Re-exported 
 			fetchVolumeCifsUsers(clusterName, volume);
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
-			if (onlineServer == null) {
-				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+				if (onlineServer == null) {
+					throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
+				}
 			}
 			volume = glusterUtil.getVolume(volumeName, onlineServer.getName());
 			// Collect the CIFS users if CIFS Re-exported 
@@ -193,7 +199,7 @@ public class VolumeService {
 
 		try {
 			return glusterUtil.getAllVolumes(onlineServer.getName());
-		} catch (ConnectionException e) {
+		} catch (Exception e) {
 			// online server has gone offline! try with a different one.
 			onlineServer = clusterService.getNewOnlineServer(clusterName);
 			if (onlineServer == null) {
@@ -368,11 +374,14 @@ public class VolumeService {
 		try {
 			glusterUtil.createVolume(onlineServer.getName(), volumeName, volumeType, transportType, replicaCount,
 					stripeCount, bricks, accessProtocols, options);
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
-			if (onlineServer == null) {
-				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+				if (onlineServer == null) {
+					throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
+				}
 			}
 
 			glusterUtil.createVolume(onlineServer.getName(), volumeName, volumeType, transportType, replicaCount,
@@ -629,10 +638,13 @@ public class VolumeService {
 			}
 
 			performOperation(clusterName, volumeName, operation, onlineServer);
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
-			performOperation(clusterName, volumeName, operation, onlineServer);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+				performOperation(clusterName, volumeName, operation, onlineServer);
+			}
 		}
 	}
 
@@ -689,9 +701,12 @@ public class VolumeService {
 	private void removeBricks(String clusterName, String volumeName, List<String> brickList, GlusterServer onlineServer) {
 		try {
 			glusterUtil.removeBricks(volumeName, brickList, onlineServer.getName());
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+			}
 			if (onlineServer == null) {
 				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
 			}
@@ -793,13 +808,16 @@ public class VolumeService {
 
 		try {
 			glusterUtil.resetOptions(volumeName, onlineServer.getName());
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+			}
 			if (onlineServer == null) {
 				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
 			}
-			
+
 			glusterUtil.resetOptions(volumeName, onlineServer.getName());
 		}
 	}
@@ -832,13 +850,16 @@ public class VolumeService {
 		
 		try {
 			glusterUtil.setOption(volumeName, key, value, onlineServer.getName());
-		} catch (ConnectionException e) {
-			// online server has gone offline! try with a different one.
-			onlineServer = clusterService.getNewOnlineServer(clusterName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(onlineServer) == false) {
+				// online server has gone offline! try with a different one.
+				onlineServer = clusterService.getNewOnlineServer(clusterName);
+			}
 			if (onlineServer == null) {
 				throw new GlusterRuntimeException("No online servers found in cluster [" + clusterName + "]");
 			}
-			
+
 			glusterUtil.setOption(volumeName, key, value, onlineServer.getName());
 		}
 	}
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/MigrateBrickTask.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/MigrateBrickTask.java
index 9dcb38c4..a1fce1eb 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/MigrateBrickTask.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/MigrateBrickTask.java
@@ -20,6 +20,7 @@
  */
 package com.gluster.storage.management.gateway.tasks;
 
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
 import org.springframework.web.context.ContextLoader;
 
@@ -30,6 +31,7 @@ import com.gluster.storage.management.core.model.TaskStatus;
 import com.gluster.storage.management.core.utils.ProcessResult;
 import com.gluster.storage.management.gateway.services.ClusterService;
 import com.gluster.storage.management.gateway.utils.GlusterUtil;
+import com.gluster.storage.management.gateway.utils.ServerUtil;
 import com.sun.jersey.core.util.Base64;
 
 public class MigrateBrickTask extends Task {
@@ -38,6 +40,9 @@ public class MigrateBrickTask extends Task {
 	private String toBrick;
 	private Boolean autoCommit;
 	private GlusterUtil glusterUtil;
+	
+	@Autowired
+	protected ServerUtil serverUtil;
 
 	public String getFromBrick() {
 		return fromBrick;
@@ -89,9 +94,12 @@ public class MigrateBrickTask extends Task {
 	public void start() {
 		try {
 			startMigration(getOnlineServer().getName());
-		} catch (ConnectionException e) {
-			// online server might have gone Offline. try with a new one.
-			startMigration(getNewOnlineServer().getName());
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(getOnlineServer()) == false) {
+				// online server might have gone Offline. try with a new one.
+				startMigration(getNewOnlineServer().getName());
+			}
 		}
 	}
 
@@ -110,9 +118,12 @@ public class MigrateBrickTask extends Task {
 	public void pause() {
 		try {
 			pauseMigration(getOnlineServer().getName());
-		} catch (ConnectionException e) {
-			// online server might have gone offline. try with a new one.
-			pauseMigration(getNewOnlineServer().getName());
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(getOnlineServer()) == false) {
+				// online server might have gone offline. try with a new one.
+				pauseMigration(getNewOnlineServer().getName());
+			}
 		}
 	}
 
@@ -138,9 +149,12 @@ public class MigrateBrickTask extends Task {
 	public void commit() {
 		try {
 			commitMigration(getOnlineServer().getName());
-		} catch (ConnectionException e) {
-			// online server might have gone offline. try with a new one.
-			commitMigration(getNewOnlineServer().getName());
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(getOnlineServer()) == false) {
+				// online server might have gone offline. try with a new one.
+				commitMigration(getNewOnlineServer().getName());
+			}
 		}
 	}
 	
@@ -162,9 +176,12 @@ public class MigrateBrickTask extends Task {
 	public void stop() {
 		try {
 			stopMigration(getOnlineServer().getName());
-		} catch (ConnectionException e) {
-			// online server might have gone offline. try with a new one.
-			stopMigration(getNewOnlineServer().getName());
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(getOnlineServer()) == false) {
+				// online server might have gone offline. try with a new one.
+				stopMigration(getNewOnlineServer().getName());
+			}
 		}
 	}
 
@@ -184,10 +201,14 @@ public class MigrateBrickTask extends Task {
 	public TaskStatus checkStatus() {
 		try {
 			return checkMigrationStatus(getOnlineServer().getName());
-		} catch (ConnectionException e) {
-			// online server might have gone offline. try with a new one.
-			return checkMigrationStatus(getNewOnlineServer().getName());
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(getOnlineServer()) == false) {
+				// online server might have gone offline. try with a new one.
+				return checkMigrationStatus(getNewOnlineServer().getName());
+			}
 		}
+		return null;
 	}
 	
 	private TaskStatus checkMigrationStatus(String serverName) {
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/RebalanceVolumeTask.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/RebalanceVolumeTask.java
index 1be71c0d..8c844f5b 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/RebalanceVolumeTask.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/tasks/RebalanceVolumeTask.java
@@ -66,10 +66,13 @@ public class RebalanceVolumeTask extends Task {
 		try {
 			serverName = getOnlineServer().getName();
 			startRebalance(serverName);
-		} catch(ConnectionException e) {
-			// online server might have gone offline. try with a new one
-			serverName = getNewOnlineServer().getName();
-			startRebalance(serverName);
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(getOnlineServer()) == false) {
+				// online server might have gone offline. try with a new one
+				serverName = getNewOnlineServer().getName();
+				startRebalance(serverName);
+			}
 		}
 	}
 
@@ -90,9 +93,12 @@ public class RebalanceVolumeTask extends Task {
 	public void stop() {
 		try {
 			glusterUtil.stopRebalance(serverName, getTaskInfo().getReference());
-		} catch (ConnectionException e) {
-			// online server might have gone offline. update the failure status
-			getTaskInfo().setStatus(new TaskStatus(new Status(Status.STATUS_CODE_FAILURE, e.getMessage())));
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(getOnlineServer()) == false) {
+				// online server might have gone offline. update the failure status
+				getTaskInfo().setStatus(new TaskStatus(new Status(Status.STATUS_CODE_FAILURE, e.getMessage())));
+			}
 		}
 	}
 
@@ -107,11 +113,15 @@ public class RebalanceVolumeTask extends Task {
 	public TaskStatus checkStatus() {
 		try {
 			return glusterUtil.checkRebalanceStatus(serverName, getTaskInfo().getReference());
-		} catch(ConnectionException e) {
-			// online server might have gone offline. update the failure status
-			getTaskInfo().setStatus(new TaskStatus(new Status(Status.STATUS_CODE_FAILURE, e.getMessage())));
-			return getTaskInfo().getStatus();
+		} catch (Exception e) {
+			// check if online server has gone offline. If yes, try again one more time.
+			if (e instanceof ConnectionException || serverUtil.isServerOnline(getOnlineServer()) == false) {
+				// online server might have gone offline. update the failure status
+				getTaskInfo().setStatus(new TaskStatus(new Status(Status.STATUS_CODE_FAILURE, e.getMessage())));
+				return getTaskInfo().getStatus();
+			}
 		}
+		return null;
 	}
 	
 	public void setLayout(String layout) {
diff --git a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/utils/ServerUtil.java b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/utils/ServerUtil.java
index 9364ea68..e057787f 100644
--- a/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/utils/ServerUtil.java
+++ b/src/com.gluster.storage.management.gateway/src/com/gluster/storage/management/gateway/utils/ServerUtil.java
@@ -99,6 +99,12 @@ public class ServerUtil {
 		server.setDisks(((Server) response).getDisks());
 	}
 	
+	public boolean isServerOnline(Server server) {
+		// fetch latest details and check if server is still online
+		fetchServerDetails(server);
+		return server.isOnline();
+	}
+	
 	public String fetchHostName(String serverName) {
 		Object response = fetchServerDetails(serverName);
 		return ((Server) response).getName();