snapshot/scheduler: Do not enable scheduler if another scheduler is running

     Backport of http://review.gluster.org/#/c/10641/

Check if another snapshot scheduler is running before
enabling the scheduler

Also introducing a hidden option, disable_force

"snapshot_scheduler.py disable_force" will disable the
cli snapshot scheduler from any node, even though the node
has not been initialised for the scheduler, as long as the
shared storage is mounted

This option is hidden, because we don't want to encourage
users to use all commands from nodes that are not initialised.

Change-Id: I7ad941fbbab834225a36e740c61f8e740813e7c8
BUG: 1226120
Signed-off-by: Avra Sengupta <asengupt@redhat.com>
Reviewed-on: http://review.gluster.org/10641
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Tested-by: NetBSD Build System
Reviewed-by: Kaushal M <kaushal@redhat.com>
(cherry picked from commit d67eb34b2a5b5e3cb926ff4c86a163148743829c)
Reviewed-on: http://review.gluster.org/10993
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Aravinda VK <avishwan@redhat.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Krishnan Parthasarathi <kparthas@redhat.com>

1 files changed, 85 insertions, 2 deletions

diff --git a/extras/snap_scheduler/snap_scheduler.py b/extras/snap_scheduler/snap_scheduler.py
index 68ac08937cb..e7c1791a15e 100755
--- a/extras/snap_scheduler/snap_scheduler.py
+++ b/extras/snap_scheduler/snap_scheduler.py
@@ -33,8 +33,10 @@ LOCK_FILE_DIR = SHARED_STORAGE_DIR+"/snaps/lock_files/"
 LOCK_FILE = LOCK_FILE_DIR+"lock_file"
 TMP_FILE = SHARED_STORAGE_DIR+"/snaps/tmp_file"
 GCRON_UPDATE_TASK = "/etc/cron.d/gcron_update_task"
+CURRENT_SCHEDULER = SHARED_STORAGE_DIR+"/snaps/current_scheduler"
 tasks = {}
 longest_field = 12
+current_scheduler = ""
 
 
 def output(msg):
@@ -81,11 +83,28 @@ def scheduler_status():
 
     return success
 
-
 def enable_scheduler():
     ret = scheduler_status()
     if ret:
         if not scheduler_enabled:
+
+            # Check if another scheduler is active.
+            ret = get_current_scheduler()
+            if ret:
+                if (current_scheduler != "none"):
+                    print_str = "Failed to enable snapshot scheduling. " \
+                                "Error: Another scheduler is active."
+                    log.error(print_str)
+                    output(print_str)
+                    ret = False
+                    return ret
+            else:
+                print_str = "Failed to get current scheduler info."
+                log.error(print_str)
+                output(print_str)
+                ret = False
+                return ret
+
             log.info("Enabling snapshot scheduler.")
             try:
                 if os.path.exists(GCRON_DISABLED):
@@ -102,6 +121,7 @@ def enable_scheduler():
                     ret = False
                     return ret
                 os.symlink(GCRON_ENABLED, GCRON_TASKS)
+                update_current_scheduler("cli")
                 log.info("Snapshot scheduling is enabled")
                 output("Snapshot scheduling is enabled")
             except IOError as (errno, strerror):
@@ -130,6 +150,20 @@ def disable_scheduler():
         if scheduler_enabled:
             log.info("Disabling snapshot scheduler.")
             try:
+                # Check if another scheduler is active. If not, then
+                # update current scheduler to "none". Else do nothing.
+                ret = get_current_scheduler()
+                if ret:
+                    if (current_scheduler == "cli"):
+                        update_current_scheduler("none")
+                else:
+                    print_str = "Failed to disable snapshot scheduling. " \
+                                "Error: Failed to get current scheduler info."
+                    log.error(print_str)
+                    output(print_str)
+                    ret = False
+                    return ret
+
                 if os.path.exists(GCRON_DISABLED):
                     os.remove(GCRON_DISABLED)
                 if os.path.lexists(GCRON_TASKS):
@@ -176,6 +210,7 @@ def load_tasks_from_file():
                 longest_field = max(longest_field, len(jobname), len(volname),
                                     len(schedule))
                 tasks[jobname] = schedule+":"+volname
+            f.close()
         ret = True
     except IOError as (errno, strerror):
         log.error("Failed to open %s. Error: %s.", GCRON_ENABLED, strerror)
@@ -184,6 +219,20 @@ def load_tasks_from_file():
     return ret
 
 
+def get_current_scheduler():
+    global current_scheduler
+    try:
+        with open(CURRENT_SCHEDULER, 'r') as f:
+            current_scheduler = f.readline().rstrip('\n')
+            f.close()
+        ret = True
+    except IOError as (errno, strerror):
+        log.error("Failed to open %s. Error: %s.", CURRENT_SCHEDULER, strerror)
+        ret = False
+
+    return ret
+
+
 def list_schedules():
     log.info("Listing snapshot schedules.")
     ret = load_tasks_from_file()
@@ -230,6 +279,7 @@ def write_tasks_to_file():
                 f.write("\n")
                 f.flush()
                 os.fsync(f.fileno())
+            f.close()
     except IOError as (errno, strerror):
         log.error("Failed to open %s. Error: %s.", TMP_FILE, strerror)
         ret = False
@@ -240,6 +290,24 @@ def write_tasks_to_file():
 
     return ret
 
+def update_current_scheduler(data):
+    ret = False
+    try:
+        with open(TMP_FILE, "w", 0644) as f:
+            f.write("%s" % data)
+            f.flush()
+            os.fsync(f.fileno())
+            f.close()
+    except IOError as (errno, strerror):
+        log.error("Failed to open %s. Error: %s.", TMP_FILE, strerror)
+        ret = False
+        return ret
+
+    shutil.move(TMP_FILE, CURRENT_SCHEDULER)
+    ret = True
+
+    return ret
+
 
 def add_schedules(jobname, schedule, volname):
     log.info("Adding snapshot schedules.")
@@ -342,6 +410,7 @@ def initialise_scheduler():
             f.write("%s\n" % updater)
             f.flush()
             os.fsync(f.fileno())
+            f.close()
     except IOError as (errno, strerror):
         log.error("Failed to open /tmp/crontab. Error: %s.", strerror)
         ret = False
@@ -411,6 +480,13 @@ def perform_operation(args):
             output("Failed to initialise snapshot scheduling")
         return ret
 
+    # Disable snapshot scheduler
+    if args.action == "disable_force":
+        ret = disable_scheduler()
+        if ret:
+            subprocess.Popen(["touch", "-h", GCRON_TASKS])
+        return ret
+
     # Check if the symlink to GCRON_TASKS is properly set in the shared storage
     if (not os.path.lexists(GCRON_UPDATE_TASK) or
         not os.path.lexists(GCRON_CROND_TASK) or
@@ -489,7 +565,10 @@ def main():
     initLogger()
     ret = -1
     parser = argparse.ArgumentParser()
-    subparsers = parser.add_subparsers(dest="action")
+    subparsers = parser.add_subparsers(dest="action",
+                                       metavar=('{init, status, enable,'
+                                               ' disable, list, add,'
+                                               ' delete, edit}'))
     subparsers.add_parser('init',
                           help="Initialise the node for snapshot scheduling")
 
@@ -500,6 +579,7 @@ def main():
                           help="Enable snapshot scheduling")
     subparsers.add_parser("disable",
                           help="Disable snapshot scheduling")
+    subparsers.add_parser("disable_force")
     subparsers.add_parser("list",
                           help="List snapshot schedules")
     parser_add = subparsers.add_parser("add",
@@ -549,6 +629,9 @@ def main():
                 output("Failed to create %s. Error: %s"
                        % (LOCK_FILE_DIR, strerror))
 
+    if not os.path.exists(CURRENT_SCHEDULER):
+        update_current_scheduler("none")
+
     try:
         f = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0644)
         try: