tools/glusterfind: add --end-time option

Add optional --end-time argument to be used with --since-time
when using the "query" command.

"start" and "end" times are passed in the command-line to
changelog.py only if --full has not been specified on command-line.
-1 is passed to changelog.py as end time if user has not supplied
--end-time on command-line.

brickfind.py:
Remove unused "start" command-line argument.

Also:
Minor indentation changes to keep flake8-2.7 happy.

Change-Id: I063ef5459916f711503881ade5c4fc32374edad0
BUG: 1453151
Signed-off-by: Milind Changire <mchangir@redhat.com>
Reviewed-on: https://review.gluster.org/17439
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Aravinda VK <avishwan@redhat.com>

3 files changed, 45 insertions, 24 deletions

diff --git a/tools/glusterfind/src/brickfind.py b/tools/glusterfind/src/brickfind.py
index 439c8816f92..e914bacd4ab 100644
--- a/tools/glusterfind/src/brickfind.py
+++ b/tools/glusterfind/src/brickfind.py
@@ -62,9 +62,9 @@ def _get_args():
 
     parser.add_argument("session", help="Session Name")
     parser.add_argument("volume", help="Volume Name")
+    parser.add_argument("node", help="Node Name")
     parser.add_argument("brick", help="Brick Name")
     parser.add_argument("outfile", help="Output File")
-    parser.add_argument("start", help="Start Time", type=float)
     parser.add_argument("tag", help="Tag to prefix file name with")
     parser.add_argument("--only-query", help="Only query, No session update",
                         action="store_true")
diff --git a/tools/glusterfind/src/changelog.py b/tools/glusterfind/src/changelog.py
index 721b8d0ca3a..ec38bfa13b0 100644
--- a/tools/glusterfind/src/changelog.py
+++ b/tools/glusterfind/src/changelog.py
@@ -40,8 +40,6 @@ history_turn_time = 0
 logger = logging.getLogger()
 
 
-
-
 def pgfid_to_path(brick, changelog_data):
     """
     For all the pgfids in table, converts into path using recursive
@@ -94,7 +92,7 @@ def populate_pgfid_and_inodegfid(brick, changelog_data):
                 path = symlink_gfid_to_path(brick, gfid)
                 path = output_path_prepare(path, args)
                 changelog_data.gfidpath_update({"path1": path},
-                                                {"gfid": gfid})
+                                               {"gfid": gfid})
             except (IOError, OSError) as e:
                 logger.warn("Error converting to path: %s" % e)
                 continue
@@ -160,10 +158,10 @@ def gfid_to_path_using_pgfid(brick, changelog_data, args):
         try:
             path = symlink_gfid_to_path(brick, row[0])
             find(os.path.join(brick, path),
-                callback_func=output_callback,
-                filter_func=inode_filter,
-                ignore_dirs=ignore_dirs,
-                subdirs_crawl=False)
+                 callback_func=output_callback,
+                 filter_func=inode_filter,
+                 ignore_dirs=ignore_dirs,
+                 subdirs_crawl=False)
         except (IOError, OSError) as e:
             logger.warn("Error converting to path: %s" % e)
             continue
@@ -272,8 +270,8 @@ def get_changes(brick, hash_dir, log_file, start, end, args):
         actual_end = libgfchangelog.cl_history_changelog(
             cl_path, start, end, CHANGELOGAPI_NUM_WORKERS)
     except libgfchangelog.ChangelogException as e:
-        fail("%s Historical Changelogs not available: %s" % (brick, e),
-             logger=logger)
+        fail("%s: %s Historical Changelogs not available: %s" %
+             (args.node, brick, e), logger=logger)
 
     try:
         # scan followed by getchanges till scan returns zero.
@@ -296,7 +294,7 @@ def get_changes(brick, hash_dir, log_file, start, end, args):
                     libgfchangelog.cl_history_done(change)
                 except IOError as e:
                     logger.warn("Error parsing changelog file %s: %s" %
-                        (change, e))
+                                (change, e))
 
             changelog_data.commit()
     except libgfchangelog.ChangelogException as e:
@@ -350,9 +348,11 @@ def _get_args():
 
     parser.add_argument("session", help="Session Name")
     parser.add_argument("volume", help="Volume Name")
+    parser.add_argument("node", help="Node Name")
     parser.add_argument("brick", help="Brick Name")
     parser.add_argument("outfile", help="Output File")
     parser.add_argument("start", help="Start Time", type=int)
+    parser.add_argument("end", help="End Time", type=int)
     parser.add_argument("--only-query", help="Query mode only (no session)",
                         action="store_true")
     parser.add_argument("--debug", help="Debug", action="store_true")
@@ -387,6 +387,7 @@ if __name__ == "__main__":
 
     if args.only_query:
         start = args.start
+        end = args.end
     else:
         try:
             with open(status_file) as f:
@@ -394,7 +395,11 @@ if __name__ == "__main__":
         except (ValueError, OSError, IOError):
             start = args.start
 
-    end = int(time.time()) - get_changelog_rollover_time(args.volume)
+    # end time is optional; so a -1 may be sent to use the default method of
+    # identifying the end time
+    if end == -1:
+        end = int(time.time()) - get_changelog_rollover_time(args.volume)
+
     logger.info("%s Started Changelog Crawl - Start: %s End: %s" % (args.brick,
                                                                     start,
                                                                     end))
diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py
index e6199a17a3c..3d0f02a65d4 100644
--- a/tools/glusterfind/src/main.py
+++ b/tools/glusterfind/src/main.py
@@ -139,9 +139,11 @@ def run_cmd_nodes(task, args, **kwargs):
             cmd = [change_detector,
                    args.session,
                    args.volume,
+                   host,
                    brick,
-                   node_outfile,
-                   str(kwargs.get("start"))] + \
+                   node_outfile] + \
+                ([str(kwargs.get("start")), str(kwargs.get("end"))]
+                    if not args.full else []) + \
                 ([tag] if tag is not None else []) + \
                 ["--output-prefix", args.output_prefix] + \
                 (["--debug"] if args.debug else []) + \
@@ -174,9 +176,11 @@ def run_cmd_nodes(task, args, **kwargs):
             cmd = [change_detector,
                    args.session,
                    args.volume,
+                   host,
                    brick,
-                   node_outfile,
-                   str(kwargs.get("start"))] + \
+                   node_outfile] + \
+                ([str(kwargs.get("start")), str(kwargs.get("end"))]
+                    if not args.full else []) + \
                 ([tag] if tag is not None else []) + \
                 ["--only-query"] + \
                 ["--output-prefix", args.output_prefix] + \
@@ -362,6 +366,8 @@ def _get_args():
                               action=StoreAbsPath)
     parser_query.add_argument("--since-time", help="UNIX epoch time since "
                               "which listing is required", type=int)
+    parser_query.add_argument("--end-time", help="UNIX epoch time upto "
+                              "which listing is required", type=int)
     parser_query.add_argument("--no-encode",
                               help="Do not encode path in output file",
                               action="store_true")
@@ -569,27 +575,37 @@ def mode_query(session_dir, args):
     enable_volume_options(args)
 
     # Test options
-    if not args.since_time and not args.full:
-        fail("Please specify either --since-time or --full", logger=logger)
+    if not args.since_time and not args.end_time and not args.full:
+        fail("Please specify either {--since-time and optionally --end-time} "
+             "or --full", logger=logger)
 
-    if args.since_time and args.full:
-        fail("Please specify either --since-time or --full, but not both",
+    if args.since_time and args.end_time and args.full:
+        fail("Please specify either {--since-time and optionally --end-time} "
+             "or --full, but not both",
              logger=logger)
 
+    if args.end_time and not args.since_time:
+        fail("Please specify --since-time as well", logger=logger)
+
     # Start query command processing
+    start = -1
+    end = -1
     if args.since_time:
         start = args.since_time
+        if args.end_time:
+            end = args.end_time
     else:
         start = 0  # --full option is handled separately
 
     logger.debug("Query is called - Session: %s, Volume: %s, "
-                 "Start time: %s"
-                 % ("default", args.volume, start))
+                 "Start time: %s, End time: %s"
+                 % ("default", args.volume, start, end))
 
     prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
     gtmpfilename = prefix + next(tempfile._get_candidate_names())
 
-    run_cmd_nodes("query", args, start=start, tmpfilename=gtmpfilename)
+    run_cmd_nodes("query", args, start=start, end=end,
+                  tmpfilename=gtmpfilename)
 
     # Merger
     if args.full:
@@ -648,7 +664,7 @@ def mode_pre(session_dir, args):
     prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
     gtmpfilename = prefix + next(tempfile._get_candidate_names())
 
-    run_cmd_nodes("pre", args, start=start, tmpfilename=gtmpfilename)
+    run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename)
 
     # Merger
     if args.full: