summaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/glusterfind/src/changelog.py18
-rw-r--r--tools/glusterfind/src/main.py175
2 files changed, 155 insertions, 38 deletions
diff --git a/tools/glusterfind/src/changelog.py b/tools/glusterfind/src/changelog.py
index 4d0a190286e..d6f3dc188ac 100644
--- a/tools/glusterfind/src/changelog.py
+++ b/tools/glusterfind/src/changelog.py
@@ -351,6 +351,8 @@ def _get_args():
parser.add_argument("brick", help="Brick Name")
parser.add_argument("outfile", help="Output File")
parser.add_argument("start", help="Start Time", type=int)
+ parser.add_argument("--only-query", help="Query mode only (no session)",
+ action="store_true")
parser.add_argument("--debug", help="Debug", action="store_true")
parser.add_argument("--output-prefix", help="File prefix in output",
default=".")
@@ -378,19 +380,23 @@ if __name__ == "__main__":
mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
logger=logger)
- try:
- with open(status_file) as f:
- start = int(f.read().strip())
- except (ValueError, OSError, IOError):
+ if args.only_query:
start = args.start
+ else:
+ try:
+ with open(status_file) as f:
+ start = int(f.read().strip())
+ except (ValueError, OSError, IOError):
+ start = args.start
end = int(time.time()) - get_changelog_rollover_time(args.volume)
logger.info("%s Started Changelog Crawl - Start: %s End: %s" % (args.brick,
start,
end))
actual_end = changelog_crawl(args.brick, start, end, args)
- with open(status_file_pre, "w", buffering=0) as f:
- f.write(str(actual_end))
+ if not args.only_query:
+ with open(status_file_pre, "w", buffering=0) as f:
+ f.write(str(actual_end))
logger.info("%s Finished Changelog Crawl - End: %s" % (args.brick,
actual_end))
diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py
index 9bc4872ad9b..6d03cbed5f2 100644
--- a/tools/glusterfind/src/main.py
+++ b/tools/glusterfind/src/main.py
@@ -126,6 +126,25 @@ def run_cmd_nodes(task, args, **kwargs):
opts["node_outfile"] = node_outfile
opts["copy_outfile"] = True
+ elif task == "query":
+ # If Full backup is requested or start time is zero, use brickfind
+ change_detector = conf.get_change_detector("changelog")
+ node_outfiles.append(node_outfile)
+
+ cmd = [change_detector,
+ args.session,
+ args.volume,
+ brick,
+ node_outfile,
+ str(kwargs.get("start"))] + \
+ ["--only-query"] + \
+ ["--output-prefix", args.output_prefix] + \
+ (["--debug"] if args.debug else []) + \
+ (["--only-namespace-changes"]
+ if args.only_namespace_changes else [])
+
+ opts["node_outfile"] = node_outfile
+ opts["copy_outfile"] = True
elif task == "cleanup":
# After pre run, cleanup the working directory and other temp files
# Remove the copied node_outfile in main node
@@ -271,6 +290,23 @@ def _get_args():
help="List only namespace changes",
action="store_true")
+ # query <VOLUME> <OUTFILE> --since-time <SINCE_TIME>
+ # [--output-prefix <OUTPUT_PREFIX>] [--full]
+ parser_pre = subparsers.add_parser('query')
+ parser_pre.add_argument("volume", help="Volume Name")
+ parser_pre.add_argument("outfile", help="Output File",
+ action=StoreAbsPath)
+ parser_pre.add_argument("--since-time", help="UNIX epoch time since which "
+ "listing is required", type=int)
+ parser_pre.add_argument("--debug", help="Debug", action="store_true")
+ parser_pre.add_argument("--disable-partial", help="Disable Partial find, "
+ "Fail when one node fails", action="store_true")
+ parser_pre.add_argument("--output-prefix", help="File prefix in output",
+ default=".")
+ parser_pre.add_argument("-N", "--only-namespace-changes",
+ help="List only namespace changes",
+ action="store_true")
+
# post <SESSION> <VOLUME>
parser_post = subparsers.add_parser('post')
parser_post.add_argument("session", help="Session Name")
@@ -333,6 +369,45 @@ def ssh_setup(args):
logger.info("Ssh key added to authorized_keys of Volume nodes")
+def enable_volume_options(args):
+ execute(["gluster", "volume", "set",
+ args.volume, "build-pgfid", "on"],
+ exit_msg="Failed to set volume option build-pgfid on",
+ logger=logger)
+ logger.info("Volume option set %s, build-pgfid on" % args.volume)
+
+ execute(["gluster", "volume", "set",
+ args.volume, "changelog.changelog", "on"],
+ exit_msg="Failed to set volume option "
+ "changelog.changelog on", logger=logger)
+ logger.info("Volume option set %s, changelog.changelog on"
+ % args.volume)
+
+ execute(["gluster", "volume", "set",
+ args.volume, "changelog.capture-del-path", "on"],
+ exit_msg="Failed to set volume option "
+ "changelog.capture-del-path on", logger=logger)
+ logger.info("Volume option set %s, changelog.capture-del-path on"
+ % args.volume)
+
+
+def write_output(args, outfilemerger):
+ with open(args.outfile, "a") as f:
+ for row in outfilemerger.get():
+ # Multiple paths in case of Hardlinks
+ paths = row[1].split(",")
+ row_2_rep = None
+ for p in paths:
+ if p == "":
+ continue
+ p_rep = p.replace("%2F%2F", "%2F")
+ if not row_2_rep:
+ row_2_rep = row[2].replace("%2F%2F", "%2F")
+ if p_rep == row_2_rep:
+ continue
+ f.write("%s %s %s\n" % (row[0], p_rep, row_2_rep))
+
+
def mode_create(session_dir, args):
logger.debug("Init is called - Session: %s, Volume: %s"
% (args.session, args.volume))
@@ -360,26 +435,7 @@ def mode_create(session_dir, args):
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
-
- execute(["gluster", "volume", "set",
- args.volume, "build-pgfid", "on"],
- exit_msg="Failed to set volume option build-pgfid on",
- logger=logger)
- logger.info("Volume option set %s, build-pgfid on" % args.volume)
-
- execute(["gluster", "volume", "set",
- args.volume, "changelog.changelog", "on"],
- exit_msg="Failed to set volume option "
- "changelog.changelog on", logger=logger)
- logger.info("Volume option set %s, changelog.changelog on"
- % args.volume)
-
- execute(["gluster", "volume", "set",
- args.volume, "changelog.capture-del-path", "on"],
- exit_msg="Failed to set volume option "
- "changelog.capture-del-path on", logger=logger)
- logger.info("Volume option set %s, changelog.capture-del-path on"
- % args.volume)
+ enable_volume_options(args)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
@@ -398,6 +454,59 @@ def mode_create(session_dir, args):
sys.exit(0)
+def mode_query(session_dir, args):
+ # Verify volume status
+ cmd = ["gluster", 'volume', 'info', args.volume, "--xml"]
+ _, data, _ = execute(cmd,
+ exit_msg="Failed to Run Gluster Volume Info",
+ logger=logger)
+ try:
+ tree = etree.fromstring(data)
+ statusStr = tree.find('volInfo/volumes/volume/statusStr').text
+ except (ParseError, AttributeError) as e:
+ fail("Invalid Volume: %s" % e, logger=logger)
+
+ if statusStr != "Started":
+ fail("Volume %s is not online" % args.volume, logger=logger)
+
+ mkdirp(session_dir, exit_on_err=True, logger=logger)
+ mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
+ logger=logger)
+ mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
+
+ # Configure cluster for pasword-less SSH
+ ssh_setup(args)
+
+ # Enable volume options for changelog capture
+ enable_volume_options(args)
+
+ # Start query command processing
+ if args.since_time:
+ start = args.since_time
+ logger.debug("Query is called - Session: %s, Volume: %s, "
+ "Start time: %s"
+ % ("default", args.volume, start))
+
+ run_cmd_nodes("query", args, start=start)
+
+ # Merger
+ # Read each Changelogs db and generate finaldb
+ create_file(args.outfile, exit_on_err=True, logger=logger)
+ outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
+ write_output(args, outfilemerger)
+
+ try:
+ os.remove(args.outfile + ".db")
+ except (IOError, OSError):
+ pass
+
+ run_cmd_nodes("cleanup", args)
+
+ sys.stdout.write("Generated output file %s\n" % args.outfile)
+ else:
+ fail("Please specify --since-time option")
+
+
def mode_pre(session_dir, args):
"""
Read from Session file and write to session.pre file
@@ -441,15 +550,7 @@ def mode_pre(session_dir, args):
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
- with open(args.outfile, "a") as f:
- for row in outfilemerger.get():
- # Multiple paths in case of Hardlinks
- paths = row[1].split(",")
- for p in paths:
- if p == "" or p.replace("%2F%2F","%2F") == \
- row[2].replace("%2F%2F","%2F"):
- continue
- f.write("%s %s %s\n" % (row[0], p, row[2]))
+ write_output(args, outfilemerger)
try:
os.remove(args.outfile + ".db")
@@ -566,18 +667,28 @@ def main():
args = _get_args()
mkdirp(conf.get_opt("session_dir"), exit_on_err=True)
+ # force the default session name if mode is "query"
+ if args.mode == "query":
+ args.session = "default"
+
if args.mode == "list":
session_dir = conf.get_opt("session_dir")
else:
session_dir = os.path.join(conf.get_opt("session_dir"),
args.session)
- if not os.path.exists(session_dir) and args.mode not in ["create",
- "list"]:
+ if not os.path.exists(session_dir) and \
+ args.mode not in ["create", "list", "query"]:
+ fail("Invalid session %s" % args.session)
+
+ # "default" is a system defined session name
+ if args.mode in ["create", "post", "pre", "delete"] and \
+ args.session == "default":
fail("Invalid session %s" % args.session)
vol_dir = os.path.join(session_dir, args.volume)
- if not os.path.exists(vol_dir) and args.mode not in ["create", "list"]:
+ if not os.path.exists(vol_dir) and args.mode not in \
+ ["create", "list", "query"]:
fail("Session %s not created with volume %s" %
(args.session, args.volume))