1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com/>
# This file is part of GlusterFS.
#
# This file is licensed to you under your choice of the GNU Lesser
# General Public License, version 3 or any later version (LGPLv3 or
# later), or the GNU General Public License, version 2 (GPLv2), in all
# cases as published by the Free Software Foundation.
import os
import sys
import logging
from argparse import ArgumentParser, RawDescriptionHelpFormatter
try:
import urllib.parse as urllib
except ImportError:
import urllib
import time
from utils import mkdirp, setup_logger, create_file, output_write, find
import conf
PROG_DESCRIPTION = """
Changelog Crawler
"""
logger = logging.getLogger()
def brickfind_crawl(brick, args):
if brick.endswith("/"):
brick = brick[0:len(brick)-1]
working_dir = os.path.dirname(args.outfile)
mkdirp(working_dir, exit_on_err=True, logger=logger)
create_file(args.outfile, exit_on_err=True, logger=logger)
with open(args.outfile, "a+") as fout:
brick_path_len = len(brick)
def output_callback(path, filter_result, is_dir):
path = path.strip()
path = path[brick_path_len+1:]
if args.type == "both":
output_write(fout, path, args.output_prefix,
encode=(not args.no_encode), tag=args.tag,
field_separator=args.field_separator)
else:
if (is_dir and args.type == "d") or (
(not is_dir) and args.type == "f"):
output_write(fout, path, args.output_prefix,
encode=(not args.no_encode), tag=args.tag,
field_separator=args.field_separator)
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
find(brick, callback_func=output_callback,
ignore_dirs=ignore_dirs)
fout.flush()
os.fsync(fout.fileno())
def _get_args():
parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
description=PROG_DESCRIPTION)
parser.add_argument("session", help="Session Name")
parser.add_argument("volume", help="Volume Name")
parser.add_argument("node", help="Node Name")
parser.add_argument("brick", help="Brick Name")
parser.add_argument("outfile", help="Output File")
parser.add_argument("tag", help="Tag to prefix file name with")
parser.add_argument("--only-query", help="Only query, No session update",
action="store_true")
parser.add_argument("--debug", help="Debug", action="store_true")
parser.add_argument("--no-encode",
help="Do not encode path in outfile",
action="store_true")
parser.add_argument("--output-prefix", help="File prefix in output",
default=".")
parser.add_argument('--type', help="type: f, f-files only"
" d, d-directories only, by default = both",
default='both')
parser.add_argument("--field-separator", help="Field separator",
default=" ")
return parser.parse_args()
if __name__ == "__main__":
args = _get_args()
session_dir = os.path.join(conf.get_opt("session_dir"), args.session)
status_file = os.path.join(session_dir, args.volume,
"%s.status" % urllib.quote_plus(args.brick))
status_file_pre = status_file + ".pre"
mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
logger=logger)
mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume),
exit_on_err=True)
log_file = os.path.join(conf.get_opt("log_dir"),
args.session,
args.volume,
"brickfind.log")
setup_logger(logger, log_file, args.debug)
time_to_update = int(time.time())
brickfind_crawl(args.brick, args)
if not args.only_query:
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(time_to_update))
sys.exit(0)
|
