diff options
Diffstat (limited to 'tests/utils/create-files.py')
| -rwxr-xr-x | tests/utils/create-files.py | 593 |
1 files changed, 593 insertions, 0 deletions
diff --git a/tests/utils/create-files.py b/tests/utils/create-files.py new file mode 100755 index 00000000000..04736e9c73b --- /dev/null +++ b/tests/utils/create-files.py @@ -0,0 +1,593 @@ + +# This script was developed by Vijaykumar Koppad (vkoppad@redhat.com) +# The latest version of this script can found at +# http://github.com/vijaykumar-koppad/crefi + +from __future__ import with_statement +import os +import re +import sys +import time +import errno +import xattr +import string +import random +import logging +import tarfile +import argparse + +datsiz = 0 +timr = 0 + +def get_ascii_upper_alpha_digits(): + if sys.version_info > (3,0): + return string.ascii_uppercase+string.digits + else: + return string.uppercase+string.digits + +def setLogger(filename): + global logger + logger = logging.getLogger(filename) + logger.setLevel(logging.DEBUG) + return + + +def setupLogger(filename): + logger = logging.getLogger(filename) + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(message)s') + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + ch.setFormatter(formatter) + logger.addHandler(ch) + return logger + + +def os_rd(src, size): + global datsiz + fd = os.open(src, os.O_RDONLY) + data = os.read(fd, size) + os.close(fd) + datsiz = datsiz + size + return data + + +def os_wr(dest, data): + global timr + st = time.time() + fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + os.write(fd, data) + os.close(fd) + ed = time.time() + timr = timr+(ed-st) + return + + +def create_sparse_file(fil, size, mins, maxs, rand): + if rand: + size = random.randint(mins, maxs) + else: + size = size + data = os_rd("/dev/zero", size) + os_wr(fil, data) + return + + +def create_binary_file(fil, size, mins, maxs, rand): + if rand: + size = random.randint(mins, maxs) + else: + size = size + data = os_rd("/dev/urandom", size) + os_wr(fil, data) + return + + +def create_txt_file(fil, size, mins, maxs, rand): + if rand: + size = random.randint(mins, maxs) + if size < 500*1024: + data = os_rd("/etc/services", size) + os_wr(fil, data) + else: + data = os_rd("/etc/services", 512*1024) + file_size = 0 + fd = os.open(fil, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + while file_size < size: + os.write(fd, data) + file_size += 500*1024 + os.close(fd) + return + + +def create_tar_file(fil, size, mins, maxs, rand): + if rand: + size = random.randint(mins, maxs) + else: + size = size + data = os_rd("/dev/urandom", size) + os_wr(fil, data) + tar = tarfile.open(fil+".tar.gz", "w:gz") + tar.add(fil) + tar.close() + os.unlink(fil) + return + + +def get_filename(flen): + size = flen + char = get_ascii_upper_alpha_digits() + st = ''.join(random.choice(char) for i in range(size)) + ti = str((hex(int(str(time.time()).split('.')[0])))[2:]) + return ti+"%%"+st + + +def text_files(files, file_count, inter, size, mins, maxs, rand, + flen, randname, dir_path): + global datsiz, timr + for k in range(files): + if not file_count % inter: + logger.info("Total files created -- "+str(file_count)) + if not randname: + fil = dir_path+"/"+"file"+str(k) + else: + fil = dir_path+"/"+get_filename(flen) + create_txt_file(fil, size, mins, maxs, rand) + file_count += 1 + return file_count + + +def sparse_files(files, file_count, inter, size, mins, maxs, + rand, flen, randname, dir_path): + for k in range(files): + if not file_count % inter: + logger.info("Total files created -- "+str(file_count)) + if not randname: + fil = dir_path+"/"+"file"+str(k) + else: + fil = dir_path+"/"+get_filename(flen) + create_sparse_file(fil, size, mins, maxs, rand) + file_count += 1 + return file_count + + +def binary_files(files, file_count, inter, size, mins, maxs, + rand, flen, randname, dir_path): + for k in range(files): + if not file_count % inter: + logger.info("Total files created -- "+str(file_count)) + if not randname: + fil = dir_path+"/"+"file"+str(k) + else: + fil = dir_path+"/"+get_filename(flen) + create_binary_file(fil, size, mins, maxs, rand) + file_count += 1 + return file_count + + +def tar_files(files, file_count, inter, size, mins, maxs, + rand, flen, randname, dir_path): + for k in range(files): + if not file_count % inter: + logger.info("Total files created -- "+str(file_count)) + if not randname: + fil = dir_path+"/"+"file"+str(k) + else: + fil = dir_path+"/"+get_filename(flen) + create_tar_file(fil, size, mins, maxs, rand) + file_count += 1 + return file_count + + +def setxattr_files(files, randname, dir_path): + char = get_ascii_upper_alpha_digits() + if not randname: + for k in range(files): + v = ''.join(random.choice(char) for i in range(10)) + n = "user."+v + xattr.setxattr(dir_path+"/"+"file"+str(k), n, v) + else: + dirs = os.listdir(dir_path+"/") + for fil in dirs: + v = ''.join(random.choice(char) for i in range(10)) + n = "user."+v + xattr.setxattr(dir_path+"/"+fil, n, v) + return + + +def rename_files(files, flen, randname, dir_path): + if not randname: + for k in range(files): + os.rename(dir_path + "/" + "file" + str(k), + dir_path + "/" + "file" + str(files+k)) + else: + dirs = os.listdir(dir_path) + for fil in dirs: + if not os.path.isdir(fil): + newfil = get_filename(flen) + os.rename(dir_path + "/" + fil, + dir_path + "/" + newfil) + return + + +def truncate_files(files, mins, maxs, randname, dir_path): + if not randname: + for k in range(files): + byts = random.randint(mins, maxs) + fd = os.open(dir_path + "/" + "file" + str(k), os.O_WRONLY) + os.ftruncate(fd, byts) + os.close(fd) + else: + dirs = os.listdir(dir_path) + for fil in dirs: + if not os.path.isdir(dir_path+"/"+fil): + byts = random.randint(mins, maxs) + fd = os.open(dir_path+"/"+fil, os.O_WRONLY) + os.ftruncate(fd, byts) + os.close(fd) + return + + +def chmod_files(files, flen, randname, dir_path): + if not randname: + for k in range(files): + mod = random.randint(0, 511) + os.chmod(dir_path+"/"+"file"+str(k), mod) + else: + dirs = os.listdir(dir_path) + for fil in dirs: + mod = random.randint(0, 511) + os.chmod(dir_path+"/"+fil, mod) + return + +def random_og(path): + u = random.randint(1025, 65536) + g = -1 + os.chown(path, u, g) + +def chown_files(files, flen, randname, dir_path): + if not randname: + for k in range(files): + random_og(dir_path+"/"+"file"+str(k)) + else: + dirs = os.listdir(dir_path) + for fil in dirs: + random_og(dir_path+"/"+fil) + return + + +def chgrp_files(files, flen, randname, dir_path): + if not randname: + for k in range(files): + random_og(dir_path+"/"+"file"+str(k)) + else: + dirs = os.listdir(dir_path) + for fil in dirs: + random_og(dir_path+"/"+fil) + return + + +def symlink_files(files, flen, randname, dir_path): + try: + os.makedirs(dir_path+"/"+"symlink_to_files") + except OSError as ex: + if ex.errno is not errno.EEXIST: + raise + if not randname: + for k in range(files): + src_file = "file"+str(k) + os.symlink(dir_path+"/"+src_file, + dir_path+"/"+"symlink_to_files/file"+str(k)+"_sym") + else: + dirs = os.listdir(dir_path) + for fil in dirs: + newfil = get_filename(flen) + os.symlink(dir_path+"/"+fil, + dir_path+"/"+"symlink_to_files/"+newfil) + return + + +def hardlink_files(files, flen, randname, dir_path): + try: + os.makedirs(dir_path+"/"+"hardlink_to_files") + except OSError as ex: + if ex.errno is not errno.EEXIST: + raise + if not randname: + for k in range(files): + src_file = "file"+str(k) + os.link(dir_path+"/"+src_file, + dir_path+"/"+"hardlink_to_files/file"+str(k)+"_hard") + else: + dirs = os.listdir(dir_path) + for fil in dirs: + if not os.path.isdir(dir_path+"/"+fil): + newfil = get_filename(flen) + os.link(dir_path+"/"+fil, + dir_path+"/"+"hardlink_to_files/"+newfil) + return + + +def human2bytes(size): + size_short = { + 1024: ['K', 'KB', 'KiB', 'k', 'kB', 'kiB'], + 1024*1024: ['M', 'MB', 'MiB'], + 1024*1024*1024: ['G', 'GB', 'GiB'] + } + num = re.search('(\d+)', size).group() + ext = size[len(num):] + num = int(num) + if ext == '': + return num + for value, keys in size_short.items(): + if ext in keys: + size = num*value + return size + + +def bytes2human(byts): + abbr = { + 1 << 30: "GB", + 1 << 20: "MB", + 1 << 10: "KB", + 1: "bytes" + } + if byts == 1: + return '1 bytes' + for factor, suffix in abbr.items(): + if byts >= factor: + break + return "%.3f %s" % (byts / factor, suffix) + + +def multipledir(mnt_pnt, brdth, depth, files, fop, file_type="text", + inter="1000", size="100K", mins="10K", maxs="500K", + rand=False, l=10, randname=False): + files_count = 1 + size = human2bytes(size) + maxs = human2bytes(maxs) + mins = human2bytes(mins) + for i in range(brdth): + dir_path = mnt_pnt + for j in range(depth): + dir_path = dir_path+"/"+"level"+str(j)+str(i) + try: + os.makedirs(dir_path) + except OSError as ex: + if ex.errno is not errno.EEXIST: + raise + + if fop == "create": + logger.info("Entering the directory level"+str(j)+str(i)) + if file_type == "text": + files_count = text_files(files, files_count, inter, size, + mins, maxs, rand, l, randname, + dir_path) + elif file_type == "sparse": + files_count = sparse_files(files, files_count, inter, size, + mins, maxs, rand, l, randname, + dir_path) + elif file_type == "binary": + files_count = binary_files(files, files_count, inter, size, + mins, maxs, rand, l, randname, + dir_path) + elif file_type == "tar": + files_count = tar_files(files, files_count, inter, size, + mins, maxs, rand, l, randname, + dir_path) + else: + logger.error("Not a valid file type") + sys.exit(1) + + elif fop == "rename": + logger.info("Started renaming files for the files 0 to " + + str(files)+" in the directory level"+str(j) + + str(i)+" ...") + rename_files(files, l, randname, dir_path) + logger.info("Finished renaming files for the files 0 to " + + str(files)+" in the directory level"+str(j)+str(i)) + + elif fop == "chmod": + logger.info("Started changing permission of files for the " + + "files 0 to "+str(files)+" in the directory level" + + str(j)+str(i)+" ...") + chmod_files(files, l, randname, dir_path) + logger.info("Finished changing permission of files for " + + "the files 0 to "+str(files) + + " in the directory level"+str(j)+str(i)) + + elif fop == "chown": + logger.info("Started changing ownership of files for the " + + "files 0 to " + str(files) + + " in the directory level"+str(j)+str(i)+" ...") + chown_files(files, l, randname, dir_path) + logger.info("Finished changing ownership of files for " + + "the files 0 to "+str(files) + + " in the directory level"+str(j)+str(i)) + + elif fop == "chgrp": + logger.info("Started changing group ownership of files for " + + "the files 0 to " + str(files) + + " in the directory level"+str(j)+str(i)+" ...") + chgrp_files(files, l, randname, dir_path) + logger.info("Finished changing group ownership of files for " + + "the files 0 to "+str(files) + + " in the directory level"+str(j)+str(i)) + + elif fop == "symlink": + logger.info("Started creating symlink to the files 0 to " + + str(files)+" in the directory level" + + str(j)+str(i)+"...") + symlink_files(files, l, randname, dir_path) + logger.info("Finished creating symlink to the files 0 to " + + str(files) + " in the directory level" + + str(j)+str(i)) + + elif fop == "hardlink": + logger.info("Started creating hardlink to the files 0 to " + + str(files)+" in the directory level" + + str(j)+str(i)+"...") + hardlink_files(files, l, randname, dir_path) + logger.info("Finished creating hardlink to the files 0 to " + + str(files) + " in the directory level" + + str(j)+str(i)) + + elif fop == "truncate": + logger.info("Started truncating the files 0 to " + + str(files)+" in the directory level" + + str(j)+str(i)+"...") + truncate_files(files, mins, maxs, randname, dir_path) + logger.info("Finished truncating the files 0 to " + + str(files)+" in the directory level" + + str(j)+str(i)) + + elif fop == "setxattr": + logger.info("Started setxattr to the files 0 to " + + str(files)+" in the directory level" + + str(j)+str(i)+"...") + setxattr_files(files, randname, dir_path) + logger.info("Finished setxattr to the files 0 to " + + str(files)+" in the directory level" + + str(j)+str(i)) + + if fop == "create": + thrpt = datsiz / timr + logger.info("finished creating files with throughput ---- " + + bytes2human(thrpt)+"ps") + + +def singledir(mnt_pnt, files, fop, file_type="text", inter="1000", size="100K", + mins="10K", maxs="500K", rand=False, l=10, randname=False): + + files_count = 1 + size = human2bytes(size) + maxs = human2bytes(maxs) + mins = human2bytes(mins) + if fop == "create": + if file_type == "text": + files_count = text_files(files, files_count, inter, size, mins, + maxs, rand, l, randname, mnt_pnt) + elif file_type == "sparse": + files_count = sparse_files(files, files_count, inter, size, mins, + maxs, rand, l, randname, mnt_pnt) + elif file_type == "binary": + files_count = binary_files(files, files_count, inter, size, mins, + maxs, rand, l, randname, mnt_pnt) + elif file_type == "tar": + files_count = tar_files(files, files_count, inter, size, mins, + maxs, rand, l, randname, mnt_pnt) + else: + logger.info("Not a valid file type") + sys.exit(1) + thrpt = datsiz / timr + logger.info("finished creating files with avg throughput ---- " + + bytes2human(thrpt)+"ps") + + elif fop == "rename": + logger.info("Started renaming files for the files 0 to " + + str(files) + "...") + rename_files(files, l, randname, mnt_pnt) + logger.info("Finished renaming files for the files 0 to "+str(files)) + + elif fop == "chmod": + logger.info("Started changing permission for the files 0 to " + + str(files)+" ...") + chmod_files(files, l, randname, mnt_pnt) + logger.info("Finished changing permission files for the files 0 to " + + str(files)) + + elif fop == "chown": + logger.info("Started changing ownership for the files 0 to " + + str(files)+"...") + chown_files(files, l, randname, mnt_pnt) + logger.info("Finished changing ownership for the files 0 to " + + str(files)) + + elif fop == "chgrp": + logger.info("Started changing group ownership for the files 0 to " + + str(files)+"...") + chgrp_files(files, l, randname, mnt_pnt) + logger.info("Finished changing group ownership for the files 0 to " + + str(files)) + + elif fop == "symlink": + logger.info("Started creating symlink to the files 0 to " + + str(files)+"...") + symlink_files(files, l, randname, mnt_pnt) + logger.info("Finished creating symlink to the files 0 to " + + str(files)) + + elif fop == "hardlink": + logger.info("Started creating hardlink to the files 0 to " + + str(files)+"...") + hardlink_files(files, l, randname, mnt_pnt) + logger.info("Finished creating hardlink to the files 0 to " + + str(files)) + + elif fop == "truncate": + logger.info("Started truncating the files 0 to " + str(files)+"...") + truncate_files(files, mins, maxs, randname, mnt_pnt) + logger.info("Finished truncating the files 0 to " + str(files)) + + elif fop == "setxattr": + logger.info("Started setxattr to the files 0 to " + str(files)+"...") + setxattr_files(files, randname, mnt_pnt) + logger.info("Finished setxattr to the files 0 to " + str(files)) + + +if __name__ == '__main__': + usage = "usage: %prog [option] <MNT_PT>" + parser = argparse.ArgumentParser(formatter_class=argparse. + ArgumentDefaultsHelpFormatter) + parser.add_argument("-n", dest="files", type=int, default=100, + help="number of files in each level ") + parser.add_argument("--size", action="store", default="100k", + help="size of the files to be used ") + parser.add_argument("--random", action="store_true", default=False, + help="random size of the file between --min and --max") + parser.add_argument("--max", action="store", default="500K", + help="maximum size of the files, if random is True") + parser.add_argument("--min", action="store", default="10K", + help="minimum size of the files, if random is True") + parser.add_argument("--single", action="store_true", dest="dir", + default=True, help="create files in single directory") + parser.add_argument("--multi", action="store_false", dest="dir", + help="create files in multiple directories") + parser.add_argument("-b", dest="brdth", type=int, default=5, + help="number of directories in one level(works " + + "with --multi) ") + parser.add_argument("-d", dest="depth", type=int, default=5, + help="number of levels of directories (works " + + "with --multi) ") + parser.add_argument("-l", dest="flen", type=int, default=10, + help="number of bytes for filename ( Used only when " + + "randname is enabled) ") + parser.add_argument("-t", action="store", dest="file_type", + default="text", choices=["text", "sparse", "binary", + "tar"], + help="type of the file to be created ()") + parser.add_argument("-I", dest="inter", type=int, default=100, + help="print number files created of interval") + parser.add_argument("--fop", action="store", dest="fop", default="create", + choices=["create", "rename", "chmod", "chown", "chgrp", + "symlink", "hardlink", "truncate", + "setxattr"], + help="fop to be performed on the files") + parser.add_argument("-R", dest="randname", action="store_false", + default=True, help="To disable random file name " + + "(default: Enabled)") + parser.add_argument("mntpnt", help="Mount point") + + args = parser.parse_args() + logger = setupLogger("testlost") + args.mntpnt = os.path.abspath(args.mntpnt) + + if args.dir: + singledir(args.mntpnt, args.files, args.fop, args.file_type, + args.inter, args.size, args.min, args.max, + args.random, args.flen, args.randname) + else: + multipledir(args.mntpnt, args.brdth, args.depth, args.files, + args.fop, args.file_type, args.inter, args.size, + args.min, args.max, args.random, args.flen, + args.randname) |