test-lz4-speed Namespace Reference

Functions
def	hashfile (hasher, fname, blocksize=65536)
def	log (text)
def	execute (command, print_command=True, print_output=False, print_error=True, param_shell=True)
def	does_command_exist (command)
def	send_email (emails, topic, text, have_mutt, have_mail)
def	send_email_with_attachments (branch, commit, last_commit, args, text, results_files, logFileName, have_mutt, have_mail)
def	git_get_branches ()
def	git_get_changes (branch, commit, last_commit)
def	get_last_results (resultsFileName)
def	benchmark_and_compare (branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, testFilePath, fileName, last_csize, last_cspeed, last_dspeed)
def	update_config_file (branch, commit)
def	double_check (branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName)
def	test_commit (branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)

Variables
string	script_version = 'v1.7.2 (2016-11-08)'
string	default_repo_url = 'https://github.com/lz4/lz4.git'
string	working_dir_name = 'speedTest'
string	working_path = os.getcwd() + '/' + working_dir_name
string	clone_path = working_path + '/' + 'lz4'
string	email_header = 'lz4_speedTest'
	pid = str(os.getpid())
bool	verbose = False
string	clang_version = "unknown"
string	gcc_version = "unknown"
	args = None
	parser = argparse.ArgumentParser()
	help
	default
	type
	float
	int
	dest
	action
	testFileNames = args.testFileNames.split()
list	testFilePaths = []
	fileName = os.path.expanduser(fileName)
def	have_mutt = does_command_exist("mutt -h")
def	have_mail = does_command_exist("mail -V")
	cwd
string	pidfile = "./speedTest.pid"
string	branch = ""
string	commit = ""
bool	first_time = True
	loadavg = os.getloadavg()[0]
def	branches = git_get_branches()
def	last_commit = update_config_file(branch, commit)
	stack = traceback.format_exc()
string	email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)

Function Documentation

benchmark_and_compare()

def test-lz4-speed.benchmark_and_compare	(		branch,
			commit,
			last_commit,
			args,
			executableName,
			md5sum,
			compilerVersion,
			resultsFileName,
			testFilePath,
			fileName,
			last_csize,
			last_cspeed,
			last_dspeed
	)

Definition at line 147 of file test-lz4-speed.py.

                          testFilePath, fileName, last_csize, last_cspeed, last_dspeed):
    sleepTime = 30
    while os.getloadavg()[0] > args.maxLoadAvg:
        log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds"
            % (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
        time.sleep(sleepTime)
    start_load = str(os.getloadavg())
    result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True)   
    end_load = str(os.getloadavg())
    linesExpected = args.lastCLevel + 1
    if len(result) != linesExpected:
        raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result)))
    with open(resultsFileName, "a") as myfile:
        myfile.write('%s %s %s md5=%s\n' % (branch, commit, compilerVersion, md5sum))
        myfile.write('\n'.join(result) + '\n')
        myfile.close()
        if (last_cspeed == None):
            log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName))
            return ""
        commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
        text = ""
        for i in range(0, min(len(cspeed), len(last_cspeed))):
            print("%s:%s -%d cSpeed=%6.2f cLast=%6.2f cDiff=%1.4f dSpeed=%6.2f dLast=%6.2f dDiff=%1.4f ratioDiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], float(last_csize[i])/csize[i], fileName))
            if (cspeed[i]/last_cspeed[i] < args.lowerLimit):
                text += "WARNING: %s -%d cSpeed=%.2f cLast=%.2f cDiff=%.4f %s\n" % (executableName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName)
            if (dspeed[i]/last_dspeed[i] < args.lowerLimit):
                text += "WARNING: %s -%d dSpeed=%.2f dLast=%.2f dDiff=%.4f %s\n" % (executableName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName)
            if (float(last_csize[i])/csize[i] < args.ratioLimit):
                text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName)
        if text:
            text = args.message + ("\nmaxLoadAvg=%s  load average at start=%s end=%s\n%s  last_commit=%s  md5=%s\n" % (args.maxLoadAvg, start_load, end_load, compilerVersion, last_commit, md5sum)) + text
        return text
 
 

References execute(), float, get_last_results(), len, min, capstone.range, and cmd_descs_generate.str.

Referenced by double_check().

does_command_exist()

def test-lz4-speed.does_command_exist

(

command

)

Definition at line 68 of file test-lz4-speed.py.

def does_command_exist(command):
    try:
        execute(command, verbose, False, False)
    except Exception:
        return False
    return True
 
 

References execute().

double_check()

def test-lz4-speed.double_check	(		branch,
			commit,
			args,
			executableName,
			md5sum,
			compilerVersion,
			resultsFileName,
			filePath,
			fileName
	)

Definition at line 193 of file test-lz4-speed.py.

def double_check(branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName):
    last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
    if not args.dry_run:
        text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
        if text:
            log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
            text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
    return text
 
 

References benchmark_and_compare(), and get_last_results().

Referenced by test_commit().

execute()

def test-lz4-speed.execute	(		command,
			print_command = True,
			print_output = False,
			print_error = True,
			param_shell = True
	)

Definition at line 48 of file test-lz4-speed.py.

def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
    if print_command:
        log("> " + command)
    popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd)
    stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout)
    stderr_lines = stderr_lines.decode("utf-8")
    stdout_lines = stdout_lines.decode("utf-8")
    if print_output:
        if stdout_lines:
            print(stdout_lines)
        if stderr_lines:
            print(stderr_lines)
    if popen.returncode is not None and popen.returncode != 0:
        if stderr_lines and not print_output and print_error:
            print(stderr_lines)
        raise RuntimeError(stdout_lines + stderr_lines)
    return (stdout_lines + stderr_lines).splitlines()
execute.cwd = None
 
 

Referenced by benchmark_and_compare(), does_command_exist(), git_get_branches(), git_get_changes(), send_email(), send_email_with_attachments(), and test_commit().

get_last_results()

def test-lz4-speed.get_last_results

(

resultsFileName

)

Definition at line 125 of file test-lz4-speed.py.

def get_last_results(resultsFileName):
    if not os.path.isfile(resultsFileName):
        return None, None, None, None
    commit = None
    csize = []
    cspeed = []
    dspeed = []
    with open(resultsFileName, 'r') as f:
        for line in f:
            words = line.split()
            if len(words) <= 4:   # branch + commit + compilerVer + md5
                commit = words[1]
                csize = []
                cspeed = []
                dspeed = []
            if (len(words) == 8) or (len(words) == 9):  # results: "filename" or "XX files"
                csize.append(int(words[1]))
                cspeed.append(float(words[3]))
                dspeed.append(float(words[5]))
    return commit, csize, cspeed, dspeed
 
 

References float, int, and len.

Referenced by benchmark_and_compare(), and double_check().

git_get_branches()

def test-lz4-speed.git_get_branches

(

)

Definition at line 106 of file test-lz4-speed.py.

def git_get_branches():
    execute('git fetch -p', verbose)
    branches = execute('git branch -rl', verbose)
    output = []
    for line in branches:
        if ("HEAD" not in line) and ("coverity_scan" not in line) and ("gh-pages" not in line):
            output.append(line.strip())
    return output
 
 

References execute().

git_get_changes()

def test-lz4-speed.git_get_changes	(		branch,
			commit,
			last_commit
	)

Definition at line 116 of file test-lz4-speed.py.

def git_get_changes(branch, commit, last_commit):
    fmt = '--format="%h: (%an) %s, %ar"'
    if last_commit is None:
        commits = execute('git log -n 10 %s %s' % (fmt, commit))
    else:
        commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
    return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)
 
 

References execute(), and cmd_descs_generate.str.

hashfile()

def test-lz4-speed.hashfile	(		hasher,
			fname,
			blocksize = 65536
	)

Definition at line 37 of file test-lz4-speed.py.

def hashfile(hasher, fname, blocksize=65536):
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(blocksize), b""):
            hasher.update(chunk)
    return hasher.hexdigest()
 
 

Referenced by test_commit().

log()

def test-lz4-speed.log

(

text

)

Definition at line 44 of file test-lz4-speed.py.

def log(text):
    print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
 
 

send_email()

def test-lz4-speed.send_email	(		emails,
			topic,
			text,
			have_mutt,
			have_mail
	)

Definition at line 76 of file test-lz4-speed.py.

def send_email(emails, topic, text, have_mutt, have_mail):
    logFileName = working_path + '/' + 'tmpEmailContent'
    with open(logFileName, "w") as myfile:
        myfile.writelines(text)
        myfile.close()
        if have_mutt:
            execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
        elif have_mail:
            execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
        else:
            log("e-mail cannot be sent (mail or mutt not found)")
 
 

References execute().

send_email_with_attachments()

def test-lz4-speed.send_email_with_attachments	(		branch,
			commit,
			last_commit,
			args,
			text,
			results_files,
			logFileName,
			have_mutt,
			have_mail
	)

Definition at line 89 of file test-lz4-speed.py.

                                logFileName, have_mutt, have_mail):
    with open(logFileName, "w") as myfile:
        myfile.writelines(text)
        myfile.close()
        email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
                      % (email_header, pid, branch, commit, last_commit,
                         args.lowerLimit, args.ratioLimit)
        if have_mutt:
            execute('mutt -s "' + email_topic + '" ' + args.emails + ' -a ' + results_files
                    + ' < ' + logFileName)
        elif have_mail:
            execute('mail -s "' + email_topic + '" ' + args.emails + ' < ' + logFileName)
        else:
            log("e-mail cannot be sent (mail or mutt not found)")
 
 

References execute().

Referenced by test_commit().

test_commit()

def test-lz4-speed.test_commit	(		branch,
			commit,
			last_commit,
			args,
			testFilePaths,
			have_mutt,
			have_mail
	)

Definition at line 203 of file test-lz4-speed.py.

def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
    local_branch = branch.split('/')[1]
    version = local_branch.rpartition('-')[2] + '_' + commit
    if not args.dry_run:
        execute('make -C programs clean lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DLZ4_GIT_COMMIT=%s" && ' % version +
                'mv programs/lz4 programs/lz4_clang && ' +
                'make -C programs clean lz4 lz4c32 MOREFLAGS="-DLZ4_GIT_COMMIT=%s"' % version)
    md5_lz4 = hashfile(hashlib.md5(), clone_path + '/programs/lz4')
    md5_lz4c32 = hashfile(hashlib.md5(), clone_path + '/programs/lz4c32')
    md5_lz4_clang = hashfile(hashlib.md5(), clone_path + '/programs/lz4_clang')
    print("md5(lz4)=%s\nmd5(lz4c32)=%s\nmd5(lz4_clang)=%s" % (md5_lz4, md5_lz4c32, md5_lz4_clang))
    print("gcc_version=%s clang_version=%s" % (gcc_version, clang_version))
 
    logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
    text_to_send = []
    results_files = ""
 
    for filePath in testFilePaths:
        fileName = filePath.rpartition('/')[2]
        resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
        text = double_check(branch, commit, args, 'lz4', md5_lz4, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
        if text:
            text_to_send.append(text)
            results_files += resultsFileName + " "
        resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
        text = double_check(branch, commit, args, 'lz4c32', md5_lz4c32, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
        if text:
            text_to_send.append(text)
            results_files += resultsFileName + " "
        resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
        text = double_check(branch, commit, args, 'lz4_clang', md5_lz4_clang, 'clang_version='+clang_version, resultsFileName, filePath, fileName)
        if text:
            text_to_send.append(text)
            results_files += resultsFileName + " "
    if text_to_send:
        send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail)
 
 

References double_check(), execute(), hashfile(), and send_email_with_attachments().

update_config_file()

def test-lz4-speed.update_config_file	(		branch,
			commit
	)

Definition at line 182 of file test-lz4-speed.py.

def update_config_file(branch, commit):
    last_commit = None
    commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
    if os.path.isfile(commitFileName):
        with open(commitFileName, 'r') as infile:
            last_commit = infile.read()
    with open(commitFileName, 'w') as outfile:
        outfile.write(commit)
    return last_commit
 
 

Variable Documentation

action

test-lz4-speed.action

Definition at line 253 of file test-lz4-speed.py.

Referenced by _server_handle_vCont(), actions2mask(), alone_decode(), alone_encode(), apprentice_1(), apprentice_load(), auto_decode(), block_decode(), block_encode(), coder_loop(), coder_normal(), compress(), copy_or_code(), decompress(), delta_decode(), delta_encode(), encode(), file_apprentice(), fill_window(), get_bin_info(), getstr(), getvalue(), handle_socket(), load_1(), load_b(), lz_decode(), lz_encode(), LZMA_API(), main(), parse(), parser__advance(), parser__do_potential_reductions(), rz_debug_kill_setup(), rz_main_rz_bin(), simple_code(), stack__iter(), stream_decode(), stream_encode(), stream_encode_in(), stream_encode_mt(), tar(), ts_language_next_state(), ts_parser__advance(), ts_parser__do_all_potential_reductions(), ts_query__analyze_patterns(), ts_reduce_action_set_add(), uncompress(), worker_encode(), xz_compress(), and xz_decompress().

args

test-lz4-speed.args = None

Definition at line 34 of file test-lz4-speed.py.

branch

string test-lz4-speed.branch = ""

Definition at line 317 of file test-lz4-speed.py.

Referenced by anop32(), bf_rlimit(), decode_known(), i8080_disasm(), INST_HANDLER(), pic_midrange_disassemble(), print_mips16_insn_arg(), rz_il_op_effect_free(), and rz_il_op_new_branch().

branches

def test-lz4-speed.branches = git_get_branches()

Definition at line 330 of file test-lz4-speed.py.

clang_version

def test-lz4-speed.clang_version = "unknown"

Definition at line 32 of file test-lz4-speed.py.

clone_path

string test-lz4-speed.clone_path = working_path + '/' + 'lz4'

Definition at line 28 of file test-lz4-speed.py.

commit

def test-lz4-speed.commit = ""

Definition at line 318 of file test-lz4-speed.py.

cwd

test-lz4-speed.cwd

Definition at line 300 of file test-lz4-speed.py.

Referenced by getcommapath(), main(), path_search_walk_ext(), rz_cmd_shell_cd_handler(), rz_cmd_shell_pwd_handler(), rz_comment_filelink_handler(), rz_file_abspath(), rz_file_abspath_rel(), rz_test_chdir_fromtest(), rz_test_main(), search_path(), search_path_join_test(), uv_fs_event_start(), uv_spawn(), and visual_comma().

default

test-lz4-speed.default

Definition at line 245 of file test-lz4-speed.py.

Referenced by d68020_cpgen(), ExtReadWrite_image(), fullSpeedBench(), get_ext_suffix(), and main().

default_repo_url

string test-lz4-speed.default_repo_url = 'https://github.com/lz4/lz4.git'

Definition at line 25 of file test-lz4-speed.py.

dest

test-lz4-speed.dest

Definition at line 253 of file test-lz4-speed.py.

email_header

string test-lz4-speed.email_header = 'lz4_speedTest'

Definition at line 29 of file test-lz4-speed.py.

email_topic

string test-lz4-speed.email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)

Definition at line 345 of file test-lz4-speed.py.

fileName

test-lz4-speed.fileName = os.path.expanduser(fileName)

Definition at line 262 of file test-lz4-speed.py.

Referenced by __already_loaded(), fileCheck(), GetSystemModules(), DotZLib.GZipStream.GZipStream(), and loadFile().

first_time

bool test-lz4-speed.first_time = True

Definition at line 319 of file test-lz4-speed.py.

float

test-lz4-speed.float

Definition at line 247 of file test-lz4-speed.py.

Referenced by benchmark_and_compare(), cmd_prc(), format_output(), get_last_results(), getvalue(), mprint(), rz_str_scale(), test-lz4-list.TestNonVerbose.test_ratio(), and test-lz4-list.TestVerbose.test_ratio().

gcc_version

def test-lz4-speed.gcc_version = "unknown"

Definition at line 33 of file test-lz4-speed.py.

have_mail

def test-lz4-speed.have_mail = does_command_exist("mail -V")

Definition at line 271 of file test-lz4-speed.py.

have_mutt

def test-lz4-speed.have_mutt = does_command_exist("mutt -h")

Definition at line 270 of file test-lz4-speed.py.

help

test-lz4-speed.help

(

void

)

Definition at line 243 of file test-lz4-speed.py.

int

test-lz4-speed.int

Definition at line 250 of file test-lz4-speed.py.

Referenced by get_last_results().

last_commit

def test-lz4-speed.last_commit = update_config_file(branch, commit)

Definition at line 333 of file test-lz4-speed.py.

loadavg

test-lz4-speed.loadavg = os.getloadavg()[0]

Definition at line 328 of file test-lz4-speed.py.

Referenced by uv_loadavg().

parser

test-lz4-speed.parser = argparse.ArgumentParser()

Definition at line 242 of file test-lz4-speed.py.

pid

test-lz4-speed.pid = str(os.getpid())

Definition at line 30 of file test-lz4-speed.py.

pidfile

string test-lz4-speed.pidfile = "./speedTest.pid"

Definition at line 308 of file test-lz4-speed.py.

script_version

string test-lz4-speed.script_version = 'v1.7.2 (2016-11-08)'

Definition at line 24 of file test-lz4-speed.py.

stack

test-lz4-speed.stack = traceback.format_exc()

Definition at line 344 of file test-lz4-speed.py.

testFileNames

test-lz4-speed.testFileNames = args.testFileNames.split()

Definition at line 259 of file test-lz4-speed.py.

testFilePaths

list test-lz4-speed.testFilePaths = []

Definition at line 260 of file test-lz4-speed.py.

type

test-lz4-speed.type

Definition at line 247 of file test-lz4-speed.py.

verbose

bool test-lz4-speed.verbose = False

Definition at line 31 of file test-lz4-speed.py.

working_dir_name

string test-lz4-speed.working_dir_name = 'speedTest'

Definition at line 26 of file test-lz4-speed.py.

working_path

string test-lz4-speed.working_path = os.getcwd() + '/' + working_dir_name

Definition at line 27 of file test-lz4-speed.py.