diff options
Diffstat (limited to 'resources')
| -rw-r--r-- | resources/tools/integrated/compare_perpatch.py | 187 |
1 files changed, 92 insertions, 95 deletions
diff --git a/resources/tools/integrated/compare_perpatch.py b/resources/tools/integrated/compare_perpatch.py index 9058332158..62e55629f6 100644 --- a/resources/tools/integrated/compare_perpatch.py +++ b/resources/tools/integrated/compare_perpatch.py @@ -28,103 +28,100 @@ import sys from resources.libraries.python import jumpavg -def hack(value_list): - """Return middle two quartiles, hoping to reduce influence of outliers. +def main(): + """Execute the main logic, return the code to return as return code. - Currently "middle two" is "all", but that can change in future. - - :param value_list: List to pick subset from. - :type value_list: list of float - :returns: New list containing middle values. - :rtype: list of float + :returns: Return code, 0 or 3 based on the comparison result. + :rtype: int """ - tmp = sorted(value_list) - eight = len(tmp) // 8 - ret = tmp[3*eight:-eight] - return tmp # ret - - -iteration = -1 -parent_iterations = list() -current_iterations = list() -num_tests = None -while 1: - iteration += 1 - parent_lines = list() - current_lines = list() - filename = f"csit_parent/{iteration}/results.txt" - try: - with open(filename) as parent_file: - parent_lines = parent_file.readlines() - except IOError: - break - num_lines = len(parent_lines) - filename = f"csit_current/{iteration}/results.txt" - with open(filename) as current_file: - current_lines = current_file.readlines() - if num_lines != len(current_lines): - print(f"Number of tests does not match within iteration {iteration}") - sys.exit(1) - if num_tests is None: - num_tests = num_lines - elif num_tests != num_lines: + iteration = -1 + parent_iterations = list() + current_iterations = list() + num_tests = None + while 1: + iteration += 1 + parent_lines = list() + current_lines = list() + filename = f"csit_parent/{iteration}/results.txt" + try: + with open(filename) as parent_file: + parent_lines = parent_file.readlines() + except IOError: + break + num_lines = len(parent_lines) + filename = f"csit_current/{iteration}/results.txt" + with open(filename) as current_file: + current_lines = current_file.readlines() + if num_lines != len(current_lines): + print( + f"Number of tests does not match within iteration {iteration}", + file=sys.stderr + ) + return 1 + if num_tests is None: + num_tests = num_lines + elif num_tests != num_lines: + print( + f"Number of tests does not match previous at iteration " + f"{iteration}", file=sys.stderr + ) + return 1 + parent_iterations.append(parent_lines) + current_iterations.append(current_lines) + exit_code = 0 + for test_index in range(num_tests): + parent_values = list() + current_values = list() + for iteration_index in range(len(parent_iterations)): + parent_values.extend( + json.loads(parent_iterations[iteration_index][test_index]) + ) + current_values.extend( + json.loads(current_iterations[iteration_index][test_index]) + ) + print(f"Time-ordered MRR values for parent build: {parent_values}") + print(f"Time-ordered MRR values for current build: {current_values}") + parent_values = sorted(parent_values) + current_values = sorted(current_values) + max_value = max([1.0] + parent_values + current_values) + parent_stats = jumpavg.AvgStdevStats.for_runs(parent_values) + current_stats = jumpavg.AvgStdevStats.for_runs(current_values) + parent_group_list = jumpavg.BitCountingGroupList( + max_value=max_value).append_group_of_runs([parent_stats]) + combined_group_list = parent_group_list.copy( + ).extend_runs_to_last_group([current_stats]) + separated_group_list = parent_group_list.append_group_of_runs( + [current_stats]) + print(f"Value-ordered MRR values for parent build: {parent_values}") + print(f"Value-ordered MRR values for current build: {current_values}") + avg_diff = (current_stats.avg - parent_stats.avg) / parent_stats.avg + print(f"Difference of averages relative to parent: {100 * avg_diff}%") + print(f"Jumpavg representation of parent group: {parent_stats}") + print(f"Jumpavg representation of current group: {current_stats}") print( - f"Number of tests does not match previous at iteration {iteration}" + f"Jumpavg representation of both as one group:" + f" {combined_group_list[0].stats}" ) - sys.exit(1) - parent_iterations.append(parent_lines) - current_iterations.append(current_lines) -exit_code = 0 -for test_index in range(num_tests): - parent_values = list() - current_values = list() - for iteration_index in range(len(parent_iterations)): - parent_values.extend( - json.loads(parent_iterations[iteration_index][test_index]) - ) - current_values.extend( - json.loads(current_iterations[iteration_index][test_index]) + bits_diff = separated_group_list.bits - combined_group_list.bits + compared = u"longer" if bits_diff >= 0 else u"shorter" + print( + f"Separate groups are {compared} than single group" + f" by {abs(bits_diff)} bits" ) - print(f"Time-ordered MRR values for parent build: {parent_values}") - print(f"Time-ordered MRR values for current build: {current_values}") - parent_values = hack(parent_values) - current_values = hack(current_values) - max_value = max([1.0] + parent_values + current_values) - parent_stats = jumpavg.AvgStdevStats.for_runs(parent_values) - current_stats = jumpavg.AvgStdevStats.for_runs(current_values) - parent_group_list = jumpavg.BitCountingGroupList( - max_value=max_value).append_group_of_runs([parent_stats]) - combined_group_list = parent_group_list.copy().extend_runs_to_last_group( - [current_stats]) - separated_group_list = parent_group_list.append_group_of_runs( - [current_stats]) - print(f"Value-ordered MRR values for parent build: {parent_values}") - print(f"Value-ordered MRR values for current build: {current_values}") - avg_diff = (current_stats.avg - parent_stats.avg) / parent_stats.avg - print(f"Difference of averages relative to parent: {100 * avg_diff}%") - print(f"Jumpavg representation of parent group: {parent_stats}") - print(f"Jumpavg representation of current group: {current_stats}") - print( - f"Jumpavg representation of both as one group:" - f" {combined_group_list[0].stats}" - ) - bits_diff = separated_group_list.bits - combined_group_list.bits - compared = u"longer" if bits_diff >= 0 else u"shorter" - print( - f"Separate groups are {compared} than single group" - f" by {abs(bits_diff)} bits" - ) - # TODO: Version of classify that takes max_value and list of stats? - # That matters if only stats (not list of floats) are given. - classified_list = jumpavg.classify([parent_values, current_values]) - if len(classified_list) < 2: - print(f"Test test_index {test_index}: normal (no anomaly)") - continue - anomaly = classified_list[1].comment - if anomaly == u"regression": - print(f"Test test_index {test_index}: anomaly regression") - exit_code = 3 # 1 or 2 can be caused by other errors - continue - print(f"Test test_index {test_index}: anomaly {anomaly}") -print(f"Exit code: {exit_code}") -sys.exit(exit_code) + # TODO: Version of classify that takes max_value and list of stats? + # That matters if only stats (not list of floats) are given. + classified_list = jumpavg.classify([parent_values, current_values]) + if len(classified_list) < 2: + print(f"Test test_index {test_index}: normal (no anomaly)") + continue + anomaly = classified_list[1].comment + if anomaly == u"regression": + print(f"Test test_index {test_index}: anomaly regression") + exit_code = 3 # 1 or 2 can be caused by other errors + continue + print(f"Test test_index {test_index}: anomaly {anomaly}") + print(f"Exit code: {exit_code}") + return exit_code + +if __name__ == u"__main__": + sys.exit(main()) |