1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Script for analyzing 3 result sets for "git bisect" purposes.
Jumpavg library is used for comparing description length of three groupings.
The mid result is grouped with early or late result, or as a separate group.
The jump we are looking for is between the mid and the smaller group
of the grouping with less bits.
Except when a grouping with all three sets as separate groups is the smallest.
In that case we chose the bigger difference in averages.
"""
import sys
from typing import List, Tuple
from resources.libraries.python import jumpavg
from resources.libraries.python.model.parse import parse
def read_from_dir(dirname: str) -> Tuple[List[float], float]:
"""Parse samples from dir, print them and stats, return them as list.
In case there are more test cases, their results are concatenated.
:param direname: The directory name (maybe with path) to parse.
:type dirname: str
:returns: The samples, deserialized from json, and the average.
:rtype: Tuple[List[float], float]
:raises RuntimeError: On parsing error.
"""
results = parse(dirname)
samples = []
for result in results.values():
samples.extend(result)
print(f"Read {dirname}: {samples!r}")
stats = jumpavg.AvgStdevStats.for_runs(samples)
print(f"Stats: {stats!r}")
return samples, stats.avg
def main() -> int:
"""Execute the main logic, return the return code.
:returns: The return code, 0 or 3 depending on the comparison result.
:rtype: int
"""
early_results, early_avg = read_from_dir("csit_early")
late_results, late_avg = read_from_dir("csit_late")
mid_results, mid_avg = read_from_dir("csit_mid")
rel_diff_to_early = abs(early_avg - mid_avg) / max(early_avg, mid_avg)
rel_diff_to_late = abs(late_avg - mid_avg) / max(late_avg, mid_avg)
max_value = max(early_results + mid_results + late_results)
# Create a common group list with just the early group.
common_group_list = jumpavg.BitCountingGroupList(
max_value=max_value
).append_group_of_runs(early_results)
# Try grouping the mid with the early.
early_group_list = common_group_list.copy()
early_group_list.extend_runs_to_last_group(mid_results)
early_group_list.append_group_of_runs(late_results)
early_bits = early_group_list.bits
print(f"Early group list bits: {early_bits}")
# Now the same, but grouping the mid with the late.
late_group_list = common_group_list.copy()
late_group_list.append_group_of_runs(mid_results)
late_group_list.extend_runs_to_last_group(late_results)
late_bits = late_group_list.bits
print(f"Late group list bits: {late_bits}")
# Finally, group each separately, as if double anomaly happened.
double_group_list = common_group_list.copy()
double_group_list.append_group_of_runs(mid_results)
double_group_list.append_group_of_runs(late_results)
double_bits = double_group_list.bits
print(f"Double group list bits: {double_bits}")
single_bits = min(early_bits, late_bits)
if double_bits <= single_bits:
# In this case, comparing early_bits with late_bits is not the best,
# as that would probably select based on stdev, not based on diff.
# Example: mid (small stdev) is closer to early (small stdev),
# and farther from late (big stdev).
# As grouping mid with early would increase their combined stdev,
# it is not selected. This means a noisy late bound can affect
# what human perceives as the more interesting region.
# So we select only based on averages.
print("Perhaps two different anomalies. Selecting by averages only.")
diff = single_bits - double_bits
print(f"Saved {diff} ({100*diff/single_bits}%) bits.")
if rel_diff_to_early > rel_diff_to_late:
print("The mid results are considered late.")
print("Preferring relative difference of averages:")
print(f"{100*rel_diff_to_early}% to {100*rel_diff_to_late}%.")
# rc==1 is when command is not found.
# rc==2 is when python interpreter does not find the script.
exit_code = 3
else:
print("The mid results are considered early.")
print("Preferring relative difference of averages:")
print(f"{100*rel_diff_to_late}% to {100*rel_diff_to_early}%.")
exit_code = 0
else:
# When difference of averages is within stdev,
# we let jumpavg decide, as here difference in stdev
# can be the more interesting signal.
diff = early_bits - late_bits
if early_bits > late_bits:
print("The mid results are considered late.")
print(f"Saved {diff} ({100*diff/early_bits}%) bits.")
print(f"New relative difference is {100*rel_diff_to_early}%.")
exit_code = 3
else:
print("The mid results are considered early.")
print(f"Saved {-diff} ({-100*diff/late_bits}%) bits.")
print(f"New relative difference is {100*rel_diff_to_late}%.")
exit_code = 0
print(f"Exit code {exit_code}")
return exit_code
if __name__ == "__main__":
sys.exit(main())
|
