Discussion:
[PATCH 0/3] benchtests: keep comparing in case of missing fields
(too old to reply)
l***@linux.intel.com
2018-12-10 23:05:01 UTC
Permalink
From: Leonardo Sandoval <***@linux.intel.com>

Without this patch series, the script immediately quits the first time
a routine to compare has neither 'timings' nor 'min/mean' stats field,
so making it useless at least for bench-math benchset. Also, it includes
a new argument that allows user to define the desired statistic(s),
defaulting to min and mean.

Leonardo Sandoval (3):
benchtests: keep comparing even if function timings do not match
benchtests: keep comparing even if value does not exist
benchtests: send non-consumable data to stderr

benchtests/scripts/compare_bench.py | 34 +++++++++++++++++++----------
1 file changed, 22 insertions(+), 12 deletions(-)
--
2.19.2
l***@linux.intel.com
2018-12-10 23:05:02 UTC
Permalink
From: Leonardo Sandoval <***@linux.intel.com>

---
benchtests/scripts/compare_bench.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchtests/scripts/compare_bench.py b/benchtests/scripts/compare_bench.py
index f4b7742f90..9cbbda6be6 100755
--- a/benchtests/scripts/compare_bench.py
+++ b/benchtests/scripts/compare_bench.py
@@ -77,7 +77,7 @@ def compare_runs(pts1, pts2, threshold):
# timing info for the function variant.
if 'timings' not in pts1['functions'][func][var].keys() or \
'timings' not in pts2['functions'][func][var].keys():
- return
+ continue

# If two lists do not have the same length then it is likely that
# the performance characteristics of the function have changed.
--
2.19.2
l***@linux.intel.com
2018-12-10 23:05:04 UTC
Permalink
From: Leonardo Sandoval <***@linux.intel.com>

---
benchtests/scripts/compare_bench.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/benchtests/scripts/compare_bench.py b/benchtests/scripts/compare_bench.py
index add4023944..08c9034aee 100755
--- a/benchtests/scripts/compare_bench.py
+++ b/benchtests/scripts/compare_bench.py
@@ -47,6 +47,7 @@ def do_compare(func, var, tl1, tl2, par, threshold):
v2 = tl2[str(par)]
d = abs(v2 - v1) * 100 / v1
except KeyError:
+ sys.stderr.write('%s(%s)[%s]: stat does not exist\n' % (func, var, par))
return
except ZeroDivisionError:
return
@@ -85,7 +86,7 @@ def compare_runs(pts1, pts2, threshold, stats):
# timing info for the function variant.
if 'timings' not in pts1['functions'][func][var].keys() or \
'timings' not in pts2['functions'][func][var].keys():
- continue
+ continue

# If two lists do not have the same length then it is likely that
# the performance characteristics of the function have changed.
@@ -133,7 +134,7 @@ def plot_graphs(bench1, bench2):
# No point trying to print a graph if there are no detailed
# timings.
if u'timings' not in bench1['functions'][func][var].keys():
- print('Skipping graph for %s(%s)' % (func, var))
+ sys.stderr.write('Skipping graph for %s(%s)\n' % (func, var))
continue

pylab.clf()
@@ -157,7 +158,7 @@ def plot_graphs(bench1, bench2):
filename = "%s-%s.png" % (func, var)
else:
filename = "%s.png" % func
- print('Writing out %s' % filename)
+ sys.stderr.write('Writing out %s' % filename)
pylab.savefig(filename)

def main(bench1, bench2, schema, threshold, stats):
--
2.19.2
l***@linux.intel.com
2018-12-10 23:05:03 UTC
Permalink
From: Leonardo Sandoval <***@linux.intel.com>

---
benchtests/scripts/compare_bench.py | 27 ++++++++++++++++++---------
1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/benchtests/scripts/compare_bench.py b/benchtests/scripts/compare_bench.py
index 9cbbda6be6..add4023944 100755
--- a/benchtests/scripts/compare_bench.py
+++ b/benchtests/scripts/compare_bench.py
@@ -42,17 +42,25 @@ def do_compare(func, var, tl1, tl2, par, threshold):
threshold: The threshold for differences, beyond which the script should
print a warning.
"""
- d = abs(tl2[par] - tl1[par]) * 100 / tl1[str(par)]
+ try:
+ v1 = tl1[str(par)]
+ v2 = tl2[str(par)]
+ d = abs(v2 - v1) * 100 / v1
+ except KeyError:
+ return
+ except ZeroDivisionError:
+ return
+
if d > threshold:
- if tl1[par] > tl2[par]:
+ if v1 > v2:
ind = '+++'
else:
ind = '---'
print('%s %s(%s)[%s]: (%.2lf%%) from %g to %g' %
- (ind, func, var, par, d, tl1[par], tl2[par]))
+ (ind, func, var, par, d, v1, v2))


-def compare_runs(pts1, pts2, threshold):
+def compare_runs(pts1, pts2, threshold, stats):
"""Compare two benchmark runs

Args:
@@ -70,8 +78,8 @@ def compare_runs(pts1, pts2, threshold):

# Compare the consolidated numbers
# do_compare(func, var, tl1, tl2, 'max', threshold)
- do_compare(func, var, tl1, tl2, 'min', threshold)
- do_compare(func, var, tl1, tl2, 'mean', threshold)
+ for stat in stats.split():
+ do_compare(func, var, tl1, tl2, stat, threshold)

# Skip over to the next variant or function if there is no detailed
# timing info for the function variant.
@@ -152,7 +160,7 @@ def plot_graphs(bench1, bench2):
print('Writing out %s' % filename)
pylab.savefig(filename)

-def main(bench1, bench2, schema, threshold):
+def main(bench1, bench2, schema, threshold, stats):
bench1 = bench.parse_bench(bench1, schema)
bench2 = bench.parse_bench(bench2, schema)

@@ -161,7 +169,7 @@ def main(bench1, bench2, schema, threshold):
bench.compress_timings(bench1)
bench.compress_timings(bench2)

- compare_runs(bench1, bench2, threshold)
+ compare_runs(bench1, bench2, threshold, stats)


if __name__ == '__main__':
@@ -176,7 +184,8 @@ if __name__ == '__main__':
default=os.path.join(os.path.dirname(os.path.realpath(__file__)),'benchout.schema.json'),
help='JSON file to validate source/dest files (default: %(default)s)')
parser.add_argument('--threshold', default=10.0, type=float, help='Only print those with equal or higher threshold (default: %(default)s)')
+ parser.add_argument('--stats', default="min mean", type=str, help='Only consider the values from the specified statistics')

args = parser.parse_args()

- main(args.bench1, args.bench2, args.schema, args.threshold)
+ main(args.bench1, args.bench2, args.schema, args.threshold, args.stats)
--
2.19.2
Loading...